diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9d35e3f97f..f5c82fe002 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -5,7 +5,15 @@ ARG PYTHON_PACKAGE_MANAGER=conda FROM ${BASE} as pip-base +RUN apt update -y \ + && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ + # faiss dependencies + libblas-dev \ + liblapack-dev \ + && rm -rf /tmp/* /var/tmp/* /var/cache/apt/* /var/lib/apt/lists/*; + ENV DEFAULT_VIRTUAL_ENV=rapids +ENV RAPIDS_LIBUCX_PREFER_SYSTEM_LIBRARY=true FROM ${BASE} as conda-base @@ -13,6 +21,8 @@ ENV DEFAULT_CONDA_ENV=rapids FROM ${PYTHON_PACKAGE_MANAGER}-base +ARG TARGETARCH + ARG CUDA ENV CUDAARCHS="RAPIDS" ENV CUDA_VERSION="${CUDA_VERSION:-${CUDA}}" @@ -24,7 +34,29 @@ ENV PYTHONSAFEPATH="1" ENV PYTHONUNBUFFERED="1" ENV PYTHONDONTWRITEBYTECODE="1" +ENV HISTFILE="/home/coder/.cache/._bash_history" + +### +# sccache configuration +### +ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs" ENV SCCACHE_REGION="us-east-2" ENV SCCACHE_BUCKET="rapids-sccache-devs" -ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai" -ENV HISTFILE="/home/coder/.cache/._bash_history" +ENV SCCACHE_IDLE_TIMEOUT=0 + +### +# sccache-dist configuration +### +# Enable sccache-dist by default +ENV DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST=1 +# Compile locally if max retries exceeded +ENV SCCACHE_DIST_FALLBACK_TO_LOCAL_COMPILE=true +# Retry transient errors 4 times (for a total of 5 attempts) +ENV SCCACHE_DIST_MAX_RETRIES=4 +# 1hr 59min (to accommodate debug builds) +ENV SCCACHE_DIST_REQUEST_TIMEOUT=7140 +ENV SCCACHE_DIST_URL="https://${TARGETARCH}.linux.sccache.rapids.nvidia.com" + +# Build as much in parallel as possible +ENV INFER_NUM_DEVICE_ARCHITECTURES=1 +ENV MAX_DEVICE_OBJ_TO_COMPILE_IN_PARALLEL=20 diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda12.9-conda/devcontainer.json similarity index 80% rename from .devcontainer/cuda11.8-conda/devcontainer.json rename to .devcontainer/cuda12.9-conda/devcontainer.json index 76ce8599a8..f7565bbeaa 100644 --- a/.devcontainer/cuda11.8-conda/devcontainer.json +++ b/.devcontainer/cuda12.9-conda/devcontainer.json @@ -3,19 +3,26 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "11.8", + "CUDA": "12.9", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.12-cpp-mambaforge" } }, + "runArgs": [ + "--rm", + "--name", + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.12-cuda12.9-conda", + "--ulimit", + "nofile=500000" + ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.9-envs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cuvs,type=bind,consistency=consistent", @@ -24,7 +31,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda11.8-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.9-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda12.9-pip/devcontainer.json similarity index 69% rename from .devcontainer/cuda11.8-pip/devcontainer.json rename to .devcontainer/cuda12.9-pip/devcontainer.json index 3a126b36ef..b7b43b9b45 100644 --- a/.devcontainer/cuda11.8-pip/devcontainer.json +++ b/.devcontainer/cuda12.9-pip/devcontainer.json @@ -3,21 +3,35 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "11.8", + "CUDA": "12.9", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda11.8-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.12-cpp-cuda12.9-ucx1.19.0-openmpi5.0.7" } }, + "runArgs": [ + "--rm", + "--name", + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.12-cuda12.9-pip", + "--ulimit", + "nofile=500000" + ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.2": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + "ghcr.io/rapidsai/devcontainers/features/cuda:25.12": { + "version": "12.9", + "installcuBLAS": true, + "installcuSOLVER": true, + "installcuRAND": true, + "installcuSPARSE": true + }, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", + "ghcr.io/rapidsai/devcontainers/features/cuda", "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cuvs,type=bind,consistency=consistent", @@ -25,7 +39,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.9-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda13.0-conda/devcontainer.json similarity index 80% rename from .devcontainer/cuda12.0-conda/devcontainer.json rename to .devcontainer/cuda13.0-conda/devcontainer.json index 426aaef98f..f4e2e662eb 100644 --- a/.devcontainer/cuda12.0-conda/devcontainer.json +++ b/.devcontainer/cuda13.0-conda/devcontainer.json @@ -3,19 +3,26 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.0", + "CUDA": "13.0", "PYTHON_PACKAGE_MANAGER": "conda", - "BASE": "rapidsai/devcontainers:24.02-cpp-mambaforge-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.12-cpp-mambaforge" } }, + "runArgs": [ + "--rm", + "--name", + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.12-cuda13.0-conda", + "--ulimit", + "nofile=500000" + ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda13.0-envs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cuvs,type=bind,consistency=consistent", @@ -24,7 +31,7 @@ "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda12.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.conda/${localWorkspaceFolderBasename}-cuda13.0-envs,target=/home/coder/.conda/envs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda13.0-pip/devcontainer.json similarity index 69% rename from .devcontainer/cuda12.0-pip/devcontainer.json rename to .devcontainer/cuda13.0-pip/devcontainer.json index 1ef2fdcb64..1fd011180e 100644 --- a/.devcontainer/cuda12.0-pip/devcontainer.json +++ b/.devcontainer/cuda13.0-pip/devcontainer.json @@ -3,21 +3,35 @@ "context": "${localWorkspaceFolder}/.devcontainer", "dockerfile": "${localWorkspaceFolder}/.devcontainer/Dockerfile", "args": { - "CUDA": "12.0", + "CUDA": "13.0", "PYTHON_PACKAGE_MANAGER": "pip", - "BASE": "rapidsai/devcontainers:24.02-cpp-llvm16-cuda12.0-ubuntu22.04" + "BASE": "rapidsai/devcontainers:25.12-cpp-cuda13.0-ucx1.19.0-openmpi5.0.7" } }, + "runArgs": [ + "--rm", + "--name", + "${localEnv:USER:anon}-rapids-${localWorkspaceFolderBasename}-25.12-cuda13.0-pip", + "--ulimit", + "nofile=500000" + ], "hostRequirements": {"gpu": "optional"}, "features": { - "ghcr.io/rapidsai/devcontainers/features/ucx:24.2": {"version": "1.14.1"}, - "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:24.2": {} + "ghcr.io/rapidsai/devcontainers/features/cuda:25.12": { + "version": "13.0", + "installcuBLAS": true, + "installcuSOLVER": true, + "installcuRAND": true, + "installcuSPARSE": true + }, + "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils:25.12": {} }, "overrideFeatureInstallOrder": [ "ghcr.io/rapidsai/devcontainers/features/ucx", + "ghcr.io/rapidsai/devcontainers/features/cuda", "ghcr.io/rapidsai/devcontainers/features/rapids-build-utils" ], - "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs}"], + "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda13.0-venvs}"], "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"], "workspaceFolder": "/home/coder", "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/cuvs,type=bind,consistency=consistent", @@ -25,7 +39,7 @@ "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent", "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent", - "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" + "source=${localWorkspaceFolder}/../.local/share/${localWorkspaceFolderBasename}-cuda13.0-venvs,target=/home/coder/.local/share/venvs,type=bind,consistency=consistent" ], "customizations": { "vscode": { diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 407c5448e5..73320903c6 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,21 +1,40 @@ #cpp code owners cpp/ @rapidsai/cuvs-cpp-codeowners +examples/cpp/ @rapidsai/cuvs-cpp-codeowners +examples/c/ @rapidsai/cuvs-cpp-codeowners + +#java code owners +java/ @rapidsai/cuvs-java-codeowners +examples/java/ @rapidsai/cuvs-java-codeowners #python code owners python/ @rapidsai/cuvs-python-codeowners +#rust code owners +rust/ @rapidsai/cuvs-rust-codeowners +examples/rust/ @rapidsai/cuvs-rust-codeowners + +#docs code owners +docs/ @rapidsai/cuvs-docs-codeowners + #cmake code owners -**/CMakeLists.txt @rapidsai/cuvs-cmake-codeowners -**/cmake/ @rapidsai/cuvs-cmake-codeowners -python/setup.py @rapidsai/cuvs-cmake-codeowners -build.sh @rapidsai/cuvs-cmake-codeowners -**/build.sh @rapidsai/cuvs-cmake-codeowners +CMakeLists.txt @rapidsai/cuvs-cmake-codeowners +**/cmake/ @rapidsai/cuvs-cmake-codeowners +*.cmake @rapidsai/cuvs-cmake-codeowners +cpp/scripts/run-cmake-format.sh @rapidsai/cuvs-cmake-codeowners + +#CI code owners +/.github/ @rapidsai/ci-codeowners +/ci/ @rapidsai/ci-codeowners +/.shellcheckrc @rapidsai/ci-codeowners -#build/ops code owners -.github/ @rapidsai/ops-codeowners -ci/ @rapidsai/ops-codeowners -conda/ @rapidsai/ops-codeowners -**/Dockerfile @rapidsai/ops-codeowners -**/.dockerignore @rapidsai/ops-codeowners -docker/ @rapidsai/ops-codeowners -dependencies.yaml @rapidsai/ops-codeowners +#packaging code owners +/.pre-commit-config.yaml @rapidsai/packaging-codeowners +/.devcontainer/ @rapidsai/packaging-codeowners +/conda/ @rapidsai/packaging-codeowners +dependencies.yaml @rapidsai/packaging-codeowners +/build.sh @rapidsai/packaging-codeowners +pyproject.toml @rapidsai/packaging-codeowners +python/setup.py @rapidsai/packaging-codeowners +build.sh @rapidsai/packaging-codeowners +**/build.sh @rapidsai/packaging-codeowners diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9c42cda720..23d3a11feb 100755 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ .*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" "${FILE}" +done + +sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" java/README.md +sed_runner "s|-[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}\.jar|-${NEXT_FULL_JAVA_TAG}\.jar|g" java/examples/README.md +sed_runner "s|/[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}/|/${NEXT_FULL_JAVA_TAG}/|g" java/examples/README.md diff --git a/ci/run_ctests.sh b/ci/run_ctests.sh new file mode 100755 index 0000000000..6bf83961b9 --- /dev/null +++ b/ci/run_ctests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support customizing the ctests' install location +cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcuvs/" + +ctest --output-on-failure --no-tests=error "$@" diff --git a/ci/run_cuvs_pytests.sh b/ci/run_cuvs_pytests.sh new file mode 100755 index 0000000000..4de8927b17 --- /dev/null +++ b/ci/run_cuvs_pytests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Support invoking run_pytests.sh outside the script directory +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuvs/cuvs + +pytest --cache-clear --verbose "$@" tests diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 95ae6a69bd..477054b813 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,36 +1,40 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) + rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ - --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --file-key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --yes -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libcuvs libcuvs-tests - rapids-logger "Check GPU usage" nvidia-smi +# RAPIDS_DATASET_ROOT_DIR is used by test scripts +RAPIDS_DATASET_ROOT_DIR=${RAPIDS_TESTS_DIR}/dataset +export RAPIDS_DATASET_ROOT_DIR +./ci/get_test_data.sh --NEIGHBORS_ANN_VAMANA_TEST + EXITCODE=0 trap "EXITCODE=1" ERR set +e diff --git a/ci/test_java.sh b/ci/test_java.sh new file mode 100755 index 0000000000..20c19b6dee --- /dev/null +++ b/ci/test_java.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright (c) 2022-2025, NVIDIA CORPORATION. + +set -euo pipefail + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Check GPU usage" +nvidia-smi + +rapids-logger "Run Java build and tests" + +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +export RAPIDS_CUDA_MAJOR + +# TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs +# ref: https://github.com/rapidsai/cuvs/issues/868 +ci/build_java.sh --run-java-tests + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_python.sh b/ci/test_python.sh index a65469928c..beefebf6c4 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -1,38 +1,36 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. set -euo pipefail . /opt/conda/etc/profile.d/conda.sh +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-github cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-github python) + rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ - --file_key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --file-key test_python \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" \ + --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --force -f env.yaml -n test +rapids-mamba-retry env create --yes -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcuvs cuvs - rapids-logger "Check GPU usage" nvidia-smi @@ -43,14 +41,25 @@ set +e rapids-logger "pytest cuvs" pushd python/cuvs/cuvs pytest \ - --cache-clear \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cuvs.xml" \ - --cov-config=../.coveragerc \ - --cov=cuvs \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuvs-coverage.xml" \ - --cov-report=term \ - test + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cuvs.xml" \ + --cov-config=../.coveragerc \ + --cov=cuvs \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuvs-coverage.xml" \ + --cov-report=term \ + tests + +rapids-logger "pytest cuvs-bench" popd +pushd python/cuvs_bench/cuvs_bench +pytest \ + --cache-clear \ + --junitxml="${RAPIDS_TESTS_DIR}/junit-cuvs.xml" \ + --cov-config=../.coveragerc \ + --cov=cuvs \ + --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuvs-bench-coverage.xml" \ + --cov-report=term \ + tests rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index 6b213d399a..8ce684c11b 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -1,18 +1,20 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. set -euo pipefail -mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +source rapids-init-pip + +# Delete system libnccl.so to ensure the wheel is used +rm -rf /usr/lib64/libnccl* + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" +LIBCUVS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) +CUVS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github python) # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuvs*.whl)[test] +rapids-pip-retry install \ + "${LIBCUVS_WHEELHOUSE}"/libcuvs*.whl \ + "$(echo "${CUVS_WHEELHOUSE}"/cuvs*.whl)[test]" -# Run smoke tests for aarch64 pull requests -if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then - python ./ci/wheel_smoke_test_cuvs.py -else - python -m pytest ./python/cuvs/cuvs/test -fi +python -m pytest ./python/cuvs/cuvs/tests diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh new file mode 100755 index 0000000000..550ac8fb4c --- /dev/null +++ b/ci/validate_wheel.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir=$1 +wheel_dir_relative_path=$2 + +cd "${package_dir}" + +rapids-logger "validate packages with 'pydistcheck'" + +pydistcheck \ + --inspect \ + "${PYDISTCHECK_ARGS[@]}" \ + "$(echo "${wheel_dir_relative_path}"/*.whl)" + +rapids-logger "validate packages with 'twine'" + +twine check \ + --strict \ + "$(echo "${wheel_dir_relative_path}"/*.whl)" diff --git a/ci/wheel_smoke_test_cuvs.py b/ci/wheel_smoke_test_cuvs.py deleted file mode 100644 index 65b5fb8b2e..0000000000 --- a/ci/wheel_smoke_test_cuvs.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import numpy as np -from scipy.spatial.distance import cdist - -from pylibrat.common import Handle, Stream, device_ndarray -from cuvs.distance import pairwise_distance - - -if __name__ == "__main__": - metric = "euclidean" - n_rows = 1337 - n_cols = 1337 - - input1 = np.random.random_sample((n_rows, n_cols)) - input1 = np.asarray(input1, order="C").astype(np.float64) - - output = np.zeros((n_rows, n_rows), dtype=np.float64) - - expected = cdist(input1, input1, metric) - - expected[expected <= 1e-5] = 0.0 - - input1_device = device_ndarray(input1) - output_device = None - - s2 = Stream() - handle = Handle(stream=s2) - ret_output = pairwise_distance( - input1_device, input1_device, output_device, metric, handle=handle - ) - handle.sync() - - output_device = ret_output - - actual = output_device.copy_to_host() - - actual[actual <= 1e-5] = 0.0 - - assert np.allclose(expected, actual, rtol=1e-4) diff --git a/cmake/RAPIDS.cmake b/cmake/RAPIDS.cmake new file mode 100644 index 0000000000..ddef819498 --- /dev/null +++ b/cmake/RAPIDS.cmake @@ -0,0 +1,90 @@ +# ============================================================================= +# Copyright (c) 2021-2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +# +# This is the preferred entry point for projects using rapids-cmake +# +# Enforce the minimum required CMake version for all users +cmake_minimum_required(VERSION 3.30.4 FATAL_ERROR) + +# Allow users to control which version is used +if(NOT (rapids-cmake-branch OR rapids-cmake-version)) + message( + FATAL_ERROR "The CMake variable `rapids-cmake-branch` or `rapids-cmake-version` must be defined" + ) +endif() + +# Allow users to control which GitHub repo is fetched +if(NOT rapids-cmake-repo) + # Define a default repo if the user doesn't set one + set(rapids-cmake-repo rapidsai/rapids-cmake) +endif() + +# Allow users to control which branch is fetched +if(NOT rapids-cmake-branch) + # Define a default branch if the user doesn't set one + set(rapids-cmake-branch "release/${rapids-cmake-version}") +endif() + +# Allow users to control the exact URL passed to FetchContent +if(NOT rapids-cmake-url) + # Construct a default URL if the user doesn't set one + set(rapids-cmake-url "https://github.com/${rapids-cmake-repo}/") + + # In order of specificity + if(rapids-cmake-fetch-via-git) + if(rapids-cmake-sha) + # An exact git SHA takes precedence over anything + set(rapids-cmake-value-to-clone "${rapids-cmake-sha}") + elseif(rapids-cmake-tag) + # Followed by a git tag name + set(rapids-cmake-value-to-clone "${rapids-cmake-tag}") + else() + # Or if neither of the above two were defined, use a branch + set(rapids-cmake-value-to-clone "${rapids-cmake-branch}") + endif() + else() + if(rapids-cmake-sha) + # An exact git SHA takes precedence over anything + set(rapids-cmake-value-to-clone "archive/${rapids-cmake-sha}.zip") + elseif(rapids-cmake-tag) + # Followed by a git tag name + set(rapids-cmake-value-to-clone "archive/refs/tags/${rapids-cmake-tag}.zip") + else() + # Or if neither of the above two were defined, use a branch + set(rapids-cmake-value-to-clone "archive/refs/heads/${rapids-cmake-branch}.zip") + endif() + endif() +endif() + +include(FetchContent) +if(rapids-cmake-fetch-via-git) + FetchContent_Declare( + rapids-cmake + GIT_REPOSITORY "${rapids-cmake-url}" + GIT_TAG "${rapids-cmake-value-to-clone}" + ) +else() + string(APPEND rapids-cmake-url "${rapids-cmake-value-to-clone}") + FetchContent_Declare(rapids-cmake URL "${rapids-cmake-url}") +endif() +FetchContent_GetProperties(rapids-cmake) +if(rapids-cmake_POPULATED) + # Something else has already populated rapids-cmake, only thing we need to do is setup the + # CMAKE_MODULE_PATH + if(NOT "${rapids-cmake-dir}" IN_LIST CMAKE_MODULE_PATH) + list(APPEND CMAKE_MODULE_PATH "${rapids-cmake-dir}") + endif() +else() + FetchContent_MakeAvailable(rapids-cmake) +endif() diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake new file mode 100644 index 0000000000..b2c54a3f27 --- /dev/null +++ b/cmake/rapids_config.cmake @@ -0,0 +1,44 @@ +# ============================================================================= +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) +if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) + set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") + set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") + set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") + set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") +else() + string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") + message( + FATAL_ERROR + "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" + ) +endif() + +# Use STRINGS to trim whitespace/newlines +file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH" _rapids_branch) +if(NOT _rapids_branch) + message( + FATAL_ERROR + "Could not determine branch name to use for checking out rapids-cmake. The file \"${CMAKE_CURRENT_LIST_DIR}/../RAPIDS_BRANCH\" is missing." + ) +endif() + +if(NOT rapids-cmake-version) + set(rapids-cmake-version "${RAPIDS_VERSION_MAJOR_MINOR}") +endif() +if(NOT rapids-cmake-branch) + set(rapids-cmake-branch "${_rapids_branch}") +endif() +include("${CMAKE_CURRENT_LIST_DIR}/RAPIDS.cmake") diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml deleted file mode 100644 index dbf92ec475..0000000000 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- breathe -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 -- cuda-version=11.8 -- cudatoolkit -- cupy>=12.0.0 -- cxx-compiler -- cython>=3.0.0 -- doxygen>=1.8.20 -- gcc_linux-aarch64=11.* -- gmock>=1.13.0 -- graphviz -- gtest>=1.13.0 -- ipython -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- nccl>=2.9.9 -- ninja -- numpy>=1.21 -- numpydoc -- nvcc_linux-aarch64=11.8 -- pre-commit -- pydata-sphinx-theme -- pytest -- pytest-cov -- recommonmark -- rmm==24.2.* -- scikit-build>=0.13.1 -- scikit-learn -- scipy -- sphinx-copybutton -- sphinx-markdown-tables -- sysroot_linux-aarch64==2.17 -name: all_cuda-118_arch-aarch64 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml deleted file mode 100644 index 2fe184f964..0000000000 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ /dev/null @@ -1,54 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai -- rapidsai-nightly -- dask/label/dev -- conda-forge -- nvidia -dependencies: -- breathe -- c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4 -- cuda-nvtx=11.8 -- cuda-profiler-api=11.8.86 -- cuda-python>=11.7.1,<12.0a0 -- cuda-version=11.8 -- cudatoolkit -- cupy>=12.0.0 -- cxx-compiler -- cython>=3.0.0 -- doxygen>=1.8.20 -- gcc_linux-64=11.* -- gmock>=1.13.0 -- graphviz -- gtest>=1.13.0 -- ipython -- libcublas-dev=11.11.3.6 -- libcublas=11.11.3.6 -- libcurand-dev=10.3.0.86 -- libcurand=10.3.0.86 -- libcusolver-dev=11.4.1.48 -- libcusolver=11.4.1.48 -- libcusparse-dev=11.7.5.86 -- libcusparse=11.7.5.86 -- nccl>=2.9.9 -- ninja -- numpy>=1.21 -- numpydoc -- nvcc_linux-64=11.8 -- pre-commit -- pydata-sphinx-theme -- pytest -- pytest-cov -- recommonmark -- rmm==24.2.* -- scikit-build>=0.13.1 -- scikit-learn -- scipy -- sphinx-copybutton -- sphinx-markdown-tables -- sysroot_linux-64==2.17 -name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml new file mode 100644 index 0000000000..af56842a11 --- /dev/null +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -0,0 +1,56 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- breathe>=4.35.0 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=12.9.2,<13.0a0 +- cuda-version=12.9 +- cupy>=13.6.0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- doxygen>=1.8.20 +- gcc_linux-aarch64=14.* +- go +- graphviz +- ipython +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- librmm==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- numpy>=1.23,<3.0a0 +- numpydoc +- openblas +- pre-commit +- pylibraft==25.12.*,>=0.0.0a0 +- pytest +- pytest-cov +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- recommonmark +- rust +- scikit-build-core>=0.10.0 +- scikit-learn +- sphinx-copybutton +- sphinx-markdown-tables +- sphinx>=8.0.0 +- sysroot_linux-aarch64==2.28 +- pip: + - nvidia-sphinx-theme +name: all_cuda-129_arch-aarch64 diff --git a/conda/environments/all_cuda-120_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml similarity index 53% rename from conda/environments/all_cuda-120_arch-aarch64.yaml rename to conda/environments/all_cuda-129_arch-x86_64.yaml index 1b7f3908ad..44739ffd6a 100644 --- a/conda/environments/all_cuda-120_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -3,48 +3,54 @@ channels: - rapidsai - rapidsai-nightly -- dask/label/dev - conda-forge -- nvidia dependencies: -- breathe +- _go_select *=cgo +- breathe>=4.35.0 - c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4 +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 -- cuda-version=12.0 -- cupy>=12.0.0 +- cuda-python>=12.9.2,<13.0a0 +- cuda-version=12.9 +- cupy>=13.6.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 -- gcc_linux-aarch64=11.* -- gmock>=1.13.0 +- gcc_linux-64=14.* +- go - graphviz -- gtest>=1.13.0 - ipython +- libclang==20.1.4 - libcublas-dev - libcurand-dev - libcusolver-dev - libcusparse-dev -- nccl>=2.9.9 +- librmm==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 - ninja -- numpy>=1.21 +- numpy>=1.23,<3.0a0 - numpydoc +- openblas - pre-commit -- pydata-sphinx-theme +- pylibraft==25.12.*,>=0.0.0a0 - pytest - pytest-cov +- rapids-build-backend>=0.4.0,<0.5.0.dev0 - recommonmark -- rmm==24.2.* -- scikit-build>=0.13.1 +- rust +- scikit-build-core>=0.10.0 - scikit-learn -- scipy - sphinx-copybutton - sphinx-markdown-tables -- sysroot_linux-aarch64==2.17 -name: all_cuda-120_arch-aarch64 +- sphinx>=8.0.0 +- sysroot_linux-64==2.28 +- pip: + - nvidia-sphinx-theme +name: all_cuda-129_arch-x86_64 diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml new file mode 100644 index 0000000000..3a81210868 --- /dev/null +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -0,0 +1,56 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- breathe>=4.35.0 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=13.0.1,<14.0a0 +- cuda-version=13.0 +- cupy>=13.6.0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- doxygen>=1.8.20 +- gcc_linux-aarch64=14.* +- go +- graphviz +- ipython +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- librmm==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- numpy>=1.23,<3.0a0 +- numpydoc +- openblas +- pre-commit +- pylibraft==25.12.*,>=0.0.0a0 +- pytest +- pytest-cov +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- recommonmark +- rust +- scikit-build-core>=0.10.0 +- scikit-learn +- sphinx-copybutton +- sphinx-markdown-tables +- sphinx>=8.0.0 +- sysroot_linux-aarch64==2.28 +- pip: + - nvidia-sphinx-theme +name: all_cuda-130_arch-aarch64 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml similarity index 53% rename from conda/environments/all_cuda-120_arch-x86_64.yaml rename to conda/environments/all_cuda-130_arch-x86_64.yaml index 3352279942..2e93cf1489 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -3,48 +3,54 @@ channels: - rapidsai - rapidsai-nightly -- dask/label/dev - conda-forge -- nvidia dependencies: -- breathe +- _go_select *=cgo +- breathe>=4.35.0 - c-compiler -- clang-tools=16.0.6 -- clang==16.0.6 -- cmake>=3.26.4 +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-python>=12.0,<13.0a0 -- cuda-version=12.0 -- cupy>=12.0.0 +- cuda-python>=13.0.1,<14.0a0 +- cuda-version=13.0 +- cupy>=13.6.0 - cxx-compiler - cython>=3.0.0 +- dlpack>=0.8,<1.0 - doxygen>=1.8.20 -- gcc_linux-64=11.* -- gmock>=1.13.0 +- gcc_linux-64=14.* +- go - graphviz -- gtest>=1.13.0 - ipython +- libclang==20.1.4 - libcublas-dev - libcurand-dev - libcusolver-dev - libcusparse-dev -- nccl>=2.9.9 +- librmm==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 - ninja -- numpy>=1.21 +- numpy>=1.23,<3.0a0 - numpydoc +- openblas - pre-commit -- pydata-sphinx-theme +- pylibraft==25.12.*,>=0.0.0a0 - pytest - pytest-cov +- rapids-build-backend>=0.4.0,<0.5.0.dev0 - recommonmark -- rmm==24.2.* -- scikit-build>=0.13.1 +- rust +- scikit-build-core>=0.10.0 - scikit-learn -- scipy - sphinx-copybutton - sphinx-markdown-tables -- sysroot_linux-64==2.17 -name: all_cuda-120_arch-x86_64 +- sphinx>=8.0.0 +- sysroot_linux-64==2.28 +- pip: + - nvidia-sphinx-theme +name: all_cuda-130_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-129_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-129_arch-aarch64.yaml new file mode 100644 index 0000000000..5fe174ab6a --- /dev/null +++ b/conda/environments/bench_ann_cuda-129_arch-aarch64.yaml @@ -0,0 +1,49 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- benchmark>=1.8.2 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- click +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=12.9.2,<13.0a0 +- cuda-version=12.9 +- cupy>=13.6.0 +- cuvs==25.12.*,>=0.0.0a0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- gcc_linux-aarch64=14.* +- glog>=0.6.0 +- h5py>=3.8.0 +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- librmm==25.12.*,>=0.0.0a0 +- matplotlib-base>=3.9 +- nccl>=2.19 +- ninja +- nlohmann_json>=3.11.2 +- openblas +- pandas +- pylibraft==25.12.*,>=0.0.0a0 +- pyyaml +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- requests +- scikit-learn +- setuptools +- sysroot_linux-aarch64==2.28 +- wheel +name: bench_ann_cuda-129_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml new file mode 100644 index 0000000000..098e1765fa --- /dev/null +++ b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml @@ -0,0 +1,52 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- benchmark>=1.8.2 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- click +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=12.9.2,<13.0a0 +- cuda-version=12.9 +- cupy>=13.6.0 +- cuvs==25.12.*,>=0.0.0a0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- gcc_linux-64=14.* +- glog>=0.6.0 +- h5py>=3.8.0 +- libaio +- libboost-devel=1.87 +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- librmm==25.12.*,>=0.0.0a0 +- matplotlib-base>=3.9 +- mkl-devel=2023 +- nccl>=2.19 +- ninja +- nlohmann_json>=3.11.2 +- openblas +- pandas +- pylibraft==25.12.*,>=0.0.0a0 +- pyyaml +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- requests +- scikit-learn +- setuptools +- sysroot_linux-64==2.28 +- wheel +name: bench_ann_cuda-129_arch-x86_64 diff --git a/conda/environments/bench_ann_cuda-130_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-130_arch-aarch64.yaml new file mode 100644 index 0000000000..3a54fecedb --- /dev/null +++ b/conda/environments/bench_ann_cuda-130_arch-aarch64.yaml @@ -0,0 +1,49 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- benchmark>=1.8.2 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- click +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=13.0.1,<14.0a0 +- cuda-version=13.0 +- cupy>=13.6.0 +- cuvs==25.12.*,>=0.0.0a0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- gcc_linux-aarch64=14.* +- glog>=0.6.0 +- h5py>=3.8.0 +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- librmm==25.12.*,>=0.0.0a0 +- matplotlib-base>=3.9 +- nccl>=2.19 +- ninja +- nlohmann_json>=3.11.2 +- openblas +- pandas +- pylibraft==25.12.*,>=0.0.0a0 +- pyyaml +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- requests +- scikit-learn +- setuptools +- sysroot_linux-aarch64==2.28 +- wheel +name: bench_ann_cuda-130_arch-aarch64 diff --git a/conda/environments/bench_ann_cuda-130_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-130_arch-x86_64.yaml new file mode 100644 index 0000000000..95aa4d7a13 --- /dev/null +++ b/conda/environments/bench_ann_cuda-130_arch-x86_64.yaml @@ -0,0 +1,52 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- benchmark>=1.8.2 +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- click +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-python>=13.0.1,<14.0a0 +- cuda-version=13.0 +- cupy>=13.6.0 +- cuvs==25.12.*,>=0.0.0a0 +- cxx-compiler +- cython>=3.0.0 +- dlpack>=0.8,<1.0 +- gcc_linux-64=14.* +- glog>=0.6.0 +- h5py>=3.8.0 +- libaio +- libboost-devel=1.87 +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- librmm==25.12.*,>=0.0.0a0 +- matplotlib-base>=3.9 +- mkl-devel=2023 +- nccl>=2.19 +- ninja +- nlohmann_json>=3.11.2 +- openblas +- pandas +- pylibraft==25.12.*,>=0.0.0a0 +- pyyaml +- rapids-build-backend>=0.4.0,<0.5.0.dev0 +- requests +- scikit-learn +- setuptools +- sysroot_linux-64==2.28 +- wheel +name: bench_ann_cuda-130_arch-x86_64 diff --git a/conda/environments/go_cuda-129_arch-aarch64.yaml b/conda/environments/go_cuda-129_arch-aarch64.yaml new file mode 100644 index 0000000000..b8bf557877 --- /dev/null +++ b/conda/environments/go_cuda-129_arch-aarch64.yaml @@ -0,0 +1,32 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- cxx-compiler +- dlpack>=0.8,<1.0 +- gcc_linux-aarch64=14.* +- go +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- nccl>=2.19 +- ninja +- sysroot_linux-aarch64==2.28 +name: go_cuda-129_arch-aarch64 diff --git a/conda/environments/go_cuda-129_arch-x86_64.yaml b/conda/environments/go_cuda-129_arch-x86_64.yaml new file mode 100644 index 0000000000..adc12d644b --- /dev/null +++ b/conda/environments/go_cuda-129_arch-x86_64.yaml @@ -0,0 +1,32 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- cxx-compiler +- dlpack>=0.8,<1.0 +- gcc_linux-64=14.* +- go +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- nccl>=2.19 +- ninja +- sysroot_linux-64==2.28 +name: go_cuda-129_arch-x86_64 diff --git a/conda/environments/go_cuda-130_arch-aarch64.yaml b/conda/environments/go_cuda-130_arch-aarch64.yaml new file mode 100644 index 0000000000..ca450a317c --- /dev/null +++ b/conda/environments/go_cuda-130_arch-aarch64.yaml @@ -0,0 +1,32 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- cxx-compiler +- dlpack>=0.8,<1.0 +- gcc_linux-aarch64=14.* +- go +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- nccl>=2.19 +- ninja +- sysroot_linux-aarch64==2.28 +name: go_cuda-130_arch-aarch64 diff --git a/conda/environments/go_cuda-130_arch-x86_64.yaml b/conda/environments/go_cuda-130_arch-x86_64.yaml new file mode 100644 index 0000000000..5873836633 --- /dev/null +++ b/conda/environments/go_cuda-130_arch-x86_64.yaml @@ -0,0 +1,32 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- _go_select *=cgo +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- cxx-compiler +- dlpack>=0.8,<1.0 +- gcc_linux-64=14.* +- go +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- nccl>=2.19 +- ninja +- sysroot_linux-64==2.28 +name: go_cuda-130_arch-x86_64 diff --git a/conda/environments/rust_cuda-129_arch-aarch64.yaml b/conda/environments/rust_cuda-129_arch-aarch64.yaml new file mode 100644 index 0000000000..28d7701d68 --- /dev/null +++ b/conda/environments/rust_cuda-129_arch-aarch64.yaml @@ -0,0 +1,31 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- cxx-compiler +- gcc_linux-aarch64=14.* +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- rust +- sysroot_linux-aarch64==2.28 +name: rust_cuda-129_arch-aarch64 diff --git a/conda/environments/rust_cuda-129_arch-x86_64.yaml b/conda/environments/rust_cuda-129_arch-x86_64.yaml new file mode 100644 index 0000000000..a21932185b --- /dev/null +++ b/conda/environments/rust_cuda-129_arch-x86_64.yaml @@ -0,0 +1,31 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- cxx-compiler +- gcc_linux-64=14.* +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- rust +- sysroot_linux-64==2.28 +name: rust_cuda-129_arch-x86_64 diff --git a/conda/environments/rust_cuda-130_arch-aarch64.yaml b/conda/environments/rust_cuda-130_arch-aarch64.yaml new file mode 100644 index 0000000000..7533f45e23 --- /dev/null +++ b/conda/environments/rust_cuda-130_arch-aarch64.yaml @@ -0,0 +1,31 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- cxx-compiler +- gcc_linux-aarch64=14.* +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- rust +- sysroot_linux-aarch64==2.28 +name: rust_cuda-130_arch-aarch64 diff --git a/conda/environments/rust_cuda-130_arch-x86_64.yaml b/conda/environments/rust_cuda-130_arch-x86_64.yaml new file mode 100644 index 0000000000..0b4dbd7b09 --- /dev/null +++ b/conda/environments/rust_cuda-130_arch-x86_64.yaml @@ -0,0 +1,31 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- rapidsai-nightly +- conda-forge +dependencies: +- c-compiler +- clang-tools==20.1.4 +- clang==20.1.4 +- cmake>=3.30.4 +- cuda-cudart-dev +- cuda-nvcc +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- cxx-compiler +- gcc_linux-64=14.* +- libclang==20.1.4 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- libcuvs==25.12.*,>=0.0.0a0 +- libraft==25.12.*,>=0.0.0a0 +- make +- nccl>=2.19 +- ninja +- rust +- sysroot_linux-64==2.28 +name: rust_cuda-130_arch-x86_64 diff --git a/conda/recipes/cuvs-bench-cpu/conda_build_config.yaml b/conda/recipes/cuvs-bench-cpu/conda_build_config.yaml new file mode 100644 index 0000000000..b4e099cde8 --- /dev/null +++ b/conda/recipes/cuvs-bench-cpu/conda_build_config.yaml @@ -0,0 +1,23 @@ +c_compiler_version: + - 14 + +cxx_compiler_version: + - 14 + +c_stdlib: + - sysroot + +c_stdlib_version: + - "2.28" + +cmake_version: + - ">=3.30.4" + +glog_version: + - ">=0.6.0" + +h5py_version: + - ">=3.8.0" + +nlohmann_json_version: + - ">=3.11.2" diff --git a/conda/recipes/cuvs-bench-cpu/recipe.yaml b/conda/recipes/cuvs-bench-cpu/recipe.yaml new file mode 100644 index 0000000000..e856382c68 --- /dev/null +++ b/conda/recipes/cuvs-bench-cpu/recipe.yaml @@ -0,0 +1,111 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} + head_rev: '${{ git.head_rev(".")[:8] }}' + linux64: ${{ linux and x86_64 }} + +package: + name: cuvs-bench-cpu + version: ${{ version }} + +source: + path: ../../.. + +build: + string: py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + script: + content: | + + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x + + ./build.sh bench-ann --cpu-only --no-nvtx --build-metrics=bench_ann_cpu --incl-cache-stats + cmake --install cpp/build --component ann_bench + + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: cuvs-bench-cpu/${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - cmake ${{ cmake_version }} + - ninja + - ${{ stdlib("c") }} + host: + - benchmark + - glog ${{ glog_version }} + - nlohmann_json ${{ nlohmann_json_version }} + - openblas + - pip + - python =${{ py_version }} + - rapids-build-backend>=0.4.0,<0.5.0.dev0 + - setuptools >=64.0.0 + - if: linux64 + then: + - libaio + - libboost-devel=1.87 + - mkl-devel=2023 + run: + - benchmark + - click + - glog ${{ glog_version }} + - h5py ${{ h5py_version }} + - matplotlib-base>=3.9 + - numpy >=1.23,<3.0a0 + - pandas + - pyyaml + - python + - requests + - scikit-learn>=1.5 + - if: linux64 + then: + - mkl =2023 + ignore_run_exports: + by_name: + - glog + - libaio + - libboost + - libcublas + - mkl # relaxes pin from host `mkl-devel` + + +tests: + - python: + imports: + - cuvs_bench + - cuvs_bench.generate_groundtruth + - cuvs_bench.get_dataset + - cuvs_bench.plot + - cuvs_bench.run + - cuvs_bench.split_groundtruth + pip_check: false + +about: + homepage: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.license.text }} + summary: cuVS CPU benchmark diff --git a/conda/recipes/cuvs-bench/conda_build_config.yaml b/conda/recipes/cuvs-bench/conda_build_config.yaml new file mode 100644 index 0000000000..fb8f5e5407 --- /dev/null +++ b/conda/recipes/cuvs-bench/conda_build_config.yaml @@ -0,0 +1,5 @@ +glog_version: + - ">=0.6.0" + +h5py_version: + - ">=3.8.0" diff --git a/conda/recipes/cuvs-bench/recipe.yaml b/conda/recipes/cuvs-bench/recipe.yaml new file mode 100644 index 0000000000..fe12bea9ae --- /dev/null +++ b/conda/recipes/cuvs-bench/recipe.yaml @@ -0,0 +1,78 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} + head_rev: '${{ git.head_rev(".")[:8] }}' + linux64: ${{ linux and x86_64 }} + +package: + name: cuvs-bench + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + script: + content: | + python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true ./python/cuvs_bench + +requirements: + host: + - cuda-version =${{ cuda_version }} + - libcuvs =${{ version }} + - libcuvs-bench-ann =${{ version }} + - python =${{ py_version }} + - pip + - rapids-build-backend >=0.4.0,<0.5.0.dev0 + - rmm =${{ minor_version }} + - setuptools >=64.0.0 + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - benchmark + - click + - cuda-cudart + - cupy >=13.6.0 + - cuvs =${{ version }} + - glog ${{ glog_version }} + - h5py ${{ h5py_version }} + - libcublas + - libcuvs-bench-ann =${{ version }} + - matplotlib-base>=3.9 + - pandas + - pylibraft =${{ minor_version }} + - python + - pyyaml + - requests + # rmm is needed to determine if package is gpu-enabled + - rmm =${{ minor_version }} + - scikit-learn>=1.5 + - if: linux64 + then: mkl =2023 + ignore_run_exports: + by_name: + - cuda-version + - cuda-cudart + - glog + - libaio + - libboost + - libcublas + - mkl + +tests: + - package_contents: + files: + - lib/python*/site-packages/cuvs_bench/config/datasets/datasets.yaml + +about: + homepage: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.description }} diff --git a/conda/recipes/cuvs/build.sh b/conda/recipes/cuvs/build.sh deleted file mode 100644 index 81f762068b..0000000000 --- a/conda/recipes/cuvs/build.sh +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. -#!/usr/bin/env bash - -# This assumes the script is executed from the root of the repo directory -./build.sh python --no-nvtx diff --git a/conda/recipes/cuvs/conda_build_config.yaml b/conda/recipes/cuvs/conda_build_config.yaml index e28b98da7f..1082f0d214 100644 --- a/conda/recipes/cuvs/conda_build_config.yaml +++ b/conda/recipes/cuvs/conda_build_config.yaml @@ -1,17 +1,17 @@ c_compiler_version: - - 11 + - 14 cxx_compiler_version: - - 11 + - 14 cuda_compiler: - cuda-nvcc -cuda11_compiler: - - nvcc +c_stdlib: + - sysroot -sysroot_version: - - "2.17" +c_stdlib_version: + - "2.28" cmake_version: - - ">=3.26.4" + - ">=3.30.4" diff --git a/conda/recipes/cuvs/meta.yaml b/conda/recipes/cuvs/meta.yaml deleted file mode 100644 index f22bd01d51..0000000000 --- a/conda/recipes/cuvs/meta.yaml +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -# Usage: -# conda build . -c conda-forge -c numba -c rapidsai -c pytorch -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cuvs - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% endif %} - -requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - {% if cuda_major == "11" %} - - cuda-python >=11.7.1,<12.0a0 - - cudatoolkit - {% else %} - - cuda-python >=12.0,<13.0a0 - {% endif %} - - cuda-version ={{ cuda_version }} - - cython >=3.0.0 - - pylibraft {{ version }} - - libcuvs {{ version }} - - numpy >=1.21 - - python x.x - - rmm ={{ minor_version }} - - scikit-build >=0.13.1 - - setuptools - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% endif %} - - pylibraft {{ version }} - - libcuvs {{ version }} - - python x.x - - rmm ={{ minor_version }} - -tests: - requirements: - - cuda-version ={{ cuda_version }} - imports: - - cuvs - -about: - home: https://rapids.ai/ - license: Apache-2.0 - # license_file: LICENSE - summary: cuvs python library diff --git a/conda/recipes/cuvs/recipe.yaml b/conda/recipes/cuvs/recipe.yaml new file mode 100644 index 0000000000..f3fd05f8f3 --- /dev/null +++ b/conda/recipes/cuvs/recipe.yaml @@ -0,0 +1,89 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} + head_rev: '${{ git.head_rev(".")[:8] }}' + +package: + name: cuvs + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} + script: + content: | + ./build.sh python --no-nvtx -v + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: cuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + +requirements: + build: + - cmake ${{ cmake_version }} + - ninja + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - ${{ stdlib("c") }} + host: + - cuda-version =${{ cuda_version }} + - cython >=3.0.0 + - dlpack >=0.8 + - libcuvs =${{ version }} + - pip + - pylibraft =${{ minor_version }} + - python =${{ py_version }} + - rapids-build-backend >=0.4.0,<0.5.0.dev0 + - scikit-build-core >=0.10.0 + - if: cuda_major == "12" + then: cuda-python >=12.9.2,<13.0a0 + else: cuda-python >=13.0.1,<14.0a0 + - cuda-cudart-dev + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - libcuvs =${{ version }} + - pylibraft =${{ minor_version }} + - python + - numpy >=1.23,<3.0a0 + - if: cuda_major == "12" + then: cuda-python >=12.9.2,<13.0a0 + else: cuda-python >=13.0.1,<14.0a0 + - cuda-cudart + ignore_run_exports: + by_name: + - cuda-version + - cuda-cudart + +tests: + - python: + imports: + - cuvs + pip_check: false + +about: + homepage: ${{ load_from_file("python/cuvs/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuvs/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cuvs/pyproject.toml").project.description }} diff --git a/conda/recipes/libcuvs/build_libcuvs.sh b/conda/recipes/libcuvs/build_libcuvs.sh deleted file mode 100644 index cbde787a36..0000000000 --- a/conda/recipes/libcuvs/build_libcuvs.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -./build.sh libcuvs --allgpuarch --build-metrics=compile_lib --incl-cache-stats --no-nvtx diff --git a/conda/recipes/libcuvs/build_libcuvs_static.sh b/conda/recipes/libcuvs/build_libcuvs_static.sh deleted file mode 100644 index a107a9d6d1..0000000000 --- a/conda/recipes/libcuvs/build_libcuvs_static.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -./build.sh libcuvs --allgpuarch --compile-static-lib --build-metrics=compile_lib_static --incl-cache-stats --no-nvtx -n -cmake --install cpp/build --component compiled-static diff --git a/conda/recipes/libcuvs/build_libcuvs_template.sh b/conda/recipes/libcuvs/build_libcuvs_template.sh deleted file mode 100644 index bd7719af76..0000000000 --- a/conda/recipes/libcuvs/build_libcuvs_template.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -# Just building template so we verify it uses libraft.so and fail if it doesn't build -./build.sh template diff --git a/conda/recipes/libcuvs/build_libcuvs_tests.sh b/conda/recipes/libcuvs/build_libcuvs_tests.sh deleted file mode 100644 index fb116a6920..0000000000 --- a/conda/recipes/libcuvs/build_libcuvs_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -./build.sh tests --allgpuarch --no-nvtx --build-metrics=tests_bench --incl-cache-stats -cmake --install cpp/build --component testing diff --git a/conda/recipes/libcuvs/conda_build_config.yaml b/conda/recipes/libcuvs/conda_build_config.yaml index 73c3281e66..21989d9803 100644 --- a/conda/recipes/libcuvs/conda_build_config.yaml +++ b/conda/recipes/libcuvs/conda_build_config.yaml @@ -1,64 +1,23 @@ c_compiler_version: - - 11 + - 14 cxx_compiler_version: - - 11 + - 14 cuda_compiler: - cuda-nvcc -cuda11_compiler: - - nvcc +c_stdlib: + - sysroot -sysroot_version: - - "2.17" +c_stdlib_version: + - "2.28" cmake_version: - - ">=3.26.4" - -nccl_version: - - ">=2.9.9" - -gtest_version: - - ">=1.13.0" + - ">=3.30.4" h5py_version: - ">=3.8.0" -# The CTK libraries below are missing from the conda-forge::cudatoolkit package -# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages -# and the "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_libcublas_host_version: - - "=11.11.3.6" - -cuda11_libcublas_run_version: - - ">=11.5.2.43,<12.0.0" - -cuda11_libcurand_host_version: - - "=10.3.0.86" - -cuda11_libcurand_run_version: - - ">=10.2.5.43,<10.3.1" - -cuda11_libcusolver_host_version: - - "=11.4.1.48" - -cuda11_libcusolver_run_version: - - ">=11.2.0.43,<11.4.2" - -cuda11_libcusparse_host_version: - - "=11.7.5.86" - -cuda11_libcusparse_run_version: - - ">=11.6.0.43,<12.0.0" - -# `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all -# architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the -# "*_run_*" version specifiers correspond to `11.x` packages. - -cuda11_cuda_profiler_api_host_version: - - "=11.8.86" - -cuda11_cuda_profiler_api_run_version: - - ">=11.4.240,<12" +nccl_version: + - ">=2.19" diff --git a/conda/recipes/libcuvs/meta.yaml b/conda/recipes/libcuvs/meta.yaml deleted file mode 100644 index 4e437f3e62..0000000000 --- a/conda/recipes/libcuvs/meta.yaml +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. - -# Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} - {% set cuda_major = cuda_version.split('.')[0] %} - {% set cuda_spec = ">=" + cuda_major ~ ",<" + (cuda_major | int + 1) ~ ".0a0" %} # i.e. >=11,<12.0a0 - {% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: libcuvs-split - -source: - path: ../../.. - -outputs: - - name: libcuvs-static - version: {{ version }} - script: build_libcuvs_static.sh - build: - script_env: &script_env - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libcuvs-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libcuvs-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - libraft ={{ minor_version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} - {% else %} - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - {% endif %} - run: - - libraft ={{ minor_version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libcuvs static library - - name: libcuvs - version: {{ version }} - script: build_libcuvs.sh - build: - script_env: *script_env - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - libraft ={{ minor_version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_host_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} - {% else %} - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - {% endif %} - run: - - libraft ={{ minor_version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libcuvs library - - name: libcuvs-tests - version: {{ version }} - script: build_libcuvs_tests.sh - build: - script_env: *script_env - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - libcusolver {{ cuda11_libcusolver_host_version }} - - libcusolver-dev {{ cuda11_libcusolver_host_version }} - - libcusparse {{ cuda11_libcusparse_host_version }} - - libcusparse-dev {{ cuda11_libcusparse_host_version }} - {% else %} - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - {% endif %} - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% endif %} - - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} - - gmock {{ gtest_version }} - - gtest {{ gtest_version }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libcuvs tests - - name: libcuvs-template - version: {{ version }} - script: build_libcuvs_template.sh - build: - script_env: *script_env - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} - {% endif %} - requirements: - build: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda11') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - cmake {{ cmake_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - cuda-profiler-api {{ cuda11_cuda_profiler_api_run_version }} - - libcublas {{ cuda11_libcublas_host_version }} - - libcublas-dev {{ cuda11_libcublas_host_version }} - {% else %} - - cuda-profiler-api - - libcublas-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - {% endif %} - - libraft ={{ minor_version }} - - libcuvs ={{ minor_version }} - about: - home: https://rapids.ai/ - license: Apache-2.0 - summary: libcuvs template diff --git a/conda/recipes/libcuvs/recipe.yaml b/conda/recipes/libcuvs/recipe.yaml new file mode 100644 index 0000000000..cd5f47050e --- /dev/null +++ b/conda/recipes/libcuvs/recipe.yaml @@ -0,0 +1,344 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + head_rev: '${{ git.head_rev(".")[:8] }}' + linux64: ${{ linux and x86_64 }} + +recipe: + name: libcuvs-split + +cache: + source: + path: ../../.. + + build: + script: + content: | + + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x + + ./build.sh libcuvs bench-ann tests --allgpuarch --build-metrics=compile_lib --incl-cache-stats --no-nvtx -n + + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - cmake ${{ cmake_version }} + - ninja + - ${{ stdlib("c") }} + host: + - librmm =${{ minor_version }} + - libraft-headers =${{ minor_version }} + - nccl ${{ nccl_version }} + - cuda-version =${{ cuda_version }} + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + # These are used for bench-ann + - openblas + - if: linux64 + then: + - libaio + - libboost-devel =1.87 + - mkl-devel =2023 + +outputs: + - package: + name: libcuvs + version: ${{ version }} + build: + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" + prefix_detection: + ignore_binary_files: True + script: + content: | + cmake --install cpp/build --component cuvs + cmake --install cpp/build --component c_api + cmake --install cpp/build --component hnswlib + requirements: + build: + - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} + host: + - librmm =${{ minor_version }} + - libraft-headers =${{ minor_version }} + - nccl ${{ nccl_version }} + - cuda-version =${{ cuda_version }} + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + run: + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - libraft-headers =${{ minor_version }} + - nccl + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - libaio + - libboost + - libcublas + - libcurand + - libcusolver + - libcusparse + - librmm + - mkl + - nccl + about: + homepage: ${{ load_from_file("python/libcuvs/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcuvs/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/libcuvs/pyproject.toml").project.description }} + + - package: + name: libcuvs-tests + version: ${{ version }} + build: + script: cmake --install cpp/build --component testing + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" + prefix_detection: + ignore_binary_files: True + requirements: + build: + - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} + host: + - librmm =${{ minor_version }} + - libraft-headers =${{ minor_version }} + - nccl ${{ nccl_version }} + - ${{ pin_subpackage("libcuvs", exact=True) }} + - cuda-version =${{ cuda_version }} + - openblas # required by some CPU algos in benchmarks + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + run: + - ${{ pin_subpackage("libcuvs", exact=True) }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - libraft-headers =${{ minor_version }} + - nccl + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - libaio + - libboost + - libcublas + - libcurand + - libcusolver + - libcusparse + - librmm + - mkl + - nccl + about: + homepage: ${{ load_from_file("python/libcuvs/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcuvs/pyproject.toml").project.license.text }} + summary: libcuvs test executables + + - package: + name: libcuvs-examples + version: ${{ version }} + build: + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" + prefix_detection: + ignore_binary_files: True + script: + content: | + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x + + ./build.sh examples + + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: libcuvs/${{ env.get("RAPIDS_CONDA_ARCH") }}/cuda${{ cuda_major }} + requirements: + build: + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - cmake ${{ cmake_version }} + - ninja + - ${{ stdlib("c") }} + host: + - ${{ pin_subpackage("libcuvs", exact=True) }} + - cuda-version =${{ cuda_version }} + - libraft-headers =${{ minor_version }} + - librmm =${{ minor_version }} + - nccl ${{ nccl_version }} + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + run: + - ${{ pin_subpackage("libcuvs", exact=True) }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - nccl + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - libaio + - libboost + - libcublas + - libcurand + - libcusolver + - libcusparse + - librmm + - mkl + - nccl + about: + homepage: ${{ load_from_file("python/libcuvs/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcuvs/pyproject.toml").project.license.text }} + summary: libcuvs example executables + + - package: + name: libcuvs-bench-ann + version: ${{ version }} + build: + script: + content: | + cmake --install cpp/build --component ann_bench + + if [ "$RAPIDS_CONDA_ARCH" != "aarch64" ]; then + # Need this so from the diskann compile, but there isn't a + # component associated with it. Probably a better way to do this. + cp cpp/build/_deps/diskann-build/src/libdiskann.so $PREFIX/lib/ + fi + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" + prefix_detection: + ignore_binary_files: True + requirements: + build: + - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} + host: + - ${{ pin_subpackage("libcuvs", exact=True) }} + - cuda-cudart-dev + - cuda-profiler-api + - cuda-version =${{ cuda_version }} + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + - libgomp + - libraft-headers =${{ minor_version }} + - librmm =${{ minor_version }} + - nccl ${{ nccl_version }} + - openblas # required by some CPU algos in benchmarks + - if: linux64 + then: + - libaio + - libboost-devel =1.87 + - mkl-devel =2023 + run: + - ${{ pin_subpackage("libcuvs", exact=True) }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - cuda-cudart + - libcublas + - libcurand + - libcusolver + - libcusparse + - libraft-headers =${{ minor_version }} + - nccl + - if: linux64 + then: mkl =2023 + ignore_run_exports: + by_name: + - cuda-cudart + - cuda-version + - libcublas + - libcurand + - libcusolver + - libcusparse + - librmm + - mkl + - nccl + about: + homepage: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cuvs_bench/pyproject.toml").project.license.text }} + summary: libcuvs benchmark diff --git a/cpp/.clang-format b/cpp/.clang-format index 18f376d660..26b9a5bf4c 100644 --- a/cpp/.clang-format +++ b/cpp/.clang-format @@ -27,7 +27,7 @@ AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None AlwaysBreakBeforeMultilineStrings: true AlwaysBreakTemplateDeclarations: Yes -BinPackArguments: false +BinPackArguments: false BinPackParameters: false BraceWrapping: AfterClass: false diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 3c9cd6d1b7..bd9e29fc17 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2020-2024, NVIDIA CORPORATION. +# Copyright (c) 2020-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -10,11 +10,8 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under # the License. -set(RAPIDS_VERSION "24.02") -set(CUVS_VERSION "24.02.00") - -cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../fetch_rapids.cmake) +cmake_minimum_required(VERSION 3.30.4 FATAL_ERROR) +include(../cmake/rapids_config.cmake) include(rapids-cmake) include(rapids-cpm) include(rapids-export) @@ -24,22 +21,23 @@ option(BUILD_CPU_ONLY "Build CPU only components. Applies to CUVS benchmarks cur # workaround for rapids_cuda_init_architectures not working for arch detection with # enable_language(CUDA) -set(lang_list "CXX") +set(lang_list "CXX" "C") if(NOT BUILD_CPU_ONLY) include(rapids-cuda) - rapids_cuda_init_architectures(cuVS) + rapids_cuda_init_architectures(CUVS) list(APPEND lang_list "CUDA") endif() project( - cuVS - VERSION ${CUVS_VERSION} + CUVS + VERSION "${RAPIDS_VERSION}" LANGUAGES ${lang_list} ) +set(CMAKE_INSTALL_MESSAGE LAZY) # Write the version header -rapids_cmake_write_version_file(include/cuvs/version_config.hpp) +rapids_cmake_write_version_file(include/cuvs/version_config.h) # ################################################################################################## # * build type --------------------------------------------------------------- @@ -55,31 +53,51 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(BUILD_SHARED_LIBS "Build cuvs shared libraries" ON) option(BUILD_TESTS "Build cuvs unit-tests" ON) -option(BUILD_C_LIBRARY "Build raft C API library" OFF) -option(BUILD_C_TESTS "Build raft C API tests" OFF) +option(BUILD_C_LIBRARY "Build cuVS C API library" ON) +option(BUILD_CUVS_BENCH "Build cuVS ann benchmarks" OFF) +option(BUILD_CAGRA_HNSWLIB "Build CAGRA+hnswlib interface" ON) +option(BUILD_MG_ALGOS "Build with multi-GPU support" ON) option(CUDA_ENABLE_KERNELINFO "Enable kernel resource usage info" OFF) option(CUDA_ENABLE_LINEINFO "Enable the -lineinfo option for nvcc (useful for cuda-memcheck / profiler)" OFF ) -option(CUDA_STATIC_RUNTIME "Statically link the CUDA toolkit runtime and libraries" OFF) +option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) +option(CUDA_STATIC_MATH_LIBRARIES "Statically link the CUDA math libraries" OFF) +option(CUVS_STATIC_RAPIDS_LIBRARIES "Build and statically link RAPIDS libraries" OFF) option(CUDA_LOG_COMPILE_TIME "Write a log of compilation times to nvcc_compile_log.csv" OFF) option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" ON) option(DISABLE_DEPRECATION_WARNINGS "Disable deprecaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) +option(CUVS_COMPILE_DYNAMIC_ONLY "Only build the shared library and skip the static library." OFF) option(CUVS_NVTX "Enable nvtx markers" OFF) - -if((BUILD_TESTS OR BUILD_C_LIBRARY) AND NOT BUILD_CPU_ONLY) - -endif() +option(CUVS_RAFT_CLONE_ON_PIN "Explicitly clone RAFT branch when pinned to non-feature branch" ON) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) set(BUILD_TESTS OFF) set(BUILD_C_LIBRARY OFF) + set(BUILD_CAGRA_HNSWLIB OFF) +elseif(NOT BUILD_SHARED_LIBS) + set(BUILD_TESTS OFF) + set(BUILD_C_LIBRARY OFF) + set(BUILD_CAGRA_HNSWLIB OFF) +endif() + +if(NOT BUILD_SHARED_LIBS AND CUVS_COMPILE_DYNAMIC_ONLY) + message( + VERBOSE + "cuVS: Ignoring `BUILD_SHARED_LIBS` and forcing shared library due to `CUVS_COMPILE_DYNAMIC_ONLY`" + ) endif() -if(NOT BUILD_C_LIBRARY) - set(BUILD_C_TESTS OFF) +set(cuvs_compile_mode "both") +set(_cuvs_lib_targets cuvs cuvs_static) +if(CUVS_COMPILE_DYNAMIC_ONLY) + set(cuvs_compile_mode "shared_only") + set(_cuvs_lib_targets cuvs) +elseif(NOT BUILD_SHARED_LIBS) + set(cuvs_compile_mode "static_only") + set(_cuvs_lib_targets cuvs_static) endif() # Needed because GoogleBenchmark changes the state of FindThreads.cmake, causing subsequent runs to @@ -93,15 +111,17 @@ include(CMakeDependentOption) message(VERBOSE "cuVS: Build cuVS unit-tests: ${BUILD_TESTS}") message(VERBOSE "cuVS: Build CPU only components: ${BUILD_CPU_ONLY}") +message(VERBOSE "cuVS: Build ANN benchmarks: ${BUILD_CUVS_BENCH}") +message(VERBOSE "cuVS: Build only the shared library: ${CUVS_COMPILE_DYNAMIC_ONLY}") message(VERBOSE "cuVS: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") message(VERBOSE "cuVS: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) message(VERBOSE "cuVS: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "cuVS: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "cuVS: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "cuVS: Enable nvtx markers: ${CUVS_NVTX}") -message(VERBOSE - "cuVS: Statically link the CUDA toolkit runtime and libraries: ${CUDA_STATIC_RUNTIME}" -) +message(VERBOSE "cuVS: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") +message(VERBOSE "cuVS: Statically link the CUDA math libraries: ${CUDA_STATIC_MATH_LIBRARIES}") +message(VERBOSE "cuVS: Build and statically link RAPIDS libraries: ${CUVS_STATIC_RAPIDS_LIBRARIES}") # Set RMM logging level set(RMM_LOGGING_LEVEL @@ -130,7 +150,7 @@ endif() # * compiler options ---------------------------------------------------------- set(_ctk_static_suffix "") -if(CUDA_STATIC_RUNTIME) +if(CUDA_STATIC_MATH_LIBRARIES) set(_ctk_static_suffix "_static") endif() @@ -170,213 +190,634 @@ include(cmake/modules/ConfigureCUDA.cmake) rapids_cpm_init() if(NOT BUILD_CPU_ONLY) + # We must find CCCL ourselves before raft so that we get the right version. + include(${rapids-cmake-dir}/cpm/cccl.cmake) + rapids_cpm_cccl(BUILD_EXPORT_SET cuvs-exports INSTALL_EXPORT_SET cuvs-exports) include(cmake/thirdparty/get_raft.cmake) + include(cmake/thirdparty/get_cutlass.cmake) endif() -if(BUILD_C_LIBRARY) - include(cmake/thirdparty/get_dlpack.cmake) +if(BUILD_TESTS OR BUILD_C_TESTS) + include(${rapids-cmake-dir}/cpm/gtest.cmake) + rapids_cpm_gtest(BUILD_STATIC) endif() -if(BUILD_TESTS OR BUILD_C_TESTS) - include(cmake/thirdparty/get_gtest.cmake) +if(BUILD_CUVS_BENCH) + include(${rapids-cmake-dir}/cpm/gbench.cmake) + rapids_cpm_gbench(BUILD_STATIC) endif() -include(cmake/thirdparty/get_cutlass.cmake) +if(BUILD_CAGRA_HNSWLIB) + include(cmake/thirdparty/get_hnswlib.cmake) +endif() # ################################################################################################## # * cuvs --------------------------------------------------------------------- +if(NOT BUILD_CPU_ONLY) + add_library( + cuvs-cagra-search STATIC + src/neighbors/cagra_search_float.cu + src/neighbors/cagra_search_half.cu + src/neighbors/cagra_search_int8.cu + src/neighbors/cagra_search_uint8.cu + src/neighbors/detail/cagra/compute_distance.cu + src/neighbors/detail/cagra/compute_distance_standard_BitwiseHamming_u8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_BitwiseHamming_u8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_BitwiseHamming_u8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_float_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_float_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_float_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_float_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_float_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_float_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_half_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_half_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_half_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_int8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_int8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_int8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_uint8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_uint8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_InnerProduct_uint8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_float_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_float_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_float_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_half_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_half_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_half_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_half_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_half_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_half_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_int8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_int8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_int8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_uint8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_uint8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_CosineExpanded_uint8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_int8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_int8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_int8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_uint8_uint32_dim128_t8.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_uint8_uint32_dim256_t16.cu + src/neighbors/detail/cagra/compute_distance_standard_L2Expanded_uint8_uint32_dim512_t32.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim128_t8_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim128_t8_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim256_t16_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim256_t16_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim512_t32_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_float_uint32_dim512_t32_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim128_t8_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim128_t8_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim256_t16_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim256_t16_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim512_t32_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_half_uint32_dim512_t32_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim128_t8_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim128_t8_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim256_t16_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim256_t16_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim512_t32_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_int8_uint32_dim512_t32_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim128_t8_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim128_t8_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim256_t16_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim256_t16_8pq_4subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim512_t32_8pq_2subd_half.cu + src/neighbors/detail/cagra/compute_distance_vpq_L2Expanded_uint8_uint32_dim512_t32_8pq_4subd_half.cu + src/neighbors/detail/cagra/search_multi_cta_float_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_half_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_int8_uint32.cu + src/neighbors/detail/cagra/search_multi_cta_uint8_uint32.cu + src/neighbors/detail/cagra/search_single_cta_float_uint32.cu + src/neighbors/detail/cagra/search_single_cta_half_uint32.cu + src/neighbors/detail/cagra/search_single_cta_int8_uint32.cu + src/neighbors/detail/cagra/search_single_cta_uint8_uint32.cu + ) -add_library( - cuvs SHARED - src/neighbors/cagra_build_float.cpp - src/neighbors/cagra_build_int8.cpp - src/neighbors/cagra_build_uint8.cpp - src/neighbors/cagra_optimize.cpp - src/neighbors/cagra_search_float.cpp - src/neighbors/cagra_search_int8.cpp - src/neighbors/cagra_search_uint8.cpp - src/neighbors/cagra_serialize_float.cpp - src/neighbors/cagra_serialize_int8.cpp - src/neighbors/cagra_serialize_uint8.cpp -) + file(GLOB_RECURSE compute_distance_sources "src/neighbors/detail/cagra/compute_distance_*.cu") + set_source_files_properties(${compute_distance_sources} PROPERTIES COMPILE_FLAGS -maxrregcount=64) -target_compile_options( - cuvs INTERFACE $<$:--expt-extended-lambda - --expt-relaxed-constexpr> -) + set_target_properties( + cuvs-cagra-search + PROPERTIES BUILD_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + CUDA_SEPARABLE_COMPILATION ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + POSITION_INDEPENDENT_CODE ON + ) + target_link_libraries(cuvs-cagra-search PRIVATE raft::raft) + target_include_directories( + cuvs-cagra-search PRIVATE "$" + "$" + ) + target_compile_options( + cuvs-cagra-search PRIVATE "$<$:${CUVS_CXX_FLAGS}>" + "$<$:${CUVS_CUDA_FLAGS}>" + ) -add_library(cuvs::cuvs ALIAS cuvs) + if(BUILD_MG_ALGOS) + set(CUVS_MG_ALGOS + src/neighbors/mg/mg_flat_float_int64_t.cu + src/neighbors/mg/mg_flat_half_int64_t.cu + src/neighbors/mg/mg_flat_int8_t_int64_t.cu + src/neighbors/mg/mg_flat_uint8_t_int64_t.cu + src/neighbors/mg/mg_pq_float_int64_t.cu + src/neighbors/mg/mg_pq_half_int64_t.cu + src/neighbors/mg/mg_pq_int8_t_int64_t.cu + src/neighbors/mg/mg_pq_uint8_t_int64_t.cu + src/neighbors/mg/mg_cagra_float_uint32_t.cu + src/neighbors/mg/mg_cagra_half_uint32_t.cu + src/neighbors/mg/mg_cagra_int8_t_uint32_t.cu + src/neighbors/mg/mg_cagra_uint8_t_uint32_t.cu + ) + endif() -target_include_directories( - cuvs PUBLIC "$" - "$" -) + add_library( + cuvs_objs OBJECT + src/cluster/kmeans_balanced_fit_float.cu + src/cluster/kmeans_cluster_cost.cu + src/cluster/kmeans_fit_mg_float.cu + src/cluster/kmeans_fit_mg_double.cu + src/cluster/kmeans_fit_double.cu + src/cluster/kmeans_fit_float.cu + src/cluster/kmeans_auto_find_k_float.cu + src/cluster/kmeans_fit_predict_double.cu + src/cluster/kmeans_fit_predict_float.cu + src/cluster/kmeans_predict_double.cu + src/cluster/kmeans_predict_float.cu + src/cluster/kmeans_balanced_fit_float.cu + src/cluster/kmeans_balanced_fit_predict_float.cu + src/cluster/kmeans_balanced_predict_float.cu + src/cluster/kmeans_balanced_fit_int8.cu + src/cluster/kmeans_balanced_fit_predict_int8.cu + src/cluster/kmeans_balanced_predict_int8.cu + src/cluster/kmeans_transform_double.cu + src/cluster/kmeans_transform_float.cu + src/cluster/single_linkage_float.cu + src/core/bitset.cu + src/core/omp_wrapper.cpp + src/distance/detail/kernels/gram_matrix.cu + src/distance/detail/kernels/kernel_factory.cu + src/distance/detail/kernels/kernel_matrices.cu + src/distance/detail/pairwise_matrix/dispatch_canberra_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_canberra_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_canberra_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_correlation_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_correlation_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_correlation_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_cosine_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_cosine_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_cosine_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_hamming_unexpanded_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_hellinger_expanded_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_jensen_shannon_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_kl_divergence_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_kl_divergence_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_kl_divergence_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_l1_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l1_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l1_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_expanded_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l2_unexpanded_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_l_inf_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l_inf_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_l_inf_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_lp_unexpanded_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_russel_rao_float_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_russel_rao_half_float_float_int.cu + src/distance/detail/pairwise_matrix/dispatch_russel_rao_double_double_double_int.cu + src/distance/detail/pairwise_matrix/dispatch_rbf.cu + src/distance/detail/pairwise_matrix/dispatch_l2_expanded_double_double_double_int64_t.cu + src/distance/detail/pairwise_matrix/dispatch_l2_expanded_float_float_float_int64_t.cu + src/distance/distance.cu + src/distance/pairwise_distance.cu + src/distance/sparse_distance.cu + src/embed/spectral.cu + src/neighbors/all_neighbors/all_neighbors.cu + src/neighbors/ball_cover.cu + src/neighbors/epsilon_neighborhood.cu + src/neighbors/ball_cover/detail/ball_cover/registers_eps_pass_euclidean.cu + src/neighbors/ball_cover/detail/ball_cover/registers_pass_one.cu + src/neighbors/ball_cover/detail/ball_cover/registers_pass_two.cu + src/neighbors/brute_force.cu + src/neighbors/brute_force_serialize.cu + src/neighbors/cagra_build_float.cu + src/neighbors/cagra_build_half.cu + src/neighbors/cagra_build_int8.cu + src/neighbors/cagra_build_uint8.cu + src/neighbors/cagra_extend_float.cu + src/neighbors/cagra_extend_int8.cu + src/neighbors/cagra_extend_uint8.cu + src/neighbors/cagra_optimize.cu + src/neighbors/cagra_serialize_float.cu + src/neighbors/cagra_serialize_half.cu + src/neighbors/cagra_serialize_int8.cu + src/neighbors/cagra_serialize_uint8.cu + src/neighbors/cagra_merge_float.cu + src/neighbors/cagra_merge_half.cu + src/neighbors/cagra_merge_int8.cu + src/neighbors/cagra_merge_uint8.cu + src/neighbors/iface/iface_cagra_float_uint32_t.cu + src/neighbors/iface/iface_cagra_half_uint32_t.cu + src/neighbors/iface/iface_cagra_int8_t_uint32_t.cu + src/neighbors/iface/iface_cagra_uint8_t_uint32_t.cu + src/neighbors/iface/iface_flat_float_int64_t.cu + src/neighbors/iface/iface_flat_half_int64_t.cu + src/neighbors/iface/iface_flat_int8_t_int64_t.cu + src/neighbors/iface/iface_flat_uint8_t_int64_t.cu + src/neighbors/iface/iface_pq_float_int64_t.cu + src/neighbors/iface/iface_pq_half_int64_t.cu + src/neighbors/iface/iface_pq_int8_t_int64_t.cu + src/neighbors/iface/iface_pq_uint8_t_int64_t.cu + src/neighbors/detail/cagra/topk_for_cagra/topk.cu + src/neighbors/dynamic_batching.cu + src/neighbors/cagra_index_wrapper.cu + src/neighbors/composite/index.cu + src/neighbors/composite/merge.cpp + $<$:src/neighbors/hnsw.cpp> + src/neighbors/ivf_flat_index.cpp + src/neighbors/ivf_flat/ivf_flat_build_extend_float_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_build_extend_half_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_build_extend_int8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_build_extend_uint8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_helpers.cu + src/neighbors/ivf_flat/ivf_flat_search_float_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_search_half_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_search_int8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_search_uint8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_float_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_half_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_int8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_uint8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_float_int64_t_bitset.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_half_int64_t_bitset.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_int8_t_int64_t_bitset.cu + src/neighbors/ivf_flat/ivf_flat_interleaved_scan_uint8_t_int64_t_bitset.cu + src/neighbors/ivf_flat/ivf_flat_serialize_float_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_serialize_half_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_serialize_int8_t_int64_t.cu + src/neighbors/ivf_flat/ivf_flat_serialize_uint8_t_int64_t.cu + src/neighbors/ivf_pq_index.cu + src/neighbors/ivf_pq/ivf_pq_build_common.cu + src/neighbors/ivf_pq/ivf_pq_serialize.cu + src/neighbors/ivf_pq/ivf_pq_deserialize.cu + src/neighbors/ivf_pq/detail/ivf_pq_build_extend_float_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_build_extend_half_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_build_extend_int8_t_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_build_extend_uint8_t_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_fp8_false.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_fp8_true.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_half_half.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_half.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_float.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_false.cu + src/neighbors/ivf_pq/detail/ivf_pq_compute_similarity_float_fp8_true.cu + src/neighbors/ivf_pq/detail/ivf_pq_contiguous_list_data.cu + src/neighbors/ivf_pq/detail/ivf_pq_process_and_fill_codes.cu + src/neighbors/ivf_pq/detail/ivf_pq_search_float_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_search_half_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_search_int8_t_int64_t.cu + src/neighbors/ivf_pq/detail/ivf_pq_search_uint8_t_int64_t.cu + src/neighbors/knn_merge_parts.cu + src/neighbors/nn_descent.cu + src/neighbors/nn_descent_float.cu + src/neighbors/nn_descent_half.cu + src/neighbors/nn_descent_index.cpp + src/neighbors/nn_descent_int8.cu + src/neighbors/nn_descent_uint8.cu + src/neighbors/reachability.cu + src/neighbors/refine/detail/refine_device_float_float.cu + src/neighbors/refine/detail/refine_device_half_float.cu + src/neighbors/refine/detail/refine_device_int8_t_float.cu + src/neighbors/refine/detail/refine_device_uint8_t_float.cu + src/neighbors/refine/detail/refine_host_float_float.cpp + src/neighbors/refine/detail/refine_host_half_float.cpp + src/neighbors/refine/detail/refine_host_int8_t_float.cpp + src/neighbors/refine/detail/refine_host_uint8_t_float.cpp + src/neighbors/scann/scann_build_float.cu + src/neighbors/scann/scann_serialize_float.cu + src/neighbors/sample_filter.cu + src/neighbors/tiered_index.cu + src/neighbors/sparse_brute_force.cu + src/neighbors/vamana_build_float.cu + src/neighbors/vamana_build_uint8.cu + src/neighbors/vamana_build_int8.cu + src/neighbors/vamana_codebooks_float.cu + src/neighbors/vamana_serialize_float.cu + src/neighbors/vamana_serialize_uint8.cu + src/neighbors/vamana_serialize_int8.cu + src/preprocessing/quantize/scalar.cu + src/preprocessing/quantize/binary.cu + src/preprocessing/spectral/spectral_embedding.cu + src/selection/select_k_float_int64_t.cu + src/selection/select_k_float_int32_t.cu + src/selection/select_k_float_uint32_t.cu + src/selection/select_k_half_uint32_t.cu + src/stats/silhouette_score.cu + src/stats/trustworthiness_score.cu + ${CUVS_MG_ALGOS} + ) -if(NOT BUILD_CPU_ONLY) - # Keep cuVS as lightweight as possible. Only CUDA libs and rmm should be used in global target. - target_link_libraries(cuvs PUBLIC raft::raft raft::compiled nvidia::cutlass::cutlass) -endif() + set_target_properties( + cuvs_objs + PROPERTIES CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + ) + target_compile_options( + cuvs_objs PRIVATE "$<$:${CUVS_CXX_FLAGS}>" + "$<$:${CUVS_CUDA_FLAGS}>" + ) -# Endian detection -include(TestBigEndian) -test_big_endian(BIG_ENDIAN) -if(BIG_ENDIAN) - target_compile_definitions(cuvs PRIVATE CUVS_SYSTEM_LITTLE_ENDIAN=0) -else() - target_compile_definitions(cuvs PRIVATE CUVS_SYSTEM_LITTLE_ENDIAN=1) -endif() + target_compile_definitions( + cuvs_objs PRIVATE $<$:CUVS_BUILD_CAGRA_HNSWLIB> + $<$:NVTX_ENABLED> + ) + + target_link_libraries( + cuvs_objs + PRIVATE raft::raft rmm::rmm ${CUVS_CTK_MATH_DEPENDENCIES} + $ $ + ) + + target_include_directories( + cuvs_objs + PUBLIC "$" + "$" + INTERFACE "$" + ) + + # Endian detection + include(TestBigEndian) + test_big_endian(BIG_ENDIAN) + if(BIG_ENDIAN) + target_compile_definitions(cuvs_objs PRIVATE CUVS_SYSTEM_LITTLE_ENDIAN=0) + else() + target_compile_definitions(cuvs_objs PRIVATE CUVS_SYSTEM_LITTLE_ENDIAN=1) + endif() -file( - WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" - [=[ + if(BUILD_MG_ALGOS) + rapids_find_generate_module( + NCCL + HEADER_NAMES nccl.h + LIBRARY_NAMES nccl + ) + find_package(NCCL REQUIRED) + target_link_libraries(cuvs_objs PUBLIC $) + + target_compile_definitions(cuvs_objs PUBLIC CUVS_BUILD_MG_ALGOS) + target_compile_definitions(cuvs-cagra-search PUBLIC CUVS_BUILD_MG_ALGOS) + endif() + + set(CUVS_CUSOLVER_DEPENDENCY CUDA::cusolver${_ctk_static_suffix}) + set(CUVS_CUBLAS_DEPENDENCY CUDA::cublas${_ctk_static_suffix}) + set(CUVS_CURAND_DEPENDENCY CUDA::curand${_ctk_static_suffix}) + set(CUVS_CUSPARSE_DEPENDENCY CUDA::cusparse${_ctk_static_suffix}) + + set(CUVS_CTK_MATH_DEPENDENCIES ${CUVS_CUBLAS_DEPENDENCY} ${CUVS_CUSOLVER_DEPENDENCY} + ${CUVS_CUSPARSE_DEPENDENCY} ${CUVS_CURAND_DEPENDENCY} + ) + + if(NOT cuvs_compile_mode STREQUAL "static_only") + add_library(cuvs SHARED $) + add_library(cuvs::cuvs ALIAS cuvs) + set_target_properties( + cuvs + PROPERTIES BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + POSITION_INDEPENDENT_CODE ON + ) + target_compile_options( + cuvs + INTERFACE $<$:--expt-extended-lambda + --expt-relaxed-constexpr> + PRIVATE "$<$:${CUVS_CXX_FLAGS}>" + "$<$:${CUVS_CUDA_FLAGS}>" + ) + target_compile_definitions( + cuvs PUBLIC $<$:CUVS_BUILD_CAGRA_HNSWLIB> + $<$:NVTX_ENABLED> + ) + + target_include_directories( + cuvs + PUBLIC "$" + "$" + "$" + "$" + ) + + target_link_libraries( + cuvs + PUBLIC rmm::rmm + raft::raft + ${CUVS_CTK_MATH_DEPENDENCIES} + $> + $> + $<$:CUDA::nvtx3> + PRIVATE nvidia::cutlass::cutlass $ + cuvs-cagra-search + ) + + # ensure CUDA symbols aren't relocated to the middle of the debug build binaries + file( + WRITE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld" + [=[ SECTIONS { .nvFatBinSegment : { *(.nvFatBinSegment) } .nv_fatbin : { *(.nv_fatbin) } } ]=] -) + ) + target_link_options(cuvs PRIVATE $) -# ################################################################################################## -# * NVTX support in cuvs ----------------------------------------------------- + endif() -if(CUVS_NVTX) - # This enables NVTX within the project with no option to disable it downstream. - target_link_libraries(cuvs PUBLIC CUDA::nvtx3) - target_compile_definitions(cuvs PUBLIC NVTX_ENABLED) -else() - # Allow enable NVTX downstream if not set here. This creates a new option at build/install time, - # which is set by default to OFF, but can be enabled in the dependent project. - get_property( - nvtx_option_help_string - CACHE CUVS_NVTX - PROPERTY HELPSTRING - ) - string( - CONCAT - nvtx_export_string - "option(CUVS_NVTX \"" - ${nvtx_option_help_string} - "\" OFF)" - [=[ + if(NOT cuvs_compile_mode STREQUAL "shared_only") + add_library(cuvs_static STATIC $) + add_library(cuvs::cuvs_static ALIAS cuvs_static) + + set_target_properties( + cuvs_static + PROPERTIES BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + EXPORT_NAME cuvs_static + ) -target_link_libraries(cuvs::cuvs INTERFACE $<$:CUDA::nvtx3>) -target_compile_definitions(cuvs::cuvs INTERFACE $<$:NVTX_ENABLED>) + target_compile_options(cuvs_static PRIVATE "$<$:${CUVS_CXX_FLAGS}>") + target_compile_definitions( + cuvs_static PUBLIC $<$:CUVS_BUILD_CAGRA_HNSWLIB> + $<$:NVTX_ENABLED> + ) - ]=] - ) -endif() + target_include_directories(cuvs_static INTERFACE "$") -set_target_properties( - cuvs - PROPERTIES CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON -) + # ensure CUDA symbols aren't relocated to the middle of the debug build binaries + target_link_options(cuvs_static PRIVATE $) -target_compile_options( - cuvs PRIVATE "$<$:${CUVS_CXX_FLAGS}>" - "$<$:${CUVS_CUDA_FLAGS}>" -) -# ensure CUDA symbols aren't relocated to the middle of the debug build binaries -target_link_options(cuvs PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") + target_include_directories( + cuvs_static + PUBLIC "$" + "$" + "$" + "$" + ) + target_link_libraries( + cuvs_static + PUBLIC rmm::rmm + raft::raft + ${CUVS_CTK_MATH_DEPENDENCIES} + $> + $> + $<$:CUDA::nvtx3> + PRIVATE nvidia::cutlass::cutlass $ + ) + endif() -# ################################################################################################## -# * cuvs_c ------------------------------------------------------------------------------- -if(BUILD_C_LIBRARY) - add_library(cuvs_c SHARED src/core/c_api.cpp src/neighbors/cagra_c.cpp) + # ################################################################################################ + # * NVTX support in cuvs ----------------------------------------------------- + if(NOT CUVS_NVTX) + # Allow enable NVTX downstream if not set here. This creates a new option at build/install time, + # which is set by default to OFF, but can be enabled in the dependent project. + get_property( + nvtx_option_help_string + CACHE CUVS_NVTX + PROPERTY HELPSTRING + ) + string( + CONCAT + nvtx_export_string + "option(CUVS_NVTX \"" + ${nvtx_option_help_string} + "\" OFF)" + [=[ - add_library(cuvs::c_api ALIAS cuvs_c) +target_link_libraries(cuvs::cuvs INTERFACE $<$:CUDA::nvtx3>) +target_compile_definitions(cuvs::cuvs INTERFACE $<$:NVTX_ENABLED>) - set_target_properties( - cuvs_c - PROPERTIES CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - EXPORT_NAME c_api - ) + ]=] + ) + endif() - target_compile_options(cuvs_c PRIVATE "$<$:${CUVS_CXX_FLAGS}>") + # ################################################################################################ + # * cuvs_c ------------------------------------------------------------------------------- + if(BUILD_C_LIBRARY) + add_subdirectory(../c c) + endif() - target_include_directories( - cuvs_c - PUBLIC "$" - INTERFACE "$" + # ################################################################################################ + # * install targets----------------------------------------------------------- + rapids_cmake_install_lib_dir(lib_dir) + include(GNUInstallDirs) + include(CPack) + + install( + TARGETS ${_cuvs_lib_targets} + DESTINATION ${lib_dir} + COMPONENT cuvs + EXPORT cuvs-exports ) - target_link_libraries(cuvs_c PUBLIC cuvs::cuvs) + install( + DIRECTORY include/cuvs + COMPONENT cuvs + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) - # ensure CUDA symbols aren't relocated to the middle of the debug build binaries - target_link_options(cuvs_c PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/fatbin.ld") -endif() + if(BUILD_C_LIBRARY) + install( + DIRECTORY ../c/include/cuvs + COMPONENT cuvs + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + ) -# ################################################################################################## -# * install targets----------------------------------------------------------- -rapids_cmake_install_lib_dir(lib_dir) -include(GNUInstallDirs) -include(CPack) - -install( - TARGETS cuvs - DESTINATION ${lib_dir} - COMPONENT cuvs - EXPORT cuvs-exports -) + install( + TARGETS cuvs_c + DESTINATION ${lib_dir} + COMPONENT c_api + EXPORT cuvs-c-exports + ) -install( - DIRECTORY include/cuvs - COMPONENT cuvs - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} -) + install( + FILES ${CUVS_C_BINARY_DIR}/include/cuvs/core/c_config.h + COMPONENT cuvs + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/cuvs/core/ + ) + endif() -if(BUILD_C_LIBRARY) install( - TARGETS cuvs_c - DESTINATION ${lib_dir} - COMPONENT c_api - EXPORT cuvs-c-exports + FILES ${CMAKE_CURRENT_BINARY_DIR}/include/cuvs/version_config.h + COMPONENT cuvs + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/cuvs ) -endif() -install( - FILES ${CMAKE_CURRENT_BINARY_DIR}/include/cuvs/version_config.hpp - COMPONENT cuvs - DESTINATION include/cuvs -) + if(TARGET cuvs_c) + list(APPEND cuvs_components c_api) + list(APPEND cuvs_export_sets cuvs-c-exports) + set(CUVS_C_TARGET cuvs_c) + endif() -if(TARGET cuvs_c) - list(APPEND cuvs_components c_api) - list(APPEND cuvs_export_sets cuvs-c-exports) - set(CUVS_C_TARGET cuvs_c) -endif() + # Use `rapids_export` for 22.04 as it will have COMPONENT support + rapids_export( + INSTALL cuvs + EXPORT_SET cuvs-exports + COMPONENTS ${cuvs_components} + COMPONENTS_EXPORT_SET ${cuvs_export_sets} + GLOBAL_TARGETS cuvs ${CUVS_C_TARGET} + NAMESPACE cuvs:: + ) -# Use `rapids_export` for 22.04 as it will have COMPONENT support -rapids_export( - INSTALL cuvs - EXPORT_SET cuvs-exports - COMPONENTS ${cuvs_components} - COMPONENTS_EXPORT_SET ${cuvs_export_sets} - GLOBAL_TARGETS cuvs ${CUVS_C_TARGET} - NAMESPACE cuvs:: -) + # ################################################################################################ + # * build export ------------------------------------------------------------- + rapids_export( + BUILD cuvs + EXPORT_SET cuvs-exports + COMPONENTS ${cuvs_components} + COMPONENTS_EXPORT_SET ${cuvs_export_sets} + GLOBAL_TARGETS cuvs ${CUVS_C_TARGET} + NAMESPACE cuvs:: + ) +endif() # ################################################################################################## -# * build export ------------------------------------------------------------- -rapids_export( - BUILD cuvs - EXPORT_SET cuvs-exports - COMPONENTS ${cuvs_components} - COMPONENTS_EXPORT_SET ${cuvs_export_sets} - GLOBAL_TARGETS cuvs ${CUVS_C_TARGET} - NAMESPACE cuvs:: -) +# * build test executable ---------------------------------------------------- + +if(BUILD_TESTS) + enable_testing() + add_subdirectory(internal) + add_subdirectory(tests) +endif() # ################################################################################################## -# * build test executable ---------------------------------------------------- +# * build ann benchmark executable ----------------------------------------------- -if(BUILD_TESTS OR BUILD_C_TESTS) - include(internal/CMakeLists.txt) - include(test/CMakeLists.txt) +if(BUILD_CUVS_BENCH) + add_subdirectory(bench/ann/) endif() diff --git a/cpp/bench/ann/CMakeLists.txt b/cpp/bench/ann/CMakeLists.txt new file mode 100644 index 0000000000..79d72ea71b --- /dev/null +++ b/cpp/bench/ann/CMakeLists.txt @@ -0,0 +1,397 @@ +# ============================================================================= +# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +list(APPEND CMAKE_MODULE_PATH "${CUVS_SOURCE_DIR}") + +# ################################################################################################## +# * benchmark options ------------------------------------------------------------------------------ + +option(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT "Include faiss' brute-force knn algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT "Include faiss' ivf flat algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ "Include faiss' ivf pq algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_GPU_CAGRA "Include faiss' cagra algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_GPU_CAGRA_HNSW + "Include faiss' cagra algorithm for build and hnsw for search in benchmark" ON +) +option(CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT "Include faiss' cpu brute-force algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT "Include faiss' cpu ivf flat algorithm in benchmark" + ON +) +option(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ "Include faiss' cpu ivf pq algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_FLAT "Include faiss' hnsw algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT "Include cuVS ivf flat algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ "Include cuVS ivf pq algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_CAGRA "Include cuVS CAGRA in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE "Include cuVS brute force knn in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB "Include cuVS CAGRA with HNSW search in benchmark" ON) +option(CUVS_ANN_BENCH_USE_HNSWLIB "Include hnsw algorithm in benchmark" ON) +option(CUVS_ANN_BENCH_USE_GGNN "Include ggnn algorithm in benchmark" OFF) +option(CUVS_ANN_BENCH_USE_DISKANN "Include DISKANN search in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_VAMANA "Include cuVS Vamana with DiskANN search in benchmark" ON) +option(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN "Include cuVS CAGRA with DISKANN search in benchmark" + ON +) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "(ARM|arm|aarch64)") + set(CUVS_ANN_BENCH_USE_DISKANN OFF) + set(CUVS_ANN_BENCH_USE_CUVS_VAMANA OFF) + set(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN OFF) +endif() +option(CUVS_ANN_BENCH_USE_CUVS_MG "Include cuVS ann mg algorithm in benchmark" ${BUILD_MG_ALGOS}) +option(CUVS_ANN_BENCH_SINGLE_EXE + "Make a single executable with benchmark as shared library modules" OFF +) +option(CUVS_KNN_BENCH_USE_CUVS_BRUTE_FORCE "Include cuVS brute force knn in benchmark" ON) + +# ################################################################################################## +# * Process options ---------------------------------------------------------- + +find_package(Threads REQUIRED) + +set(CUVS_ANN_BENCH_USE_FAISS ON) +set(CUVS_FAISS_ENABLE_GPU ON) +set(CUVS_USE_FAISS_STATIC ON) + +if(BUILD_CPU_ONLY) + set(CUVS_FAISS_ENABLE_GPU OFF) + set(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT OFF) + set(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ OFF) + set(CUVS_ANN_BENCH_USE_CUVS_CAGRA OFF) + set(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE OFF) + set(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB OFF) + set(CUVS_ANN_BENCH_USE_GGNN OFF) + set(CUVS_KNN_BENCH_USE_CUVS_BRUTE_FORCE OFF) + set(CUVS_ANN_BENCH_USE_CUVS_MG OFF) + set(CUVS_ANN_BENCH_USE_CUVS_VAMANA OFF) + set(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN OFF) +else() + set(CUVS_FAISS_ENABLE_GPU ON) +endif() + +set(CUVS_ANN_BENCH_USE_CUVS OFF) +if(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ + OR CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE + OR CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT + OR CUVS_ANN_BENCH_USE_CUVS_CAGRA + OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB + OR CUVS_KNN_BENCH_USE_CUVS_BRUTE_FORCE + OR CUVS_ANN_BENCH_USE_CUVS_MG + OR CUVS_ANN_BENCH_USE_CUVS_VAMANA + OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN +) + set(CUVS_ANN_BENCH_USE_CUVS ON) +endif() + +# ################################################################################################## +# * Fetch requirements ------------------------------------------------------------- + +if(CUVS_ANN_BENCH_USE_HNSWLIB OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) + include(cmake/thirdparty/get_hnswlib) +endif() + +include(cmake/thirdparty/get_nlohmann_json) + +if(CUVS_ANN_BENCH_USE_GGNN) + include(cmake/thirdparty/get_ggnn) +endif() + +if(CUVS_ANN_BENCH_USE_DISKANN + OR CUVS_ANN_BENCH_USE_CUVS_VAMANA + OR CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN +) + include(cmake/thirdparty/get_diskann) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS) + include(cmake/thirdparty/get_faiss) +endif() + +# ################################################################################################## +# * Target function ------------------------------------------------------------- + +function(ConfigureAnnBench) + + set(oneValueArgs NAME) + set(multiValueArgs PATH LINKS CXXFLAGS) + + if(NOT BUILD_CPU_ONLY) + set(GPU_BUILD ON) + endif() + + cmake_parse_arguments( + ConfigureAnnBench "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} + ) + + set(BENCH_NAME ${ConfigureAnnBench_NAME}_ANN_BENCH) + + if(CUVS_ANN_BENCH_SINGLE_EXE) + add_library(${BENCH_NAME} SHARED ${ConfigureAnnBench_PATH}) + string(TOLOWER ${BENCH_NAME} BENCH_LIB_NAME) + set_target_properties(${BENCH_NAME} PROPERTIES OUTPUT_NAME ${BENCH_LIB_NAME}) + add_dependencies(${BENCH_NAME} ANN_BENCH) + else() + add_executable(${BENCH_NAME} ${ConfigureAnnBench_PATH}) + target_compile_definitions(${BENCH_NAME} PRIVATE ANN_BENCH_BUILD_MAIN>) + target_link_libraries( + ${BENCH_NAME} PRIVATE benchmark::benchmark $<$:CUDA::nvtx3> + ) + endif() + + target_link_libraries( + ${BENCH_NAME} + PRIVATE ${ConfigureAnnBench_LINKS} + nlohmann_json::nlohmann_json + Threads::Threads + $<$:CUDA::cudart_static> + $ + $ + ) + + set_target_properties( + ${BENCH_NAME} + PROPERTIES # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + ) + + set(${ConfigureAnnBench_CXXFLAGS} ${CUVS_CXX_FLAGS} ${ConfigureAnnBench_CXXFLAGS}) + + target_compile_options( + ${BENCH_NAME} PRIVATE "$<$:${ConfigureAnnBench_CXXFLAGS}>" + "$<$:${CUVS_CUDA_FLAGS}>" + ) + + if(CUVS_ANN_BENCH_USE_${ConfigureAnnBench_NAME}) + target_compile_definitions( + ${BENCH_NAME} + PUBLIC + CUVS_ANN_BENCH_USE_${ConfigureAnnBench_NAME}=CUVS_ANN_BENCH_USE_${ConfigureAnnBench_NAME} + ) + endif() + + target_include_directories( + ${BENCH_NAME} + PUBLIC "$" + "$" + PRIVATE ${ConfigureAnnBench_INCLUDES} + ) + + install( + TARGETS ${BENCH_NAME} + COMPONENT ann_bench + DESTINATION bin/ann + ) + + add_dependencies(CUVS_ANN_BENCH_ALL ${BENCH_NAME}) +endfunction() + +# ################################################################################################## +# * Configure benchmark targets ------------------------------------------------------------- + +if(NOT TARGET CUVS_ANN_BENCH_ALL) + add_custom_target(CUVS_ANN_BENCH_ALL) +endif() + +if(CUVS_ANN_BENCH_USE_HNSWLIB) + ConfigureAnnBench(NAME HNSWLIB PATH src/hnswlib/hnswlib_benchmark.cpp LINKS hnswlib::hnswlib) + +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ) + ConfigureAnnBench( + NAME CUVS_IVF_PQ PATH src/cuvs/cuvs_benchmark.cu src/cuvs/cuvs_ivf_pq.cu LINKS cuvs + ) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT) + ConfigureAnnBench( + NAME CUVS_IVF_FLAT PATH src/cuvs/cuvs_benchmark.cu src/cuvs/cuvs_ivf_flat.cu LINKS cuvs + ) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE) + ConfigureAnnBench(NAME CUVS_BRUTE_FORCE PATH src/cuvs/cuvs_benchmark.cu LINKS cuvs) +endif() + +if(CUVS_KNN_BENCH_USE_CUVS_BRUTE_FORCE) + ConfigureAnnBench( + NAME CUVS_KNN_BRUTE_FORCE PATH + $<$:src/cuvs/cuvs_brute_force_knn.cu> LINKS cuvs + ) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_CAGRA) + ConfigureAnnBench( + NAME + CUVS_CAGRA + PATH + src/cuvs/cuvs_benchmark.cu + src/cuvs/cuvs_cagra_float.cu + src/cuvs/cuvs_cagra_half.cu + src/cuvs/cuvs_cagra_int8_t.cu + src/cuvs/cuvs_cagra_uint8_t.cu + LINKS + cuvs + ) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) + ConfigureAnnBench(NAME CUVS_CAGRA_HNSWLIB PATH src/cuvs/cuvs_cagra_hnswlib.cu LINKS cuvs) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_MG) + ConfigureAnnBench( + NAME + CUVS_MG + PATH + src/cuvs/cuvs_benchmark.cu + $<$:src/cuvs/cuvs_mg_ivf_flat.cu> + $<$:src/cuvs/cuvs_mg_ivf_pq.cu> + $<$:src/cuvs/cuvs_mg_cagra.cu> + LINKS + cuvs + nccl + ) +endif() + +message("CUVS_FAISS_TARGETS: ${CUVS_FAISS_TARGETS}") +message("CUDAToolkit_LIBRARY_DIR: ${CUDAToolkit_LIBRARY_DIR}") +if(CUVS_ANN_BENCH_USE_FAISS_CPU_FLAT) + ConfigureAnnBench( + NAME FAISS_CPU_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS ${CUVS_FAISS_TARGETS} + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_FLAT) + ConfigureAnnBench( + NAME FAISS_CPU_IVF_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS ${CUVS_FAISS_TARGETS} + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_CPU_IVF_PQ) + ConfigureAnnBench( + NAME FAISS_CPU_IVF_PQ PATH src/faiss/faiss_cpu_benchmark.cpp LINKS ${CUVS_FAISS_TARGETS} + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_CPU_HNSW_FLAT) + ConfigureAnnBench( + NAME FAISS_CPU_HNSW_FLAT PATH src/faiss/faiss_cpu_benchmark.cpp LINKS ${CUVS_FAISS_TARGETS} + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_FLAT AND CUVS_FAISS_ENABLE_GPU) + ConfigureAnnBench( + NAME FAISS_GPU_IVF_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} + raft::raft + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_GPU_IVF_PQ AND CUVS_FAISS_ENABLE_GPU) + ConfigureAnnBench( + NAME FAISS_GPU_IVF_PQ PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} + raft::raft + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_GPU_FLAT AND CUVS_FAISS_ENABLE_GPU) + ConfigureAnnBench( + NAME FAISS_GPU_FLAT PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} + raft::raft + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_GPU_CAGRA AND CUVS_FAISS_ENABLE_GPU) + ConfigureAnnBench( + NAME FAISS_GPU_CAGRA PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} + raft::raft + ) +endif() + +if(CUVS_ANN_BENCH_USE_FAISS_GPU_CAGRA_HNSW AND CUVS_FAISS_ENABLE_GPU) + ConfigureAnnBench( + NAME FAISS_GPU_CAGRA_HNSW PATH src/faiss/faiss_gpu_benchmark.cu LINKS ${CUVS_FAISS_TARGETS} + raft::raft + ) +endif() + +if(CUVS_ANN_BENCH_USE_GGNN) + include(cmake/thirdparty/get_glog) + ConfigureAnnBench( + NAME GGNN PATH src/ggnn/ggnn_benchmark.cu LINKS glog::glog ggnn::ggnn CUDA::curand + ) +endif() + +if(CUVS_ANN_BENCH_USE_DISKANN) + ConfigureAnnBench( + NAME DISKANN_MEMORY PATH src/diskann/diskann_benchmark.cpp LINKS diskann::diskann aio + ) + ConfigureAnnBench( + NAME DISKANN_SSD PATH src/diskann/diskann_benchmark.cpp LINKS diskann::diskann aio + ) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_VAMANA) + ConfigureAnnBench(NAME CUVS_VAMANA PATH src/cuvs/cuvs_vamana.cu LINKS cuvs diskann::diskann aio) +endif() + +if(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN) + ConfigureAnnBench( + NAME CUVS_CAGRA_DISKANN PATH src/cuvs/cuvs_cagra_diskann.cu LINKS cuvs diskann::diskann aio + ) +endif() + +# ################################################################################################## +# * Dynamically-loading ANN_BENCH executable ------------------------------------------------------- +if(CUVS_ANN_BENCH_SINGLE_EXE) + add_executable(ANN_BENCH src/common/benchmark.cpp) + + target_include_directories(ANN_BENCH PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) + + target_link_libraries( + ANN_BENCH PRIVATE raft::raft nlohmann_json::nlohmann_json benchmark::benchmark dl + $<$:CUDA::nvtx3> + ) + set_target_properties( + ANN_BENCH + PROPERTIES # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + ) + target_compile_definitions( + ANN_BENCH + PRIVATE + $<$:ANN_BENCH_LINK_CUDART="libcudart.so.${CUDAToolkit_VERSION_MAJOR}.${CUDAToolkit_VERSION_MINOR}.${CUDAToolkit_VERSION_PATCH}"> + ) + + target_link_options(ANN_BENCH PRIVATE -export-dynamic) + + install( + TARGETS ANN_BENCH + COMPONENT ann_bench + DESTINATION bin/ann + EXCLUDE_FROM_ALL + ) +endif() diff --git a/cpp/bench/ann/src/common/ann_types.hpp b/cpp/bench/ann/src/common/ann_types.hpp new file mode 100644 index 0000000000..464932f5a1 --- /dev/null +++ b/cpp/bench/ann/src/common/ann_types.hpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cuda_stub.hpp" // cudaStream_t + +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +/** Benchmark mode: measuring latency vs throughput. */ +enum class Mode { + kThroughput, // See how many vectors we can push through + kLatency // See how fast we can push a vector through +}; + +enum class MemoryType { + kHost, + kHostMmap, + kHostPinned, + kDevice, + kManaged, +}; + +/** Request 2MB huge pages support for an allocation */ +enum class HugePages { + /** Don't use huge pages if possible. */ + kDisable = 0, + /** Enable huge pages if possible, ignore otherwise. */ + kAsk = 1, + /** Enable huge pages if possible, warn the user otherwise. */ + kRequire = 2, + /** Force enable huge pages, throw an exception if not possible. */ + kDemand = 3 +}; + +enum class Metric { + kInnerProduct, + kEuclidean, +}; + +inline auto parse_metric(const std::string& metric_str) -> Metric +{ + if (metric_str == "inner_product") { + return cuvs::bench::Metric::kInnerProduct; + } else if (metric_str == "euclidean") { + return cuvs::bench::Metric::kEuclidean; + } else { + throw std::runtime_error("invalid metric: '" + metric_str + "'"); + } +} + +inline auto parse_memory_type(const std::string& memory_type) -> MemoryType +{ + if (memory_type == "host") { + return MemoryType::kHost; + } else if (memory_type == "mmap") { + return MemoryType::kHostMmap; + } else if (memory_type == "pinned") { + return MemoryType::kHostPinned; + } else if (memory_type == "device") { + return MemoryType::kDevice; + } else if (memory_type == "managed") { + return MemoryType::kManaged; + } else { + throw std::runtime_error("invalid memory type: '" + memory_type + "'"); + } +} + +struct algo_property { + MemoryType dataset_memory_type; + // neighbors/distances should have same memory type as queries + MemoryType query_memory_type; +}; + +class algo_base { + public: + using index_type = int64_t; + + inline algo_base(Metric metric, int dim) : metric_(metric), dim_(dim) {} + virtual ~algo_base() noexcept = default; + + protected: + Metric metric_; + int dim_; +}; + +/** + * The GPU-based algorithms, which do not perform CPU synchronization at the end of their build or + * search methods, must implement this interface. + * + * The `cuda_timer` / `cuda_lap` from `util.hpp` uses this stream to record GPU times with events + * and, if necessary, also synchronize (via events) between iterations. + * + * If the algo does not implement this interface, GPU timings are disabled. + */ +class algo_gpu { + public: + /** + * Return the main cuda stream for this algorithm. + * If any work is done in multiple streams, they should synchornize with the main stream at the + * end. + */ + [[nodiscard]] virtual auto get_sync_stream() const noexcept -> cudaStream_t = 0; + /** + * By default a GPU algorithm uses a fixed stream to order GPU operations. + * However, an algorithm may need to synchronize with the host at the end of its execution. + * In that case, also synchronizing with a benchmark event would put it at disadvantage. + * + * We can disable event sync by passing `false` here + * - ONLY IF THE ALGORITHM HAS PRODUCED ITS OUTPUT BY THE TIME IT SYNCHRONIZES WITH CPU. + */ + [[nodiscard]] virtual auto uses_stream() const noexcept -> bool { return true; } + virtual ~algo_gpu() noexcept = default; +}; + +template +class algo : public algo_base { + public: + struct search_param { + virtual ~search_param() = default; + [[nodiscard]] virtual auto needs_dataset() const -> bool { return false; }; + }; + + inline algo(Metric metric, int dim) : algo_base(metric, dim) {} + ~algo() noexcept override = default; + + virtual void build(const T* dataset, size_t nrow) = 0; + + virtual void set_search_param(const search_param& param, const void* filter_bitset) = 0; + // TODO(snanditale): this assumes that an algorithm can always return k results. + // This is not always possible. + virtual void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const = 0; + + virtual void save(const std::string& file) const = 0; + virtual void load(const std::string& file) = 0; + + [[nodiscard]] virtual auto get_preference() const -> algo_property = 0; + + // Some algorithms don't save the building dataset in their indices. + // So they should be given the access to that dataset during searching. + // The advantage of this way is that index has smaller size + // and many indices can share one dataset. + // + // search_param::needs_dataset() of such algorithm should be true, + // and set_search_dataset() should save the passed-in pointer somewhere. + // The client code should call set_search_dataset() before searching, + // and should not release dataset before searching is finished. + virtual void set_search_dataset(const T* /*dataset*/, size_t /*nrow*/) {}; + + /** + * Make a shallow copy of the algo wrapper that shares the resources and ensures thread-safe + * access to them. */ + virtual auto copy() -> std::unique_ptr> = 0; +}; + +} // namespace cuvs::bench + +#define REGISTER_ALGO_INSTANCE(DataT) \ + template auto cuvs::bench::create_algo( \ + const std::string&, const std::string&, int, const nlohmann::json&) \ + -> std::unique_ptr>; \ + template auto cuvs::bench::create_search_param(const std::string&, const nlohmann::json&) \ + -> std::unique_ptr::search_param>; diff --git a/cpp/bench/ann/src/common/benchmark.cpp b/cpp/bench/ann/src/common/benchmark.cpp new file mode 100644 index 0000000000..6d0c16a451 --- /dev/null +++ b/cpp/bench/ann/src/common/benchmark.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// clang-format off +#include "cuda_stub.hpp" // must go first +// clang-format on + +#include "ann_types.hpp" + +#include + +#include +#include +#include + +namespace cuvs::bench { + +struct lib_handle { + void* handle{nullptr}; + explicit lib_handle(const std::string& name) + { + handle = dlopen(name.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (handle == nullptr) { + auto error_msg = "Failed to load " + name; + auto err = dlerror(); + if (err != nullptr && err[0] != '\0') { error_msg += ": " + std::string(err); } + throw std::runtime_error(error_msg); + } + } + ~lib_handle() noexcept + { + if (handle != nullptr) { dlclose(handle); } + } +}; + +auto load_lib(const std::string& algo) -> void* +{ + static std::unordered_map libs{}; + auto lib_short_name = algo; + // cuvs_mg library is responsible for all mg algorithms + if (algo.rfind("cuvs_mg_", 0) == 0) { lib_short_name = "cuvs_mg"; } + + auto found = libs.find(lib_short_name); + if (found != libs.end()) { return found->second.handle; } + + auto lib_name = "lib" + lib_short_name + "_ann_bench.so"; + return libs.emplace(lib_short_name, lib_name).first->second.handle; +} + +/* + TODO(achirkin): remove this compatibility layer. + When reading old raft algo configs, we may encounter raft_xxx algorithms; + they all are renamed to cuvs_xxx algorithm. + This compatibility layer helps using old configs with the new benchmark executable. + */ +auto load_lib_raft_compat(const std::string& algo) -> void* +{ + try { + return load_lib(algo); + } catch (std::runtime_error& e) { + if (algo.rfind("raft", 0) == 0) { return load_lib("cuvs" + algo.substr(4)); } + throw e; + } +} + +auto get_fun_name(void* addr) -> std::string +{ + Dl_info dl_info; + if (dladdr(addr, &dl_info) != 0) { + if (dl_info.dli_sname != nullptr && dl_info.dli_sname[0] != '\0') { + return std::string{dl_info.dli_sname}; + } + } + throw std::logic_error("Failed to find out name of the looked up function"); +} + +template +auto create_algo(const std::string& algo, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + static auto fname = get_fun_name(reinterpret_cast(&create_algo)); + auto handle = load_lib_raft_compat(algo); + auto fun_addr = dlsym(handle, fname.c_str()); + if (fun_addr == nullptr) { + throw std::runtime_error("Couldn't load the create_algo function (" + algo + ")"); + } + auto fun = reinterpret_cast)>(fun_addr); + return fun(algo, distance, dim, conf); +} + +template +std::unique_ptr::search_param> create_search_param( + const std::string& algo, const nlohmann::json& conf) +{ + static auto fname = get_fun_name(reinterpret_cast(&create_search_param)); + auto handle = load_lib_raft_compat(algo); + auto fun_addr = dlsym(handle, fname.c_str()); + if (fun_addr == nullptr) { + throw std::runtime_error("Couldn't load the create_search_param function (" + algo + ")"); + } + auto fun = reinterpret_cast)>(fun_addr); + return fun(algo, conf); +} + +}; // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#include "benchmark.hpp" + +auto main(int argc, char** argv) -> int { return cuvs::bench::run_main(argc, argv); } diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp new file mode 100644 index 0000000000..92f2205a39 --- /dev/null +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -0,0 +1,784 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "ann_types.hpp" +#include "conf.hpp" +#include "dataset.hpp" +#include "util.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +static inline std::unique_ptr current_algo{nullptr}; +static inline std::unique_ptr current_algo_props{nullptr}; + +using kv_series = std::vector>>; + +inline auto apply_overrides(const std::vector& configs, + const kv_series& overrides, + std::size_t override_idx = 0) -> std::vector +{ + std::vector results{}; + if (override_idx >= overrides.size()) { + auto n = configs.size(); + for (size_t i = 0; i < n; i++) { + auto c = configs[i]; + c["override_suffix"] = n > 1 ? "/" + std::to_string(i) : ""; + results.push_back(c); + } + return results; + } + auto rec_configs = apply_overrides(configs, overrides, override_idx + 1); + auto [key, vals] = overrides[override_idx]; + auto n = vals.size(); + for (size_t i = 0; i < n; i++) { + const auto& val = vals[i]; + for (auto rc : rec_configs) { + if (n > 1) { + rc["override_suffix"] = + static_cast(rc["override_suffix"]) + "/" + std::to_string(i); + } + rc[key] = val; + results.push_back(rc); + } + } + return results; +} + +inline auto apply_overrides(const nlohmann::json& config, + const kv_series& overrides, + std::size_t override_idx = 0) +{ + return apply_overrides(std::vector{config}, overrides, 0); +} + +inline void dump_parameters(::benchmark::State& state, nlohmann::json params) +{ + std::string label = ""; + bool label_empty = true; + for (auto& [key, val] : params.items()) { + if (val.is_number()) { + state.counters.insert({{key, val}}); + } else if (val.is_boolean()) { + state.counters.insert({{key, val ? 1.0 : 0.0}}); + } else { + auto kv = key + "=" + val.dump(); + if (label_empty) { + label = kv; + } else { + label += "#" + kv; + } + label_empty = false; + } + } + if (!label_empty) { state.SetLabel(label); } +} + +inline auto parse_algo_property(algo_property prop, const nlohmann::json& conf) -> algo_property +{ + if (conf.contains("dataset_memory_type")) { + prop.dataset_memory_type = parse_memory_type(conf.at("dataset_memory_type")); + } + if (conf.contains("query_memory_type")) { + prop.query_memory_type = parse_memory_type(conf.at("query_memory_type")); + } + return prop; +}; + +template +void bench_build(::benchmark::State& state, + std::shared_ptr> dataset, + const configuration::index& index, + bool force_overwrite, + bool no_lap_sync) +{ + // NB: these two thread-local vars can be used within algo wrappers + cuvs::bench::benchmark_thread_id = state.thread_index(); + cuvs::bench::benchmark_n_threads = state.threads(); + dump_parameters(state, index.build_param); + if (file_exists(index.file)) { + if (force_overwrite) { + log_info("Overwriting file: %s", index.file.c_str()); + } else { + return state.SkipWithMessage( + "Index file already exists (use --force to overwrite the index)."); + } + } + + std::unique_ptr> algo; + try { + algo = create_algo(index.algo, dataset->distance(), dataset->dim(), index.build_param); + } catch (const std::exception& e) { + return state.SkipWithError("Failed to create an algo: " + std::string(e.what())); + } + + const auto algo_property = parse_algo_property(algo->get_preference(), index.build_param); + + const T* base_set = dataset->base_set(algo_property.dataset_memory_type); + std::size_t index_size = dataset->base_set_size(); + + cuda_timer gpu_timer{algo}; + { + nvtx_case nvtx{state.name()}; + /* Note: GPU timing + + The GPU time is measured between construction and destruction of `cuda_lap` objects (`gpu_all` + and `gpu_lap` variables) and added to the `gpu_timer` object. + + We sync with the GPU (cudaEventSynchronize) either each iteration (lifetime of the `gpu_lap` + variable) or once per benchmark loop (lifetime of the `gpu_all` variable). The decision is + + controlled by the `no_lap_sync` argument. In either case, we need at least one sync throughout + the benchmark loop to make sure the GPU has finished its work before we measure the total run + time. + */ + [[maybe_unused]] auto gpu_all = gpu_timer.lap(no_lap_sync); + for (auto _ : state) { + [[maybe_unused]] auto ntx_lap = nvtx.lap(); + [[maybe_unused]] auto gpu_lap = gpu_timer.lap(!no_lap_sync); + try { + algo->build(base_set, index_size); + } catch (const std::exception& e) { + state.SkipWithError(std::string(e.what())); + } + } + } + if (gpu_timer.active()) { + state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); + } + state.counters.insert({{"index_size", index_size}}); + + if (state.skipped()) { return; } + make_sure_parent_dir_exists(index.file); + algo->save(index.file); +} + +template +void bench_search(::benchmark::State& state, + const configuration::index& index, + std::size_t search_param_ix, + std::shared_ptr> dataset, + bool no_lap_sync) +{ + // NB: these two thread-local vars can be used within algo wrappers + cuvs::bench::benchmark_thread_id = state.thread_index(); + cuvs::bench::benchmark_n_threads = state.threads(); + std::size_t queries_processed = 0; + + const auto& sp_json = index.search_params[search_param_ix]; + + if (state.thread_index() == 0) { dump_parameters(state, sp_json); } + + // NB: `k` and `n_queries` are guaranteed to be populated in conf.cpp + const std::uint32_t k = sp_json["k"]; + // Amount of data processes in one go + const std::size_t n_queries = sp_json["n_queries"]; + // Round down the query data to a multiple of the batch size to loop over full batches of data + const std::size_t query_set_size = (dataset->query_set_size() / n_queries) * n_queries; + + if (dataset->query_set_size() < n_queries) { + std::stringstream msg; + msg << "Not enough queries in benchmark set. Expected " << n_queries << ", actual " + << dataset->query_set_size(); + state.SkipWithError(msg.str()); + return; + } + + // Each thread start from a different offset, so that the queries that they process do not + // overlap. + std::ptrdiff_t batch_offset = (state.thread_index() * n_queries) % query_set_size; + std::ptrdiff_t queries_stride = state.threads() * n_queries; + // Output is saved into a contiguous buffer (separate buffers for each thread). + std::ptrdiff_t out_offset = 0; + + const T* query_set = nullptr; + + if (!file_exists(index.file)) { + state.SkipWithError("Index file is missing. Run the benchmark in the build mode first."); + return; + } + + /** + * Make sure the first thread loads the algo and dataset + */ + progress_barrier load_barrier{}; + if (load_barrier.arrive(1) == 0) { + // algo is static to cache it between close search runs to save time on index loading + static std::string index_file = ""; + if (index.file != index_file) { + current_algo.reset(); + index_file = index.file; + } + + std::unique_ptr::search_param> search_param; + algo* a; + try { + if (!current_algo || (a = dynamic_cast*>(current_algo.get())) == nullptr) { + auto ualgo = + create_algo(index.algo, dataset->distance(), dataset->dim(), index.build_param); + a = ualgo.get(); + a->load(index_file); + current_algo = std::move(ualgo); + } + search_param = create_search_param(index.algo, sp_json); + } catch (const std::exception& e) { + state.SkipWithError("Failed to create an algo: " + std::string(e.what())); + return; + } + + current_algo_props = + std::make_unique(std::move(parse_algo_property(a->get_preference(), sp_json))); + + if (search_param->needs_dataset()) { + try { + a->set_search_dataset(dataset->base_set(current_algo_props->dataset_memory_type), + dataset->base_set_size()); + } catch (const std::exception& ex) { + state.SkipWithError("The algorithm '" + index.name + + "' requires the base set, but it's not available. " + + "Exception: " + std::string(ex.what())); + return; + } + } + try { + a->set_search_param(*search_param, + dataset->filter_bitset(current_algo_props->dataset_memory_type)); + } catch (const std::exception& ex) { + state.SkipWithError("An error occurred setting search parameters: " + std::string(ex.what())); + return; + } + + query_set = dataset->query_set(current_algo_props->query_memory_type); + load_barrier.arrive(state.threads()); + } else { + // All other threads will wait for the first thread to initialize the algo. + load_barrier.wait(state.threads() * 2); + // gbench ensures that all threads are synchronized at the start of the benchmark loop. + // We are accessing shared variables (like current_algo, current_algo_probs) before the + // benchmark loop, therefore the synchronization here is necessary. + } + query_set = dataset->query_set(current_algo_props->query_memory_type); + + /** + * Each thread will manage its own outputs + */ + using index_type = algo_base::index_type; + constexpr size_t kAlignResultBuf = 64; + size_t result_elem_count = k * query_set_size; + result_elem_count = + ((result_elem_count + kAlignResultBuf - 1) / kAlignResultBuf) * kAlignResultBuf; + auto& result_buf = + get_result_buffer_from_global_pool(result_elem_count * (sizeof(float) + sizeof(index_type))); + auto* neighbors_ptr = + reinterpret_cast(result_buf.data(current_algo_props->query_memory_type)); + auto* distances_ptr = reinterpret_cast(neighbors_ptr + result_elem_count); + + { + nvtx_case nvtx{state.name()}; + + std::unique_ptr> a{nullptr}; + try { + dynamic_cast*>(current_algo.get())->copy().swap(a); + } catch (const std::exception& e) { + state.SkipWithError("Algo::copy: " + std::string(e.what())); + return; + } + // Initialize with algo, so that the timer.lap() object can sync with algo::get_sync_stream() + cuda_timer gpu_timer{a}; + auto start = std::chrono::high_resolution_clock::now(); + { + /* See the note above: GPU timing */ + [[maybe_unused]] auto gpu_all = gpu_timer.lap(no_lap_sync); + for (auto _ : state) { + [[maybe_unused]] auto ntx_lap = nvtx.lap(); + [[maybe_unused]] auto gpu_lap = gpu_timer.lap(!no_lap_sync); + try { + a->search(query_set + batch_offset * dataset->dim(), + n_queries, + k, + neighbors_ptr + out_offset * k, + distances_ptr + out_offset * k); + } catch (const std::exception& e) { + state.SkipWithError("Benchmark loop: " + std::string(e.what())); + break; + } + + // advance to the next batch + batch_offset = (batch_offset + queries_stride) % query_set_size; + out_offset = (out_offset + n_queries) % query_set_size; + + queries_processed += n_queries; + } + } + auto end = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast>(end - start).count(); + if (state.thread_index() == 0) { state.counters.insert({{"end_to_end", duration}}); } + state.counters.insert({"Latency", {duration, benchmark::Counter::kAvgIterations}}); + + if (gpu_timer.active()) { + state.counters.insert({"GPU", {gpu_timer.total_time(), benchmark::Counter::kAvgIterations}}); + } + } + + state.SetItemsProcessed(queries_processed); + + // This will be the total number of queries across all threads + state.counters.insert({{"total_queries", queries_processed}}); + + if (state.skipped()) { return; } + + // Each thread calculates recall on their partition of queries. + // evaluate recall + if (dataset->max_k() >= k) { + const std::int32_t* gt = dataset->gt_set(); + const std::uint32_t* filter_bitset = dataset->filter_bitset(MemoryType::kHostMmap); + auto filter = [filter_bitset](std::int32_t i) -> bool { + if (filter_bitset == nullptr) { return true; } + auto word = filter_bitset[i >> 5]; + return word & (1 << (i & 31)); + }; + const std::uint32_t max_k = dataset->max_k(); + result_buf.transfer_data(MemoryType::kHost, current_algo_props->query_memory_type); + auto* neighbors_host = reinterpret_cast(result_buf.data(MemoryType::kHost)); + std::size_t rows = std::min(queries_processed, query_set_size); + std::size_t match_count = 0; + std::size_t total_count = 0; + + // We go through the groundtruth with same stride as the benchmark loop. + size_t out_offset = 0; + size_t batch_offset = (state.thread_index() * n_queries) % query_set_size; + while (out_offset < rows) { + for (std::size_t i = 0; i < n_queries; i++) { + size_t i_orig_idx = batch_offset + i; + size_t i_out_idx = out_offset + i; + if (i_out_idx < rows) { + /* NOTE: recall correctness & filtering + + In the loop below, we filter the ground truth values on-the-fly. + We need enough ground truth values to compute recall correctly though. + But the ground truth file only contains `max_k` values per row; if there are less valid + values than k among them, we overestimate the recall. Essentially, we compare the first + `filter_pass_count` values of the algorithm output, and this counter can be less than `k`. + In the extreme case of very high filtering rate, we may be bypassing entire rows of + results. However, this is still better than no recall estimate at all. + + TODO: consider generating the filtered ground truth on-the-fly + */ + uint32_t filter_pass_count = 0; + for (std::uint32_t l = 0; l < max_k && filter_pass_count < k; l++) { + auto exp_idx = gt[i_orig_idx * max_k + l]; + if (!filter(exp_idx)) { continue; } + filter_pass_count++; + for (std::uint32_t j = 0; j < k; j++) { + auto act_idx = static_cast(neighbors_host[i_out_idx * k + j]); + if (act_idx == exp_idx) { + match_count++; + break; + } + } + } + total_count += filter_pass_count; + } + } + out_offset += n_queries; + batch_offset = (batch_offset + queries_stride) % query_set_size; + } + double actual_recall = static_cast(match_count) / static_cast(total_count); + /* NOTE: recall in the throughput mode & filtering + + When filtering is enabled, `total_count` may vary between individual threads, but we still take + the simple average across in-thread recalls. Strictly speaking, this is incorrect, but it's good + enough under assumption that the filtering is more-or-less uniform. + */ + state.counters.insert({"Recall", {actual_recall, benchmark::Counter::kAvgThreads}}); + } +} + +inline void printf_usage() +{ + ::benchmark::PrintDefaultHelp(); + fprintf( + stdout, + " [--build|--search] \n" + " [--force]\n" + " [--data_prefix=]\n" + " [--index_prefix=]\n" + " [--override_kv=]\n" + " [--mode=\n" + " [--threads=min[:max]]\n" + " [--no-lap-sync]\n" + " .json\n" + "\n" + "Note the non-standard benchmark parameters:\n" + " --build: build mode, will build index\n" + " --search: search mode, will search using the built index\n" + " one and only one of --build and --search should be specified\n" + " --force: force overwriting existing index files\n" + " --data_prefix=:" + " prepend to dataset file paths specified in the .json (default = " + "'data/').\n" + " --index_prefix=:" + " prepend to index file paths specified in the .json (default = " + "'index/').\n" + " --override_kv=:" + " override a build/search key one or more times multiplying the number of configurations;" + " you can use this parameter multiple times to get the Cartesian product of benchmark" + " configs.\n" + " --mode=" + " run the benchmarks in latency (accumulate times spent in each batch) or " + " throughput (pipeline batches and measure end-to-end) mode\n" + " --threads=min[:max] specify the number threads to use for throughput benchmark." + " Power of 2 values between 'min' and 'max' will be used. If only 'min' is specified," + " then a single test is run with 'min' threads. By default min=1, max=.\n" + " --no-lap-sync disable CUDA event synchronization between benchmark iterations. If a GPU" + " algorithm has no sync with CPU, this can make the GPU processing significantly lag behind the" + " CPU scheduling. Then this also hides the scheduling latencies and thus improves the measured" + " throughput (QPS). Note there's a sync at the end of the benchmark loop in any case.\n"); +} + +template +void register_build(std::shared_ptr> dataset, + std::vector& indices, + bool force_overwrite, + bool no_lap_sync) +{ + for (auto& index : indices) { + auto suf = static_cast(index.build_param["override_suffix"]); + auto file_suf = suf; + index.build_param.erase("override_suffix"); + std::replace(file_suf.begin(), file_suf.end(), '/', '-'); + index.file += file_suf; + auto* b = ::benchmark::RegisterBenchmark( + index.name + suf, bench_build, dataset, index, force_overwrite, no_lap_sync); + b->Unit(benchmark::kSecond); + b->MeasureProcessCPUTime(); + b->UseRealTime(); + } +} + +template +void register_search(std::shared_ptr> dataset, + std::vector& indices, + Mode metric_objective, + const std::vector& threads, + bool no_lap_sync) +{ + for (auto& index : indices) { + for (std::size_t i = 0; i < index.search_params.size(); i++) { + auto suf = static_cast(index.search_params[i]["override_suffix"]); + index.search_params[i].erase("override_suffix"); + + auto* b = ::benchmark::RegisterBenchmark( + index.name + suf, bench_search, index, i, dataset, no_lap_sync) + ->Unit(benchmark::kMillisecond) + /** + * The following are important for getting accuracy QPS measurements on both CPU + * and GPU These make sure that + * - `end_to_end` ~ (`Time` * `Iterations`) + * - `items_per_second` ~ (`total_queries` / `end_to_end`) + * - Throughput = `items_per_second` + */ + ->MeasureProcessCPUTime() + ->UseRealTime(); + + if (metric_objective == Mode::kThroughput) { b->ThreadRange(threads[0], threads[1]); } + } + } +} + +template +void dispatch_benchmark(std::string cmdline, + configuration& conf, + bool force_overwrite, + bool build_mode, + bool search_mode, + kv_series override_kv, + Mode metric_objective, + const std::vector& threads, + bool no_lap_sync) +{ + ::benchmark::AddCustomContext("command_line", cmdline); + for (auto [key, value] : host_info()) { + ::benchmark::AddCustomContext(key, value); + } + if (cudart.found()) { + for (auto [key, value] : cuda_info()) { + ::benchmark::AddCustomContext(key, value); + } + } + auto& dataset_conf = conf.get_dataset_conf(); + auto base_file = dataset_conf.base_file; + auto query_file = dataset_conf.query_file; + auto gt_file = dataset_conf.groundtruth_neighbors_file; + auto dataset = + std::make_shared>(dataset_conf.name, + base_file, + dataset_conf.subset_first_row, + dataset_conf.subset_size, + query_file, + dataset_conf.distance, + gt_file, + search_mode ? dataset_conf.filtering_rate : std::nullopt); + ::benchmark::AddCustomContext("dataset", dataset_conf.name); + ::benchmark::AddCustomContext("distance", dataset_conf.distance); + std::vector& indices = conf.get_indices(); + if (build_mode) { + if (file_exists(base_file)) { + log_info("Using the dataset file '%s'", base_file.c_str()); + ::benchmark::AddCustomContext("n_records", std::to_string(dataset->base_set_size())); + ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); + } else { + log_warn("dataset file '%s' does not exist; benchmarking index building is impossible.", + base_file.c_str()); + } + std::vector more_indices{}; + for (auto& index : indices) { + for (auto param : apply_overrides(index.build_param, override_kv)) { + auto modified_index = index; + modified_index.build_param = param; + more_indices.push_back(modified_index); + } + } + std::swap(more_indices, indices); // update the config in case algorithms need to access it + register_build(dataset, indices, force_overwrite, no_lap_sync); + } else if (search_mode) { + if (file_exists(query_file)) { + log_info("Using the query file '%s'", query_file.c_str()); + ::benchmark::AddCustomContext("max_n_queries", std::to_string(dataset->query_set_size())); + ::benchmark::AddCustomContext("dim", std::to_string(dataset->dim())); + if (gt_file.has_value()) { + if (file_exists(*gt_file)) { + log_info("Using the ground truth file '%s'", gt_file->c_str()); + ::benchmark::AddCustomContext("max_k", std::to_string(dataset->max_k())); + } else { + log_warn("Ground truth file '%s' does not exist; the recall won't be reported.", + gt_file->c_str()); + } + } else { + log_warn( + "Ground truth file is not provided; the recall won't be reported. NB: use " + "the 'groundtruth_neighbors_file' alongside the 'query_file' key to specify the " + "path to " + "the ground truth in your conf.json."); + } + } else { + log_warn("Query file '%s' does not exist; benchmarking search is impossible.", + query_file.c_str()); + } + for (auto& index : indices) { + index.search_params = apply_overrides(index.search_params, override_kv); + } + register_search(dataset, indices, metric_objective, threads, no_lap_sync); + } +} + +inline auto parse_bool_flag(const char* arg, const char* pat, bool& result) -> bool +{ + if (strcmp(arg, pat) == 0) { + result = true; + return true; + } + return false; +} + +inline auto parse_string_flag(const char* arg, const char* pat, std::string& result) -> bool +{ + auto n = strlen(pat); + if (strncmp(pat, arg, strlen(pat)) == 0) { + result = arg + n + 1; + return true; + } + return false; +} + +inline auto run_main(int argc, char** argv) -> int +{ + bool force_overwrite = false; + bool build_mode = false; + bool search_mode = false; + bool no_lap_sync = false; + std::string data_prefix = "data"; + std::string index_prefix = "index"; + std::string new_override_kv = ""; + std::string mode = "latency"; + std::string threads_arg_txt = ""; + std::vector threads = {1, -1}; // min_thread, max_thread + kv_series override_kv{}; + + char arg0_default[] = "benchmark"; // NOLINT + char* args_default = arg0_default; + if (!argv) { + argc = 1; + argv = &args_default; + } + if (argc == 1) { + printf_usage(); + return -1; + } + // Save command line for reproducibility. + std::string cmdline(argv[0]); + for (int i = 1; i < argc; i++) { + cmdline += " " + std::string(argv[i]); + } + + char* conf_path = argv[--argc]; + std::ifstream conf_stream(conf_path); + + for (int i = 1; i < argc; i++) { + if (parse_bool_flag(argv[i], "--force", force_overwrite) || + parse_bool_flag(argv[i], "--build", build_mode) || + parse_bool_flag(argv[i], "--search", search_mode) || + parse_bool_flag(argv[i], "--no-lap-sync", no_lap_sync) || + parse_string_flag(argv[i], "--data_prefix", data_prefix) || + parse_string_flag(argv[i], "--index_prefix", index_prefix) || + parse_string_flag(argv[i], "--mode", mode) || + parse_string_flag(argv[i], "--override_kv", new_override_kv) || + parse_string_flag(argv[i], "--threads", threads_arg_txt)) { + if (!threads_arg_txt.empty()) { + auto threads_arg = split(threads_arg_txt, ':'); + threads[0] = std::stoi(threads_arg[0]); + if (threads_arg.size() > 1) { + threads[1] = std::stoi(threads_arg[1]); + } else { + threads[1] = threads[0]; + } + threads_arg_txt = ""; + } + if (!new_override_kv.empty()) { + auto kvv = split(new_override_kv, ':'); + auto key = kvv[0]; + std::vector vals{}; + for (std::size_t j = 1; j < kvv.size(); j++) { + vals.push_back(nlohmann::json::parse(kvv[j])); + } + override_kv.emplace_back(key, vals); + new_override_kv = ""; + } + for (int j = i; j < argc - 1; j++) { + argv[j] = argv[j + 1]; + } + argc--; + i--; + } + } + + Mode metric_objective = Mode::kLatency; + if (mode == "throughput") { metric_objective = Mode::kThroughput; } + + int max_threads = + (metric_objective == Mode::kThroughput) ? std::thread::hardware_concurrency() : 1; + if (threads[1] == -1) threads[1] = max_threads; + + if (metric_objective == Mode::kLatency) { + if (threads[0] != 1 || threads[1] != 1) { + log_warn("Latency mode enabled. Overriding threads arg, running with single thread."); + threads = {1, 1}; + } + } + + if (build_mode == search_mode) { + log_error("One and only one of --build and --search should be specified"); + printf_usage(); + return -1; + } + + if (!conf_stream) { + log_error("Can't open configuration file: %s", conf_path); + return -1; + } + + if (cudart.needed() && !cudart.found()) { + log_warn("cudart library is not found, GPU-based indices won't work."); + } + + auto& conf = bench::configuration::initialize(conf_stream, data_prefix, index_prefix); + std::string dtype = conf.get_dataset_conf().dtype; + + if (dtype == "float") { + dispatch_benchmark(cmdline, + conf, + force_overwrite, + build_mode, + search_mode, + override_kv, + metric_objective, + threads, + no_lap_sync); + } else if (dtype == "half") { + dispatch_benchmark(cmdline, + conf, + force_overwrite, + build_mode, + search_mode, + override_kv, + metric_objective, + threads, + no_lap_sync); + } else if (dtype == "uint8") { + dispatch_benchmark(cmdline, + conf, + force_overwrite, + build_mode, + search_mode, + override_kv, + metric_objective, + threads, + no_lap_sync); + } else if (dtype == "int8") { + dispatch_benchmark(cmdline, + conf, + force_overwrite, + build_mode, + search_mode, + override_kv, + metric_objective, + threads, + no_lap_sync); + } else { + log_error("datatype '%s' is not supported", dtype.c_str()); + return -1; + } + + ::benchmark::Initialize(&argc, argv, printf_usage); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return -1; + ::benchmark::RunSpecifiedBenchmarks(); + ::benchmark::Shutdown(); + // Release a possibly cached algo object, so that it cannot be alive longer than the handle + // to a shared library it depends on (dynamic benchmark executable). + current_algo.reset(); + current_algo_props.reset(); + reset_global_device_resources(); + return 0; +} +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/blob.hpp b/cpp/bench/ann/src/common/blob.hpp new file mode 100644 index 0000000000..3a51372cff --- /dev/null +++ b/cpp/bench/ann/src/common/blob.hpp @@ -0,0 +1,890 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "ann_types.hpp" +#include "util.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +/** RAII wrapper for a file descriptor. */ +struct file_descriptor { + explicit file_descriptor(std::string file_name) : fd_{std::fopen(file_name.c_str(), "r")} + { + if (fd_ == nullptr) { + throw std::runtime_error( + "cuvs::bench::file_descriptor: failed to open a file (std::fopen): errno = " + + std::to_string(errno) + ", " + std::string(strerror(errno)) + ". File name: " + file_name); + } + } + + explicit file_descriptor(size_t file_size_bytes) : fd_{std::tmpfile()} + { + if (fd_ == nullptr) { + throw std::runtime_error( + "cuvs::bench::file_descriptor: failed to open a temporary file (std::tmpfile): errno = " + + std::to_string(errno) + ", " + std::string(strerror(errno))); + } + if (ftruncate(fileno(fd_), file_size_bytes) == -1) { + throw std::runtime_error( + "cuvs::bench::file_descriptor: failed to call `ftruncate` to allocate memory for a " + "temporary file."); + } + } + + // No copies for owning struct + file_descriptor(const file_descriptor& res) = delete; + auto operator=(const file_descriptor& other) -> file_descriptor& = delete; + // Moving is fine + file_descriptor(file_descriptor&& other) : fd_{std::exchange(other.fd_, nullptr)} {} + auto operator=(file_descriptor&& other) -> file_descriptor& + { + std::swap(this->fd_, other.fd_); + return *this; + } + + ~file_descriptor() noexcept + { + if (fd_ != nullptr) { std::fclose(fd_); } + } + + [[nodiscard]] auto value() const -> FILE* { return fd_; } + + private: + FILE* fd_ = nullptr; +}; + +class mmap_error : public std::runtime_error { + private: + int errno_; + + public: + mmap_error(std::string extra_msg) + : std::runtime_error("cuvs::bench::mmap_owner: `mmap` error: Value of errno " + + std::to_string(errno) + ", " + std::string(strerror(errno)) + ". " + + extra_msg), + errno_(errno) + { + } + + [[nodiscard]] auto code() const noexcept { return errno_; } +}; + +/** RAII wrapper for a mmap/munmap. */ +struct mmap_owner { + /** Map a file */ + mmap_owner( + const file_descriptor& descriptor, size_t offset, size_t size, int flags, bool writable = false) + : ptr_{mmap_verbose(size, + writable ? PROT_READ | PROT_WRITE : PROT_READ, + flags, + fileno(descriptor.value()), + offset)}, + size_{size} + { + } + + /** Allocate a new memory (not backed by a file). */ + mmap_owner(size_t size, int flags) + : ptr_{mmap_verbose(size, PROT_READ | PROT_WRITE, flags, -1, 0)}, size_{size} + { + } + + ~mmap_owner() noexcept + { + if (ptr_ != nullptr) { munmap(ptr_, size_); } + } + + // No copies for owning struct + mmap_owner(const mmap_owner& res) = delete; + auto operator=(const mmap_owner& other) -> mmap_owner& = delete; + // Moving is fine + mmap_owner(mmap_owner&& other) + : ptr_{std::exchange(other.ptr_, nullptr)}, size_{std::exchange(other.size_, 0)} + { + } + auto operator=(mmap_owner&& other) -> mmap_owner& + { + std::swap(this->ptr_, other.ptr_); + std::swap(this->size_, other.size_); + return *this; + } + + [[nodiscard]] auto data() const -> void* { return ptr_; } + [[nodiscard]] auto size() const -> size_t { return size_; } + + private: + void* ptr_; + size_t size_; + + static inline auto mmap_verbose(size_t length, int prot, int flags, int fd, off_t offset) -> void* + { + auto ptr = mmap(nullptr, length, prot, flags, fd, offset); + if (ptr == MAP_FAILED) { + std::array buf; + snprintf(buf.data(), + sizeof(buf), + "Failed call: cuvs::bench::mmap_owner:mmap(nullptr, %zu, 0x%08x, 0x%08x, %d, %zd)", + length, + prot, + flags, + fd, + offset); + throw mmap_error{std::string(buf.data())}; + } + return ptr; + } +}; + +/** RAII wrapper for managed memory. */ +struct managed_mem_owner { + explicit managed_mem_owner(size_t size) : size_{size} + { +#ifndef BUILD_CPU_ONLY + auto err_code = cudaMallocManaged(&ptr_, size_); + if (err_code != cudaSuccess) { + ptr_ = nullptr; + throw std::runtime_error{ + "cuvs::bench::managed_mem_owner: call to cudaMallocManaged failed with code " + + std::to_string(err_code)}; + } +#else + throw std::runtime_error{ + "Device functions are not available when built with BUILD_CPU_ONLY flag."}; +#endif + } + + ~managed_mem_owner() noexcept + { + if (ptr_ != nullptr) { +#ifndef BUILD_CPU_ONLY + cudaFree(ptr_); +#endif + } + } + + // No copies for owning struct + managed_mem_owner(const managed_mem_owner& res) = delete; + auto operator=(const managed_mem_owner& other) -> managed_mem_owner& = delete; + // Moving is fine + managed_mem_owner(managed_mem_owner&& other) + : ptr_{std::exchange(other.ptr_, nullptr)}, size_{std::exchange(other.size_, 0)} + { + } + auto operator=(managed_mem_owner&& other) -> managed_mem_owner& + { + std::swap(this->ptr_, other.ptr_); + std::swap(this->size_, other.size_); + return *this; + } + + [[nodiscard]] auto data() const -> void* { return ptr_; } + [[nodiscard]] auto size() const -> size_t { return size_; } + + private: + void* ptr_ = nullptr; + size_t size_; +}; + +/** RAII wrapper for device memory. */ +struct device_mem_owner { + explicit device_mem_owner(size_t size) : size_{size} + { +#ifndef BUILD_CPU_ONLY + auto err_code = cudaMalloc(&ptr_, size_); + if (err_code != cudaSuccess) { + ptr_ = nullptr; + throw std::runtime_error{ + "cuvs::bench::device_mem_owner: call to cudaMalloc failed with code " + + std::to_string(err_code)}; + } +#else + throw std::runtime_error{ + "Device functions are not available when built with BUILD_CPU_ONLY flag."}; +#endif + } + ~device_mem_owner() noexcept + { + if (ptr_ != nullptr) { +#ifndef BUILD_CPU_ONLY + cudaFree(ptr_); +#endif + } + } + // No copies for owning struct + device_mem_owner(const device_mem_owner& res) = delete; + auto operator=(const device_mem_owner& other) -> device_mem_owner& = delete; + // Moving is fine + device_mem_owner(device_mem_owner&& other) + : ptr_{std::exchange(other.ptr_, nullptr)}, size_{std::exchange(other.size_, 0)} + { + } + auto operator=(device_mem_owner&& other) -> device_mem_owner& + { + std::swap(this->ptr_, other.ptr_); + std::swap(this->size_, other.size_); + return *this; + } + + [[nodiscard]] auto data() const -> void* { return ptr_; } + [[nodiscard]] auto size() const -> size_t { return size_; } + + private: + void* ptr_ = nullptr; + size_t size_; +}; + +/** Lazy-initialized file handle. */ +struct file { + explicit file(std::string file_name) : reqsize_or_name_{std::move(file_name)} {} + explicit file(size_t tmp_size_bytes) : reqsize_or_name_{tmp_size_bytes} {} + + // this shouldn't be necessary, but adds extra safety (make sure descriptors are not copied) + file(const file&) = delete; + auto operator=(const file&) -> file& = delete; + file(file&&); + auto operator=(file&&) -> file&; + + [[nodiscard]] auto descriptor() const -> const file_descriptor& + { + if (!descriptor_.has_value()) { + std::visit([&d = descriptor_](auto&& x) { d.emplace(x); }, reqsize_or_name_); + } + return descriptor_.value(); + } + + [[nodiscard]] auto path() const -> std::string + { + return std::holds_alternative(reqsize_or_name_) + ? std::get(reqsize_or_name_) + : ""; + } + + [[nodiscard]] auto size() const -> size_t + { + if (!size_.has_value()) { + size_.emplace(std::visit( + [&s = size_](auto&& x) { + if constexpr (std::is_same_v, size_t>) { + return x; + } else { + struct stat statbuf; + if (stat(x.c_str(), &statbuf) != 0) { + throw std::runtime_error{"cuvs::bench::file::size() error: `stat` failed: " + x}; + } + return static_cast(statbuf.st_size); + } + }, + reqsize_or_name_)); + } + return size_.value(); + } + + [[nodiscard]] auto is_temporary() const -> bool + { + return std::holds_alternative(reqsize_or_name_); + } + + void reset_lazy_state() const + { + descriptor_.reset(); + size_.reset(); + } + + private: + std::variant reqsize_or_name_; + mutable std::optional descriptor_ = std::nullopt; + mutable std::optional size_ = std::nullopt; +}; + +// declare the move constructors explicitly outside of the class declaration to make sure if +// anything is wrong, we catch that at compile time. +inline file::file(file&&) = default; +inline auto file::operator=(file&&) -> file& = default; + +/** + * Lazy-initialized file handle with the size information provided by our .bin format: + * The file always starts with two uint32_t values: [n_rows, n_cols]. + */ +template +struct blob_file : public file { + explicit blob_file(std::string file_name, uint32_t rows_offset = 0, uint32_t rows_limit = 0) + : file{std::move(file_name)}, rows_offset_{rows_offset}, rows_limit_{rows_limit} + { + } + + explicit blob_file(uint32_t n_rows, uint32_t n_cols) + : file{sizeof(T) * n_rows * n_cols + 2 * sizeof(uint32_t)}, rows_offset_{0}, rows_limit_{0} + { + // NB: this forces the file descriptor, thus breaking lazy-initialization. Not sure if it's + // worth refactoring in this case. + std::array h{n_rows, n_cols}; + if (std::fwrite(h.data(), sizeof(uint32_t), h.size(), descriptor().value()) != 2) { + throw std::runtime_error{ + "cuvs::bench::blob_file `fwrite` failed when initializing a tmp file."}; + } + if (std::fflush(descriptor().value()) != 0) { + throw std::runtime_error{ + "cuvs::bench::blob_file `fflush` failed with non-zero code when initializing a tmp file."}; + } + } + + blob_file(const blob_file&) = delete; + auto operator=(const blob_file&) -> blob_file& = delete; + blob_file(blob_file&&); + auto operator=(blob_file&&) -> blob_file&; + + [[nodiscard]] auto n_rows() const -> uint32_t { return header()[0]; } + [[nodiscard]] auto n_cols() const -> uint32_t { return header()[1]; } + + [[nodiscard]] auto rows_offset() const -> uint32_t { return rows_offset_; } + [[nodiscard]] auto rows_limit() const -> uint32_t + { + auto rows_max = n_rows() - std::min(rows_offset(), n_rows()); // available rows + return rows_limit_ == 0 ? rows_max : std::min(rows_limit_, rows_max); // limited rows + } + + void reset_lazy_state() const + { + file::reset_lazy_state(); + header_.reset(); + } + + private: + mutable std::optional> header_ = std::nullopt; + uint32_t rows_offset_; + uint32_t rows_limit_; + + [[nodiscard]] auto header() const -> const std::array& + { + if (!header_.has_value()) { + std::array h; + std::rewind(descriptor().value()); + if (std::fread(h.data(), sizeof(uint32_t), h.size(), descriptor().value()) != 2) { + throw std::runtime_error{"cuvs::bench::blob_file read header of bin file failed: " + + path()}; + } + header_.emplace(h); + } + return header_.value(); + } +}; + +// declare the move constructors explicitly outside of the class declaration to make sure if +// anything is wrong, we catch that at compile time. +template +inline blob_file::blob_file(blob_file&&) = default; +template +inline auto blob_file::operator=(blob_file&&) -> blob_file& = default; + +/** Lazily map or copy the file content onto host memory. */ +template +struct blob_mmap { + explicit blob_mmap(blob_file&& file, + bool copy_in_memory = false, + HugePages hugepages_2mb = HugePages::kDisable) + : file_{std::move(file)}, + copy_in_memory_{copy_in_memory}, + hugepages_2mb_requested_{hugepages_2mb}, + hugepages_2mb_actual_{hugepages_2mb > HugePages::kDisable} + { + } + explicit blob_mmap(std::string file_name, + uint32_t rows_offset = 0, + uint32_t rows_limit = 0, + bool copy_in_memory = false, + HugePages hugepages_2mb = HugePages::kDisable) + : blob_mmap{ + blob_file{std::move(file_name), rows_offset, rows_limit}, copy_in_memory, hugepages_2mb} + { + } + + private: + blob_file file_; + bool copy_in_memory_; + HugePages hugepages_2mb_requested_; + mutable bool hugepages_2mb_actual_; + + mutable std::optional> handle_; + + [[nodiscard]] auto handle() const -> const std::tuple& + { + if (!handle_.has_value()) { + size_t page_size = hugepages_2mb_actual_ ? 1024ull * 1024ull * 2ull : sysconf(_SC_PAGE_SIZE); + int flags = 0; + if (hugepages_2mb_actual_) { flags |= MAP_HUGETLB | MAP_HUGE_2MB; } + size_t data_start = sizeof(T) * file_.rows_offset() * file_.n_cols() + sizeof(uint32_t) * 2; + size_t data_end = sizeof(T) * file_.rows_limit() * file_.n_cols() + data_start; + + try { + if (copy_in_memory_) { + // Copy the content in-memory + flags |= MAP_ANONYMOUS | MAP_PRIVATE; + size_t size = data_end - data_start; + mmap_owner owner{size, flags}; + std::fseek(file_.descriptor().value(), data_start, SEEK_SET); + auto n_elems = + static_cast(file_.rows_limit()) * static_cast(file_.n_cols()); + if (std::fread(owner.data(), sizeof(T), n_elems, file_.descriptor().value()) != n_elems) { + throw std::runtime_error{"cuvs::bench::blob_mmap() fread " + file_.path() + " failed"}; + } + handle_.emplace(std::move(owner), 0); + } else { + // Map the file + // If this is a temporary file, we're supposed to write to it, hence MAP_SHARED. + flags |= file_.is_temporary() ? MAP_SHARED : MAP_PRIVATE; + size_t mmap_start = (data_start / page_size) * page_size; + size_t mmap_size = data_end - mmap_start; + handle_.emplace( + mmap_owner{file_.descriptor(), mmap_start, mmap_size, flags, file_.is_temporary()}, + data_start - mmap_start); + } + } catch (const mmap_error& e) { + bool hugepages_2mb_asked = hugepages_2mb_requested_ == HugePages::kAsk || + hugepages_2mb_requested_ == HugePages::kRequire; + if (e.code() == EPERM && hugepages_2mb_asked && hugepages_2mb_actual_) { + if (hugepages_2mb_requested_ == HugePages::kRequire) { + log_warn( + "cuvs::bench::blob_mmap: `mmap` failed to map due to EPERM, which is likely caused " + "by the permissions issue. You either need a CAP_IPC_LOCK capability or run the " + "program with sudo. We will try again without huge pages."); + } + hugepages_2mb_actual_ = false; + return handle(); + } + if (e.code() == EINVAL && hugepages_2mb_asked && hugepages_2mb_actual_ && + !copy_in_memory_) { + if (hugepages_2mb_requested_ == HugePages::kRequire) { + log_warn( + "cuvs::bench::blob_mmap: `mmap` failed to map due to EINVAL, which is likely caused " + "by the file system not supporting huge pages. We will try again without huge " + "pages."); + } + hugepages_2mb_actual_ = false; + return handle(); + } + throw; // The error is not due to huge pages or otherwise unrecoverable + } + } + return handle_.value(); + } + + public: + [[nodiscard]] auto data() const -> T* + { + auto& [owner, offset] = handle(); + return reinterpret_cast(reinterpret_cast(owner.data()) + offset); + } + + [[nodiscard]] auto is_in_memory() const -> bool { return copy_in_memory_; } + [[nodiscard]] auto is_hugepage() const -> bool { return hugepages_2mb_actual_; } + /** + * Enabling hugepages is not always possible. For convenience, we may silently disable it in some + * cases. This function helps to decide whether the current setting is compatible with the + * request.R + */ + [[nodiscard]] auto hugepage_compliant(HugePages request) const -> bool + { + if (request == hugepages_2mb_requested_) { + // whatever the actual state is, the result would be the same if we recreated + // the mapping. + return true; + } + bool hp_enabled = hugepages_2mb_actual_ && request > HugePages::kDisable; + bool hp_disabled = !hugepages_2mb_actual_ && request == HugePages::kDisable; + bool hp_not_forced = !hugepages_2mb_actual_ && request == HugePages::kAsk; + return hp_enabled || hp_disabled || hp_not_forced; + } + [[nodiscard]] auto n_rows() const -> uint32_t { return file_.rows_limit(); } + [[nodiscard]] auto n_cols() const -> uint32_t { return file_.n_cols(); } + [[nodiscard]] auto size() const -> size_t { return sizeof(T) * n_rows() * n_cols(); } + [[nodiscard]] auto unmap() && noexcept -> blob_file + { + // If we used mmap on a temporary file, then it was writable. + // Then there's a chance the user wrote something to the mmap. + // Then we must ensure the changes are visible in the file before migrating it. + bool flush_writes = file_.is_temporary() && handle_.has_value(); + if (flush_writes) { + auto& [owner, _] = handle(); + msync(owner.data(), owner.size(), MS_SYNC | MS_INVALIDATE); + } + handle_.reset(); + if (flush_writes) { std::fflush(file_.descriptor().value()); } + return blob_file{std::move(file_)}; + } + [[nodiscard]] auto release() && noexcept -> blob_file { return std::move(*this).unmap(); } + + void reset_lazy_state() const + { + file_.reset_lazy_state(); + hugepages_2mb_actual_ = hugepages_2mb_requested_ > HugePages::kDisable; + } +}; + +template +struct blob_pinned { + private: + mutable blob_mmap blob_; + mutable void* ptr_ = nullptr; + + public: + explicit blob_pinned(blob_mmap&& blob) : blob_{std::move(blob)} {} + // First map the file and then register to CUDA. + // NB: as per docs, huge pages are not supported. + explicit blob_pinned(blob_file&& blob, bool copy_in_memory = true) + : blob_{std::move(blob), copy_in_memory, false} + { + } + explicit blob_pinned(std::string file_name, + uint32_t rows_offset = 0, + uint32_t rows_limit = 0, + bool copy_in_memory = true) + : blob_pinned{blob_file{file_name, rows_offset, rows_limit}, copy_in_memory} + { + } + + ~blob_pinned() noexcept { reset_lazy_state(); } + + // No copies for owning struct + blob_pinned(const blob_pinned& res) = delete; + auto operator=(const blob_pinned& other) -> blob_pinned& = delete; + // Moving is fine + blob_pinned(blob_pinned&& other) : blob_{std::move(other.blob_)}, ptr_{other.ptr_} + { + other.ptr_ = nullptr; + } + auto operator=(blob_pinned&& other) -> blob_pinned& + { + std::swap(this->blob_, other.blob_); + std::swap(this->ptr_, other.ptr_); + return *this; + } + + [[nodiscard]] auto data() const -> T* + { + if (ptr_ == nullptr) { + void* ptr = reinterpret_cast(blob_.data()); +#ifndef BUILD_CPU_ONLY + int flags = cudaHostRegisterDefault; + auto error_code = cudaSuccess; + if (!blob_.is_in_memory()) { + flags = cudaHostRegisterIoMemory | cudaHostRegisterReadOnly; + error_code = cudaHostRegister(ptr, blob_.size(), flags); + if (error_code == cudaErrorNotSupported) { + // Sometimes read-only is not supported + flags = cudaHostRegisterIoMemory; + error_code = cudaHostRegister(ptr, blob_.size(), error_code); + } + } else { + error_code = cudaHostRegister(ptr, blob_.size(), cudaHostRegisterDefault); + } + if (error_code == cudaErrorInvalidValue && (!blob_.is_in_memory() || blob_.is_hugepage())) { + auto hugepage = + (blob_.is_hugepage() && blob_.is_in_memory()) ? HugePages::kAsk : HugePages::kDisable; + auto file = std::move(blob_).release(); + blob_ = blob_mmap{std::move(file), true, hugepage}; + return data(); + } + if (error_code != cudaSuccess) { + log_error( + "cuvs::bench::blob_pinned: cudaHostRegister(%p, %zu, %d)", ptr, blob_.size(), flags); + throw std::runtime_error{ + "cuvs::bench::blob_pinned: call to cudaHostRegister failed with code " + + std::to_string(error_code)}; + } +#endif + ptr_ = ptr; + } + return reinterpret_cast(ptr_); + } + + [[nodiscard]] auto unpin() && noexcept -> blob_mmap + { + // unregister the memory before passing it to a third party + if (ptr_ != nullptr) { +#ifndef BUILD_CPU_ONLY + cudaHostUnregister(ptr_); +#endif + ptr_ = nullptr; + } + return blob_mmap{std::move(blob_)}; + } + void reset_lazy_state() const + { + if (ptr_ != nullptr) { +#ifndef BUILD_CPU_ONLY + cudaHostUnregister(ptr_); +#endif + ptr_ = nullptr; + } + blob_.reset_lazy_state(); + } + + [[nodiscard]] auto is_in_memory() const noexcept -> bool { return true; } + [[nodiscard]] auto is_hugepage() const noexcept -> bool { return blob_.is_hugepage(); } + [[nodiscard]] auto hugepage_compliant(HugePages request) const -> bool + { + return blob_.hugepage_compliant(request); + } + [[nodiscard]] auto n_rows() const -> uint32_t { return blob_.n_rows(); } + [[nodiscard]] auto n_cols() const -> uint32_t { return blob_.n_cols(); } + [[nodiscard]] auto size() const -> size_t { return blob_.size(); } + [[nodiscard]] auto release() && noexcept -> blob_file + { + return std::move(*this).unpin().release(); + } +}; + +template +struct blob_copying { + private: + blob_mmap blob_; + mutable std::optional mem_ = std::nullopt; + + public: + explicit blob_copying(blob_mmap&& blob) : blob_{std::move(blob)} {} + // First map the file and then copy it to device; use huge pages for faster copy + explicit blob_copying(blob_file&& blob) : blob_{std::move(blob), false, HugePages::kAsk} {} + explicit blob_copying(std::string file_name, uint32_t rows_offset = 0, uint32_t rows_limit = 0) + : blob_copying{blob_file{file_name, rows_offset, rows_limit}} + { + } + + [[nodiscard]] auto data() const -> T* + { + if (!mem_.has_value()) { + mem_.emplace(blob_.size()); +#ifndef BUILD_CPU_ONLY + auto error_code = cudaMemcpy(mem_->data(), blob_.data(), blob_.size(), cudaMemcpyDefault); + if (error_code != cudaSuccess) { + throw std::runtime_error{"cuvs::bench::blob_device: call to cudaMemcpy failed with code " + + std::to_string(error_code)}; + } +#endif + } + return reinterpret_cast(mem_->data()); + } + + [[nodiscard]] auto free() && noexcept -> blob_mmap + { + mem_.reset(); + return blob_mmap{std::move(blob_)}; + } + void reset_lazy_state() const + { + mem_.reset(); + blob_.reset_lazy_state(); + } + + [[nodiscard]] auto is_in_memory() const noexcept -> bool { return true; } + [[nodiscard]] auto is_hugepage() const noexcept -> bool { return blob_.is_hugepage(); } + // For copying CUDA allocation, the hugepage setting is not relevant at all. + [[nodiscard]] auto hugepage_compliant(HugePages request) const -> bool { return true; } + [[nodiscard]] auto n_rows() const -> uint32_t { return blob_.n_rows(); } + [[nodiscard]] auto n_cols() const -> uint32_t { return blob_.n_cols(); } + [[nodiscard]] auto size() const -> size_t { return blob_.size(); } + [[nodiscard]] auto release() && noexcept -> blob_file + { + return std::move(*this).free().release(); + } +}; + +template +using blob_device = blob_copying; +template +using blob_managed = blob_copying; + +/** + * @brief A blob is a single contiguous piece of data. + * + * It can reside in host, managed, or device memory, it can be pinned in physical memory or backed + * by a filesystem. You can also control whether to use huge pages (2MB) when it is in host memory. + * + * The blob data is a one- or two-dimensional typed array (it has n_rows and n_cols properties). + * + * The blob tries to be lazy accessing the data. It reads or copies the data only when it is + * requested. This allows the benchmarking executable be more resilient; for example, a blob can + * point to a non-existent query dataset when the benchmark is in the index build mode. + * + * @tparam T the data type. + */ +template +struct blob { + private: + using blob_type = std::variant, blob_pinned, blob_device, blob_managed>; + mutable blob_type value_; + + [[nodiscard]] auto data_mmap(bool in_memory, HugePages request_hugepages_2mb) const -> T* + { + if (auto* v = std::get_if>(&value_)) { + if (v->is_in_memory() == in_memory && v->hugepage_compliant(request_hugepages_2mb)) { + return v->data(); + } + } + blob_type tmp{std::move(value_)}; + value_ = std::visit( + [in_memory, request_hugepages_2mb](auto&& val) { + return blob_mmap{std::move(val).release(), in_memory, request_hugepages_2mb}; + }, + std::move(tmp)); + + return data(); + } + + [[nodiscard]] auto data_pinned(HugePages request_hugepages_2mb) const -> T* + { + // The requested type is there + if (auto* v = std::get_if>(&value_)) { + if (v->hugepage_compliant(request_hugepages_2mb)) { return v->data(); } + } + // If there's already an mmap allocation, we just need to pin it. + if (auto* v = std::get_if>(&value_)) { + if (v->hugepage_compliant(request_hugepages_2mb)) { + blob_mmap tmp{std::move(*v)}; + return value_.template emplace>(std::move(tmp)).data(); + } + } + // otherwise do full reset + blob_type tmp{std::move(value_)}; + value_ = std::visit( + [request_hugepages_2mb](auto&& val) { + blob_mmap tmp{std::move(val).release(), true, request_hugepages_2mb}; + return blob_pinned{std::move(tmp)}; + }, + std::move(value_)); + + return data(); + } + + [[nodiscard]] auto data_device() const -> T* + { + // The requested type is there + if (auto* v = std::get_if>(&value_)) { return v->data(); } + // otherwise do full reset + blob_type tmp{std::move(value_)}; + value_ = std::visit([](auto&& val) { return blob_device{std::move(val).release()}; }, + std::move(tmp)); + return data(); + } + + [[nodiscard]] auto data_managed() const -> T* + { + // The requested type is there + if (auto* v = std::get_if>(&value_)) { return v->data(); } + // otherwise do full reset + blob_type tmp{std::move(value_)}; + value_ = std::visit([](auto&& val) { return blob_managed{std::move(val).release()}; }, + std::move(tmp)); + return data(); + } + + public: + explicit blob(std::string file_name, + uint32_t rows_offset = 0, + uint32_t rows_limit = 0, + bool copy_in_memory = false, + HugePages hugepages_2mb = HugePages::kDisable) + : value_{std::in_place_type>, + std::move(file_name), + rows_offset, + rows_limit, + copy_in_memory, + hugepages_2mb} + { + } + + template + explicit blob(VariantT&& blob_variant) : value_{std::move(blob_variant)} + { + } + + [[nodiscard]] auto data() const -> T* + { + return std::visit([](auto&& val) { return val.data(); }, value_); + } + + [[nodiscard]] auto data(MemoryType memory_type, + HugePages request_hugepages_2mb = HugePages::kDisable) const -> T* + { + switch (memory_type) { + case MemoryType::kHost: return data_mmap(true, request_hugepages_2mb); + case MemoryType::kHostMmap: return data_mmap(false, request_hugepages_2mb); + case MemoryType::kHostPinned: + if (request_hugepages_2mb > HugePages::kDisable) { + log_error( + "cuvs::bench::blob::data(): huge pages are requested but not supported by " + "cudaHostRegister at the moment. We will try nevertheless..."); + } + return data_pinned(request_hugepages_2mb); + case MemoryType::kDevice: return data_device(); // hugepages are not relevant here + case MemoryType::kManaged: return data_managed(); // hugepages are not relevant here + default: + throw std::runtime_error{"cuvs::bench::blob::data(): unexpected memory type " + + std::to_string(static_cast(memory_type))}; + } + } + + [[nodiscard]] auto is_in_memory() const noexcept -> bool + { + return std::visit([](auto&& val) { return val.is_in_memory(); }, value_); + } + [[nodiscard]] auto is_hugepage() const noexcept -> bool + { + return std::visit([](auto&& val) { return val.is_hugepage(); }, value_); + } + [[nodiscard]] auto n_rows() const -> uint32_t + { + return std::visit([](auto&& val) { return val.n_rows(); }, value_); + } + + [[nodiscard]] auto n_cols() const -> uint32_t + { + return std::visit([](auto&& val) { return val.n_cols(); }, value_); + } + /** Size of the blob content in bytes (doesn't include file header). */ + [[nodiscard]] auto size() const -> size_t + { + return std::visit([](auto&& val) { return val.size(); }, value_); + } + /** + * Reset the hidden internal state, e.g. release the memory allocation or close a file descriptor. + * This is useful in case when accessing the blob resulted in an error/invalid state, which should + * be reproduced at a later point in time. + */ + void reset_lazy_state() const + { + std::visit([](auto&& val) { val.reset_lazy_state(); }, value_); + } +}; + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/conf.hpp b/cpp/bench/ann/src/common/conf.hpp new file mode 100644 index 0000000000..b19cad5fed --- /dev/null +++ b/cpp/bench/ann/src/common/conf.hpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "util.hpp" + +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +class configuration { + public: + struct index { + std::string name; + std::string algo; + nlohmann::json build_param; + std::string file; + + int batch_size; + int k; + std::vector search_params; + }; + + struct dataset_conf { + std::string name; + std::string base_file; + // use only a subset of base_file, + // the range of rows is [subset_first_row, subset_first_row + subset_size) + // however, subset_size = 0 means using all rows after subset_first_row + // that is, the subset is [subset_first_row, #rows in base_file) + uint32_t subset_first_row{0}; + uint32_t subset_size{0}; + std::string query_file; + std::string distance; + std::optional groundtruth_neighbors_file{std::nullopt}; + + // data type of input dataset, possible values ["float", "int8", "uint8"] + std::string dtype; + + std::optional filtering_rate{std::nullopt}; + }; + + [[nodiscard]] inline auto get_dataset_conf() const -> const dataset_conf& + { + return dataset_conf_; + } + [[nodiscard]] inline auto get_dataset_conf() -> dataset_conf& { return dataset_conf_; } + [[nodiscard]] inline auto get_indices() const -> const std::vector& { return indices_; }; + [[nodiscard]] inline auto get_indices() -> std::vector& { return indices_; }; + + /** The benchmark initializes the configuration once and has a chance to modify it during the + * setup. */ + static inline auto initialize(std::istream& conf_stream, + std::string data_prefix, + std::string index_prefix) -> configuration& + { + singleton_ = + std::unique_ptr(new configuration{conf_stream, data_prefix, index_prefix}); + return *singleton_; + } + + /** Any algorithm can access the benchmark configuration as an immutable context. */ + [[nodiscard]] static inline auto singleton() -> const configuration& { return *singleton_; } + + private: + explicit inline configuration(std::istream& conf_stream, + std::string data_prefix, + std::string index_prefix) + { + // to enable comments in json + auto conf = nlohmann::json::parse(conf_stream, nullptr, true, true); + + parse_dataset(conf.at("dataset"), data_prefix); + parse_index(conf.at("index"), conf.at("search_basic_param"), index_prefix); + } + + inline void parse_dataset(const nlohmann::json& conf, std::string data_prefix) + { + dataset_conf_.name = conf.at("name"); + dataset_conf_.base_file = combine_path(data_prefix, conf.at("base_file")); + dataset_conf_.query_file = combine_path(data_prefix, conf.at("query_file")); + dataset_conf_.distance = conf.at("distance"); + if (conf.contains("filtering_rate")) { + dataset_conf_.filtering_rate.emplace(conf.at("filtering_rate")); + } + + if (conf.contains("groundtruth_neighbors_file")) { + dataset_conf_.groundtruth_neighbors_file = + combine_path(data_prefix, conf.at("groundtruth_neighbors_file")); + } + if (conf.contains("subset_first_row")) { + dataset_conf_.subset_first_row = conf.at("subset_first_row"); + } + if (conf.contains("subset_size")) { dataset_conf_.subset_size = conf.at("subset_size"); } + + if (conf.contains("dtype")) { + dataset_conf_.dtype = conf.at("dtype"); + } else { + auto filename = dataset_conf_.base_file; + if (filename.size() > 6 && filename.compare(filename.size() - 6, 6, "f16bin") == 0) { + dataset_conf_.dtype = "half"; + } else if (filename.size() > 9 && + filename.compare(filename.size() - 9, 9, "fp16.fbin") == 0) { + dataset_conf_.dtype = "half"; + } else if (filename.size() > 4 && filename.compare(filename.size() - 4, 4, "fbin") == 0) { + dataset_conf_.dtype = "float"; + } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "u8bin") == 0) { + dataset_conf_.dtype = "uint8"; + } else if (filename.size() > 5 && filename.compare(filename.size() - 5, 5, "i8bin") == 0) { + dataset_conf_.dtype = "int8"; + } else { + log_error("Could not determine data type of the dataset %s", filename.c_str()); + } + } + } + inline void parse_index(const nlohmann::json& index_conf, + const nlohmann::json& search_basic_conf, + std::string index_prefix) + { + const int batch_size = search_basic_conf.at("batch_size"); + const int k = search_basic_conf.at("k"); + + for (const auto& conf : index_conf) { + index index; + index.name = conf.at("name"); + index.algo = conf.at("algo"); + index.build_param = conf.at("build_param"); + index.file = combine_path(index_prefix, conf.at("file")); + index.batch_size = batch_size; + index.k = k; + + for (auto param : conf.at("search_params")) { + /* ### Special parameters for backward compatibility ### + + - Local values of `k` and `n_queries` take priority. + - The legacy "batch_size" renamed to `n_queries`. + - Basic search params are used otherwise. + */ + if (!param.contains("k")) { param["k"] = k; } + if (!param.contains("n_queries")) { + if (param.contains("batch_size")) { + param["n_queries"] = param["batch_size"]; + param.erase("batch_size"); + } else { + param["n_queries"] = batch_size; + } + } + index.search_params.push_back(param); + } + + indices_.push_back(index); + } + } + + dataset_conf dataset_conf_; + std::vector indices_; + + static inline std::unique_ptr singleton_ = nullptr; +}; + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp new file mode 100644 index 0000000000..dd0e015af7 --- /dev/null +++ b/cpp/bench/ann/src/common/cuda_huge_page_resource.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include +#include + +#include + +#include +#include + +namespace raft::mr { +/** + * @brief `device_memory_resource` derived class that uses mmap to allocate memory. + * This class enables memory allocation using huge pages. + * It is assumed that the allocated memory is directly accessible on device. This currently only + * works on GH systems. + * + * TODO(tfeher): consider improving or removing this helper once we made progress with + * https://github.com/rapidsai/raft/issues/1819 + */ +class cuda_huge_page_resource final : public rmm::mr::device_memory_resource { + public: + cuda_huge_page_resource() = default; + ~cuda_huge_page_resource() override = default; + cuda_huge_page_resource(cuda_huge_page_resource const&) = default; + cuda_huge_page_resource(cuda_huge_page_resource&&) = default; + auto operator=(cuda_huge_page_resource const&) -> cuda_huge_page_resource& = default; + auto operator=(cuda_huge_page_resource&&) -> cuda_huge_page_resource& = default; + + private: + /** + * @brief Allocates memory of size at least `bytes` using cudaMalloc. + * + * The returned pointer has at least 256B alignment. + * + * @note Stream argument is ignored + * + * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled + * + * @param bytes The size, in bytes, of the allocation + * @return void* Pointer to the newly allocated memory + */ + auto do_allocate(std::size_t bytes, rmm::cuda_stream_view) -> void* override + { + void* addr{nullptr}; + addr = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { RAFT_FAIL("huge_page_resource::MAP FAILED"); } + if (madvise(addr, bytes, MADV_HUGEPAGE) == -1) { + munmap(addr, bytes); + RAFT_FAIL("huge_page_resource::madvise MADV_HUGEPAGE"); + } + memset(addr, 0, bytes); + return addr; + } + + /** + * @brief Deallocate memory pointed to by \p p. + * + * @note Stream argument is ignored. + * + * @throws Nothing. + * + * @param p Pointer to be deallocated + */ + void do_deallocate(void* ptr, std::size_t size, rmm::cuda_stream_view) noexcept override + { + if (munmap(ptr, size) == -1) { RAFT_LOG_ERROR("huge_page_resource::munmap failed"); } + } + + /** + * @brief Compare this resource to another. + * + * Two cuda_huge_page_resources always compare equal, because they can each + * deallocate memory allocated by the other. + * + * @throws Nothing. + * + * @param other The other resource to compare to + * @return true If the two resources are equivalent + * @return false If the two resources are not equal + */ + [[nodiscard]] auto do_is_equal(device_memory_resource const& other) const noexcept + -> bool override + { + return dynamic_cast(&other) != nullptr; + } +}; +} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_pinned_resource.hpp b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp new file mode 100644 index 0000000000..5d03253429 --- /dev/null +++ b/cpp/bench/ann/src/common/cuda_pinned_resource.hpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +#include + +namespace raft::mr { +/** + * @brief `device_memory_resource` derived class that uses cudaMallocHost/Free for + * allocation/deallocation. + * + * This is almost the same as rmm::mr::host::pinned_memory_resource, but it has + * device_memory_resource as base class. Pinned memory can be accessed from device, + * and using this allocator we can create device_mdarray backed by pinned allocator. + * + * TODO(tfeher): it would be preferred to just rely on the existing allocator from rmm + * (pinned_memory_resource), but that is incompatible with the container_policy class + * for device matrix, because the latter expects a device_memory_resource. We shall + * revise this once we progress with Issue https://github.com/rapidsai/raft/issues/1819 + */ +class cuda_pinned_resource final : public rmm::mr::device_memory_resource { + public: + cuda_pinned_resource() = default; + ~cuda_pinned_resource() override = default; + cuda_pinned_resource(cuda_pinned_resource const&) = default; + cuda_pinned_resource(cuda_pinned_resource&&) = default; + auto operator=(cuda_pinned_resource const&) -> cuda_pinned_resource& = default; + auto operator=(cuda_pinned_resource&&) -> cuda_pinned_resource& = default; + + private: + /** + * @brief Allocates memory of size at least `bytes` using cudaMalloc. + * + * The returned pointer has at least 256B alignment. + * + * @note Stream argument is ignored + * + * @throws `rmm::bad_alloc` if the requested allocation could not be fulfilled + * + * @param bytes The size, in bytes, of the allocation + * @return void* Pointer to the newly allocated memory + */ + auto do_allocate(std::size_t bytes, rmm::cuda_stream_view) -> void* override + { + void* ptr{nullptr}; + RMM_CUDA_TRY_ALLOC(cudaMallocHost(&ptr, bytes)); + return ptr; + } + + /** + * @brief Deallocate memory pointed to by \p p. + * + * @note Stream argument is ignored. + * + * @throws Nothing. + * + * @param p Pointer to be deallocated + */ + void do_deallocate(void* ptr, std::size_t, rmm::cuda_stream_view) noexcept override + { + RMM_ASSERT_CUDA_SUCCESS(cudaFreeHost(ptr)); + } + + /** + * @brief Compare this resource to another. + * + * Two cuda_pinned_resources always compare equal, because they can each + * deallocate memory allocated by the other. + * + * @throws Nothing. + * + * @param other The other resource to compare to + * @return true If the two resources are equivalent + * @return false If the two resources are not equal + */ + [[nodiscard]] auto do_is_equal(device_memory_resource const& other) const noexcept + -> bool override + { + return dynamic_cast(&other) != nullptr; + } +}; +} // namespace raft::mr diff --git a/cpp/bench/ann/src/common/cuda_stub.hpp b/cpp/bench/ann/src/common/cuda_stub.hpp new file mode 100644 index 0000000000..0ee087a11d --- /dev/null +++ b/cpp/bench/ann/src/common/cuda_stub.hpp @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* +The content of this header is governed by two preprocessor definitions: + + - BUILD_CPU_ONLY - whether none of the CUDA functions are used. + - ANN_BENCH_LINK_CUDART - dynamically link against this string if defined. + +___________________________________________________________________________________ +|BUILD_CPU_ONLY | ANN_BENCH_LINK_CUDART | cudart | cuda_runtime_api.h | +| | | found | needed | included | +|---------------|-----------------------|-----------|---------|--------------------| +| ON | | false | false | NO | +| ON | "cudart.so.xx.xx" | false | false | NO | +| OFF | | true | true | YES | +| OFF | "cudart.so.xx.xx" | | true | YES | +------------------------------------------------------------------------------------ +*/ + +#pragma once + +#ifndef BUILD_CPU_ONLY +#include +#include +#ifdef ANN_BENCH_LINK_CUDART +#include + +#include +#endif +#else +#include + +typedef void* cudaStream_t; +typedef void* cudaEvent_t; +typedef uint16_t half; +#endif + +namespace cuvs::bench { + +struct cuda_lib_handle { + void* handle{nullptr}; + explicit cuda_lib_handle() + { +#ifdef ANN_BENCH_LINK_CUDART + constexpr int kFlags = RTLD_NOW | RTLD_GLOBAL | RTLD_DEEPBIND | RTLD_NODELETE; + // The full name of the linked cudart library 'cudart.so.MAJOR.MINOR.PATCH' + char libname[] = ANN_BENCH_LINK_CUDART; // NOLINT + handle = dlopen(ANN_BENCH_LINK_CUDART, kFlags); + if (handle != nullptr) { return; } + // try strip the PATCH + auto p = strrchr(libname, '.'); + p[0] = 0; + handle = dlopen(libname, kFlags); + if (handle != nullptr) { return; } + // try set the MINOR version to 0 + p = strrchr(libname, '.'); + p[1] = '0'; + p[2] = 0; + handle = dlopen(libname, kFlags); + if (handle != nullptr) { return; } + // try strip the MINOR + p[0] = 0; + handle = dlopen(libname, kFlags); + if (handle != nullptr) { return; } + // try strip the MAJOR + p = strrchr(libname, '.'); + p[0] = 0; + handle = dlopen(libname, kFlags); +#endif + } + ~cuda_lib_handle() noexcept + { +#ifdef ANN_BENCH_LINK_CUDART + if (handle != nullptr) { dlclose(handle); } +#endif + } + + template + auto sym(const char* name) -> Symbol + { +#ifdef ANN_BENCH_LINK_CUDART + return reinterpret_cast(dlsym(handle, name)); +#else + return nullptr; +#endif + } + + /** Whether this is NOT a cpu-only package. */ + [[nodiscard]] constexpr inline auto needed() const -> bool + { +#if defined(BUILD_CPU_ONLY) + return false; +#else + return true; +#endif + } + + /** CUDA found, either at compile time or at runtime. */ + [[nodiscard]] inline auto found() const -> bool + { +#if defined(BUILD_CPU_ONLY) + return false; +#elif defined(ANN_BENCH_LINK_CUDART) + return handle != nullptr; +#else + return true; +#endif + } +}; + +static inline cuda_lib_handle cudart{}; + +#ifdef ANN_BENCH_LINK_CUDART +namespace stub { + +[[gnu::weak, gnu::noinline]] auto cuda_memcpy(void* dst, + const void* src, + size_t count, + enum cudaMemcpyKind kind) -> cudaError_t +{ + return cudaSuccess; +} + +[[gnu::weak, gnu::noinline]] auto cuda_malloc(void** ptr, size_t size) -> cudaError_t +{ + *ptr = nullptr; + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_memset(void* devPtr, int value, size_t count) -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_free(void* devPtr) -> cudaError_t { return cudaSuccess; } +[[gnu::weak, gnu::noinline]] auto cuda_stream_create(cudaStream_t* pStream) -> cudaError_t +{ + *pStream = nullptr; + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_stream_create_with_flags(cudaStream_t* pStream, + unsigned int flags) -> cudaError_t +{ + *pStream = nullptr; + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_stream_destroy(cudaStream_t pStream) -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_device_synchronize() -> cudaError_t { return cudaSuccess; } + +[[gnu::weak, gnu::noinline]] auto cuda_stream_synchronize(cudaStream_t pStream) -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_event_create(cudaEvent_t* event) -> cudaError_t +{ + *event = nullptr; + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_event_record(cudaEvent_t event, cudaStream_t stream) + -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_event_synchronize(cudaEvent_t event) -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_event_elapsed_time(float* ms, + cudaEvent_t start, + cudaEvent_t end) -> cudaError_t +{ + *ms = 0; + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_event_destroy(cudaEvent_t event) -> cudaError_t +{ + return cudaSuccess; +} +[[gnu::weak, gnu::noinline]] auto cuda_get_device(int* device) -> cudaError_t +{ + *device = 0; + return cudaSuccess; +}; +[[gnu::weak, gnu::noinline]] auto cuda_driver_get_version(int* driver) -> cudaError_t +{ + *driver = 0; + return cudaSuccess; +}; +[[gnu::weak, gnu::noinline]] auto cuda_runtime_get_version(int* runtime) -> cudaError_t +{ + *runtime = 0; + return cudaSuccess; +}; +[[gnu::weak, gnu::noinline]] cudaError_t cudaGetDeviceProperties(struct cudaDeviceProp* prop, + int device) +{ + *prop = cudaDeviceProp{}; + return cudaSuccess; +} + +} // namespace stub + +#define RAFT_DECLARE_CUDART(fun) \ + static inline decltype(&stub::fun) fun = \ + cudart.found() ? cudart.sym(#fun) : &stub::fun + +RAFT_DECLARE_CUDART(cuda_memcpy); +RAFT_DECLARE_CUDART(cuda_malloc); +RAFT_DECLARE_CUDART(cuda_memset); +RAFT_DECLARE_CUDART(cuda_free); +RAFT_DECLARE_CUDART(cuda_stream_create); +RAFT_DECLARE_CUDART(cuda_stream_create_with_flags); +RAFT_DECLARE_CUDART(cuda_stream_destroy); +RAFT_DECLARE_CUDART(cuda_device_synchronize); +RAFT_DECLARE_CUDART(cuda_stream_synchronize); +RAFT_DECLARE_CUDART(cuda_event_create); +RAFT_DECLARE_CUDART(cuda_event_record); +RAFT_DECLARE_CUDART(cuda_event_synchronize); +RAFT_DECLARE_CUDART(cuda_event_elapsed_time); +RAFT_DECLARE_CUDART(cuda_event_destroy); +RAFT_DECLARE_CUDART(cuda_get_device); +RAFT_DECLARE_CUDART(cuda_driver_get_version); +RAFT_DECLARE_CUDART(cuda_runtime_get_version); +RAFT_DECLARE_CUDART(cudaGetDeviceProperties); + +#undef RAFT_DECLARE_CUDART +#endif + +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/dataset.hpp b/cpp/bench/ann/src/common/dataset.hpp new file mode 100644 index 0000000000..45cfb5aec4 --- /dev/null +++ b/cpp/bench/ann/src/common/dataset.hpp @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "ann_types.hpp" +#include "blob.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +void generate_bernoulli(CarrierT* data, size_t words, double p) +{ + constexpr size_t kBitsPerCarrierValue = sizeof(CarrierT) * 8; + std::random_device rd; + std::mt19937 gen(rd()); + std::bernoulli_distribution d(p); + for (size_t i = 0; i < words; i++) { + CarrierT word = 0; + for (size_t j = 0; j < kBitsPerCarrierValue; j++) { + word |= CarrierT{d(gen)} << j; + } + data[i] = word; + } +}; + +template +struct dataset { + public: + using bitset_carrier_type = uint32_t; + static inline constexpr size_t kBitsPerCarrierValue = sizeof(bitset_carrier_type) * 8; + + private: + std::string name_; + std::string distance_; + blob base_set_; + blob query_set_; + std::optional> ground_truth_set_; + std::optional> filter_bitset_; + + // Protects the lazy mutations of the blobs accessed by multiple threads + mutable std::mutex mutex_; + // The dim can be read either from the training set or from the query set. + // This cache variable is filled from either of the two sets loaded first. + mutable std::atomic dim_ = -1; + + // Cache the dim value from the passed blob. + inline void cache_dim(const blob& blob) const + { + if (dim_.load(std::memory_order_relaxed) == -1) { + dim_.store(static_cast(blob.n_cols()), std::memory_order_relaxed); + } + } + + public: + dataset(std::string name, + std::string base_file, + uint32_t subset_first_row, + uint32_t subset_size, + std::string query_file, + std::string distance, + std::optional groundtruth_neighbors_file, + std::optional filtering_rate = std::nullopt) + : name_{std::move(name)}, + distance_{std::move(distance)}, + base_set_{base_file, subset_first_row, subset_size}, + query_set_{query_file}, + ground_truth_set_{groundtruth_neighbors_file.has_value() + ? std::make_optional>(groundtruth_neighbors_file.value()) + : std::nullopt} + { + if (filtering_rate.has_value()) { + // Generate a random bitset for filtering + auto n_rows = static_cast(subset_size) + static_cast(subset_first_row); + if (subset_size == 0) { + // Read the base set size as a last resort only - for better laziness + n_rows = base_set_size(); + } + auto bitset_size = (n_rows - 1) / kBitsPerCarrierValue + 1; + blob_file bitset_blob_file{static_cast(bitset_size), 1}; + blob_mmap bitset_blob{ + std::move(bitset_blob_file), false, HugePages::kDisable}; + generate_bernoulli(const_cast(bitset_blob.data()), + bitset_size, + 1.0 - filtering_rate.value()); + filter_bitset_.emplace(std::move(bitset_blob)); + } + } + + [[nodiscard]] auto name() const -> std::string { return name_; } + [[nodiscard]] auto distance() const -> std::string { return distance_; } + [[nodiscard]] auto dim() const -> int + { + auto d = dim_.load(std::memory_order_relaxed); + if (d > -1) { return d; } + std::lock_guard lock(mutex_); + // Otherwise, try reading both (one of the two sets may be missing) + try { + d = static_cast(query_set_.n_cols()); + } catch (const std::runtime_error& e) { + // Any exception raised above will re-raise next time we try to access the query set. + query_set_.reset_lazy_state(); + // If the query set is not accessible, use the base set. + // Don't catch the exception here, because we have nothing else to do anyway. + d = static_cast(base_set_.n_cols()); + } + dim_.store(d, std::memory_order_relaxed); + return d; + } + [[nodiscard]] auto max_k() const -> uint32_t + { + std::lock_guard lock(mutex_); + if (ground_truth_set_.has_value()) { return ground_truth_set_->n_cols(); } + return 0; + } + [[nodiscard]] auto base_set_size() const -> size_t + { + std::lock_guard lock(mutex_); + auto r = base_set_.n_rows(); + cache_dim(base_set_); + return r; + } + [[nodiscard]] auto query_set_size() const -> size_t + { + std::lock_guard lock(mutex_); + auto r = query_set_.n_rows(); + cache_dim(query_set_); + return r; + } + + [[nodiscard]] auto gt_set() const -> const IdxT* + { + std::lock_guard lock(mutex_); + if (ground_truth_set_.has_value()) { return ground_truth_set_->data(); } + return nullptr; + } + + [[nodiscard]] auto query_set() const -> const DataT* + { + std::lock_guard lock(mutex_); + auto* r = query_set_.data(); + cache_dim(query_set_); + return r; + } + [[nodiscard]] auto query_set(MemoryType memory_type, + HugePages request_hugepages_2mb = HugePages::kDisable) const + -> const DataT* + { + std::lock_guard lock(mutex_); + auto* r = query_set_.data(memory_type, request_hugepages_2mb); + cache_dim(query_set_); + return r; + } + + [[nodiscard]] auto base_set() const -> const DataT* + { + std::lock_guard lock(mutex_); + auto* r = base_set_.data(); + cache_dim(base_set_); + return r; + } + [[nodiscard]] auto base_set(MemoryType memory_type, + HugePages request_hugepages_2mb = HugePages::kDisable) const + -> const DataT* + { + std::lock_guard lock(mutex_); + auto* r = base_set_.data(memory_type, request_hugepages_2mb); + cache_dim(base_set_); + return r; + } + + [[nodiscard]] auto filter_bitset() const -> const bitset_carrier_type* + { + std::lock_guard lock(mutex_); + if (filter_bitset_.has_value()) { return filter_bitset_->data(); } + return nullptr; + } + + [[nodiscard]] auto filter_bitset(MemoryType memory_type, + HugePages request_hugepages_2mb = HugePages::kDisable) const + -> const bitset_carrier_type* + { + std::lock_guard lock(mutex_); + if (filter_bitset_.has_value()) { + return filter_bitset_->data(memory_type, request_hugepages_2mb); + } + return nullptr; + } +}; + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/common/thread_pool.hpp b/cpp/bench/ann/src/common/thread_pool.hpp new file mode 100644 index 0000000000..287ac0a2dc --- /dev/null +++ b/cpp/bench/ann/src/common/thread_pool.hpp @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +class fixed_thread_pool { + public: + explicit fixed_thread_pool(int num_threads) + { + if (num_threads < 1) { + throw std::runtime_error("num_threads must >= 1"); + } else if (num_threads == 1) { + return; + } + + tasks_ = new task[num_threads]; + + threads_.reserve(num_threads); + for (int i = 0; i < num_threads; ++i) { + threads_.emplace_back([&, i] { + auto& task = tasks_[i]; + while (true) { + std::unique_lock lock(task.mtx); + task.cv.wait(lock, + [&] { return task.has_task || finished_.load(std::memory_order_relaxed); }); + if (finished_.load(std::memory_order_relaxed)) { break; } + + task.task(); + task.has_task = false; + } + }); + } + } + + ~fixed_thread_pool() + { + if (threads_.empty()) { return; } + + finished_.store(true, std::memory_order_relaxed); + for (unsigned i = 0; i < threads_.size(); ++i) { + // NB: don't lock the task mutex here, may deadlock on .join() otherwise + auto& task = tasks_[i]; + task.cv.notify_one(); + threads_[i].join(); + } + + delete[] tasks_; + } + + template + void submit(Func f, IdxT len) + { + // Run functions in main thread if thread pool has no threads + if (threads_.empty()) { + for (IdxT i = 0; i < len; ++i) { + f(i); + } + return; + } + + const int num_threads = threads_.size(); + // one extra part for competition among threads + const IdxT items_per_thread = len / (num_threads + 1); + std::atomic cnt(items_per_thread * num_threads); + + // Wrap function + auto wrapped_f = [&](IdxT start, IdxT end) { + for (IdxT i = start; i < end; ++i) { + f(i); + } + + while (true) { + IdxT i = cnt.fetch_add(1, std::memory_order_relaxed); + if (i >= len) { break; } + f(i); + } + }; + + std::vector> futures; + futures.reserve(num_threads); + for (int i = 0; i < num_threads; ++i) { + IdxT start = i * items_per_thread; + auto& task = tasks_[i]; + { + [[maybe_unused]] std::lock_guard lock(task.mtx); + task.task = std::packaged_task([=] { wrapped_f(start, start + items_per_thread); }); + futures.push_back(task.task.get_future()); + task.has_task = true; + } + task.cv.notify_one(); + } + + for (auto& fut : futures) { + fut.wait(); + } + return; + } + + private: + struct alignas(64) task { + std::mutex mtx; + std::condition_variable cv; + bool has_task = false; + std::packaged_task task; + }; + + task* tasks_; + std::vector threads_; + std::atomic finished_{false}; +}; diff --git a/cpp/bench/ann/src/common/util.hpp b/cpp/bench/ann/src/common/util.hpp new file mode 100644 index 0000000000..11361c88fe --- /dev/null +++ b/cpp/bench/ann/src/common/util.hpp @@ -0,0 +1,715 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "ann_types.hpp" +#include "cuda_stub.hpp" // cuda-related utils + +#if __has_include() +#define ANN_BENCH_NVTX3_HEADERS_FOUND +#include +#endif + +#include // sched_getaffinity +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +/** + * Current thread id as given by the benchmark State. + * It's populated on every call of a benchmark case. + * It's relevant in the 'throughput' mode of the search benchmarks, + * where some algorithms might want to coordinate allocation of the resources. + */ +inline thread_local int benchmark_thread_id = 0; +/** + * Total concurrent thread count as given by the benchmark State. + * It's populated on every call of a benchmark case. + * It's relevant in the 'throughput' mode of the search benchmarks, + * where some algorithms might want to coordinate allocation of the resources. + */ +inline thread_local int benchmark_n_threads = 1; + +struct cuda_timer { + private: + std::optional stream_; + cudaEvent_t start_{nullptr}; + cudaEvent_t stop_{nullptr}; + double total_time_{0}; + + template + static inline auto extract_stream(AnnT* algo) -> std::optional + { + auto gpu_ann = dynamic_cast(algo); + if (gpu_ann != nullptr && gpu_ann->uses_stream()) { + return std::make_optional(gpu_ann->get_sync_stream()); + } + return std::nullopt; + } + + public: + struct cuda_lap { + private: + cudaStream_t stream_; + cudaEvent_t start_; + cudaEvent_t stop_; + double& total_time_; + + public: + cuda_lap(cudaStream_t stream, cudaEvent_t start, cudaEvent_t stop, double& total_time) + : start_(start), stop_(stop), stream_(stream), total_time_(total_time) + { +#ifndef BUILD_CPU_ONLY + cudaEventRecord(start_, stream_); +#endif + } + cuda_lap() = delete; + + ~cuda_lap() noexcept + { +#ifndef BUILD_CPU_ONLY + cudaEventRecord(stop_, stream_); + cudaEventSynchronize(stop_); + float milliseconds = 0.0f; + cudaEventElapsedTime(&milliseconds, start_, stop_); + total_time_ += milliseconds / 1000.0; +#endif + } + }; + + explicit cuda_timer(std::optional stream) : stream_{stream} + { +#ifndef BUILD_CPU_ONLY + if (stream_.has_value()) { + cudaEventCreate(&stop_); + cudaEventCreate(&start_); + } +#endif + } + + template + explicit cuda_timer(const std::unique_ptr& algo) : cuda_timer{extract_stream(algo.get())} + { + } + + ~cuda_timer() noexcept + { +#ifndef BUILD_CPU_ONLY + if (stream_.has_value()) { + cudaStreamSynchronize(stream_.value()); + cudaEventDestroy(start_); + cudaEventDestroy(stop_); + } +#endif + } + + cuda_timer() = delete; + cuda_timer(cuda_timer const&) = delete; + cuda_timer(cuda_timer&&) = delete; + auto operator=(cuda_timer const&) -> cuda_timer& = delete; + auto operator=(cuda_timer&&) -> cuda_timer& = delete; + + [[nodiscard]] auto stream() const -> std::optional { return stream_; } + + [[nodiscard]] auto active() const -> bool { return stream_.has_value(); } + + [[nodiscard]] auto total_time() const -> double { return total_time_; } + + [[nodiscard]] auto lap(bool enabled = true) -> std::optional + { + return enabled && stream_.has_value() + ? std::make_optional(stream_.value(), start_, stop_, total_time_) + : std::nullopt; + } +}; + +#ifndef BUILD_CPU_ONLY +// ATM, rmm::stream does not support passing in flags; hence this helper type. +struct non_blocking_stream { + non_blocking_stream() { cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking); } + ~non_blocking_stream() noexcept + { + if (stream_ != nullptr) { cudaStreamDestroy(stream_); } + } + non_blocking_stream(non_blocking_stream const&) = delete; + non_blocking_stream(non_blocking_stream&& other) noexcept { std::swap(stream_, other.stream_); } + auto operator=(non_blocking_stream const&) -> non_blocking_stream& = delete; + auto operator=(non_blocking_stream&&) -> non_blocking_stream& = delete; + [[nodiscard]] auto view() const noexcept -> cudaStream_t { return stream_; } + + private: + cudaStream_t stream_{nullptr}; +}; + +namespace detail { +inline std::vector global_stream_pool(0); +inline std::mutex gsp_mutex; +} // namespace detail +#endif + +/** + * Get a stream associated with the current benchmark thread. + * + * Note, the streams are reused between the benchmark cases. + * This makes it easier to profile and analyse multiple benchmark cases in one timeline using tools + * like nsys. + */ +inline auto get_stream_from_global_pool() -> cudaStream_t +{ +#ifndef BUILD_CPU_ONLY + std::lock_guard guard(detail::gsp_mutex); + if (static_cast(detail::global_stream_pool.size()) < benchmark_n_threads) { + detail::global_stream_pool.resize(benchmark_n_threads); + } + return detail::global_stream_pool[benchmark_thread_id].view(); +#else + return nullptr; +#endif +} + +/** The workspace buffer for use thread-locally. */ +struct ws_buffer { + explicit ws_buffer(size_t size, cudaStream_t stream) : size_{size}, stream_{stream} {} + ws_buffer() = delete; + ws_buffer(ws_buffer&&) = delete; + auto operator=(ws_buffer&&) -> ws_buffer& = delete; + ws_buffer(const ws_buffer&) = delete; + auto operator=(const ws_buffer&) -> ws_buffer& = delete; + ~ws_buffer() noexcept + { +#ifndef BUILD_CPU_ONLY + if (data_device_ != nullptr) { + cudaFreeAsync(data_device_, stream_); + cudaStreamSynchronize(stream_); + } + if (data_host_ != nullptr) { cudaFreeHost(data_host_); } +#else + if (data_host_ != nullptr) { free(data_host_); } +#endif + } + + [[nodiscard]] auto size() const noexcept { return size_; } + [[nodiscard]] auto data(MemoryType loc) const noexcept -> void* + { + if (size_ == 0) { return nullptr; } + switch (loc) { +#ifndef BUILD_CPU_ONLY + case MemoryType::kDevice: { + if (data_device_ == nullptr) { + cudaMallocAsync(&data_device_, size_, stream_); + cudaStreamSynchronize(stream_); + needs_cleanup_device_ = false; + } else if (needs_cleanup_device_) { + cudaMemsetAsync(data_device_, 0, size_, stream_); + cudaStreamSynchronize(stream_); + needs_cleanup_device_ = false; + } + return data_device_; + } +#endif + default: { + if (data_host_ == nullptr) { +#ifndef BUILD_CPU_ONLY + cudaMallocHost(&data_host_, size_); +#else + data_host_ = malloc(size_); +#endif + needs_cleanup_host_ = false; + } else if (needs_cleanup_host_) { + memset(data_host_, 0, size_); + needs_cleanup_host_ = false; + } + return data_host_; + } + } + } + + void transfer_data(MemoryType dst, MemoryType src) + { + // The destination is overwritten and thus does not need cleanup + if (dst == MemoryType::kDevice) { + needs_cleanup_device_ = false; + } else { + needs_cleanup_host_ = false; + } + auto dst_ptr = data(dst); + auto src_ptr = data(src); + if (dst_ptr == src_ptr) { return; } +#ifndef BUILD_CPU_ONLY + cudaMemcpyAsync(dst_ptr, src_ptr, size_, cudaMemcpyDefault, stream_); + cudaStreamSynchronize(stream_); +#endif + } + + /** Mark the buffer for reuse - it needs to be cleared to make sure the previous results are not + * leaked to the new iteration. */ + void reuse() + { + needs_cleanup_host_ = true; + needs_cleanup_device_ = true; + } + + private: + size_t size_{0}; + cudaStream_t stream_ = nullptr; + mutable void* data_host_ = nullptr; + mutable void* data_device_ = nullptr; + mutable bool needs_cleanup_host_ = false; + mutable bool needs_cleanup_device_ = false; +}; + +namespace detail { +inline std::vector> global_result_buffer_pool(0); +inline std::mutex grp_mutex; +} // namespace detail + +/** + * Get a result buffer associated with the current benchmark thread. + * + * Note, the allocations are reused between the benchmark cases. + * This reduces the setup overhead and number of times the context is being blocked + * (this is relevant if there is a persistent kernel running across multiples benchmark cases). + */ +inline auto get_result_buffer_from_global_pool(size_t size) -> ws_buffer& +{ + auto stream = get_stream_from_global_pool(); + auto& rb = [stream, size]() -> ws_buffer& { + std::lock_guard guard(detail::grp_mutex); + if (static_cast(detail::global_result_buffer_pool.size()) < benchmark_n_threads) { + detail::global_result_buffer_pool.resize(benchmark_n_threads); + } + auto& rb = detail::global_result_buffer_pool[benchmark_thread_id]; + if (!rb || rb->size() < size) { + rb = std::make_unique(size, stream); + } else { + rb->reuse(); + } + return *rb; + }(); + return rb; +} + +namespace detail { +inline std::vector> global_tmp_buffer_pool(0); +inline std::mutex gtp_mutex; +} // namespace detail + +/** + * Global temporary buffer pool for use by algorithms. + * In contrast to `get_result_buffer_from_global_pool`, the content of these buffers is never + * initialized. + */ +inline auto get_tmp_buffer_from_global_pool(size_t size) -> ws_buffer& +{ + auto stream = get_stream_from_global_pool(); + auto& rb = [stream, size]() -> ws_buffer& { + std::lock_guard guard(detail::gtp_mutex); + if (static_cast(detail::global_tmp_buffer_pool.size()) < benchmark_n_threads) { + detail::global_tmp_buffer_pool.resize(benchmark_n_threads); + } + auto& rb = detail::global_tmp_buffer_pool[benchmark_thread_id]; + if (!rb || rb->size() < size) { rb = std::make_unique(size, stream); } + return *rb; + }(); + return rb; +} + +/** + * Delete all streams and memory allocations in the global pool. + * It's called at the end of the `main` function - before global/static variables and cuda context + * is destroyed - to make sure they are destroyed gracefully and correctly seen by analysis tools + * such as nsys. + */ +inline void reset_global_device_resources() +{ +#ifndef BUILD_CPU_ONLY + std::lock_guard guard(detail::gsp_mutex); + detail::global_tmp_buffer_pool.resize(0); + detail::global_result_buffer_pool.resize(0); + detail::global_stream_pool.resize(0); +#endif +} + +inline auto host_info() +{ + std::vector> props; + + // Memory info + auto page_size = sysconf(_SC_PAGE_SIZE); + props.emplace_back("host_pagesize", std::to_string(page_size)); + + struct sysinfo sys_info; + if (sysinfo(&sys_info) != -1) { + props.emplace_back("host_total_ram_size", + std::to_string(size_t(sys_info.totalram) * size_t(sys_info.mem_unit))); + props.emplace_back("host_total_swap_size", + std::to_string(size_t(sys_info.totalswap) * size_t(sys_info.mem_unit))); + } + + // CPU info + int host_processors_configured = sysconf(_SC_NPROCESSORS_CONF); + props.emplace_back("host_processors_sysconf", std::to_string(host_processors_configured)); + std::vector affinity_mask_buf(CPU_ALLOC_SIZE(host_processors_configured)); + cpu_set_t* affinity_mask = reinterpret_cast(affinity_mask_buf.data()); + sched_getaffinity(0, affinity_mask_buf.size(), affinity_mask); + uint64_t cpu_freq_min = 0; + uint64_t cpu_freq_max = 0; + int host_processors_used = 0; + int host_cores_used = 0; + std::set> host_cores_selected{}; // pairs of (socket_id, core_id) + for (int cpu_id = 0; cpu_id < host_processors_configured; cpu_id++) { + if (CPU_ISSET_S(cpu_id, affinity_mask_buf.size(), affinity_mask) == 0) { continue; } + host_processors_used++; + std::string cpu_fpath = "/sys/devices/system/cpu/cpu" + std::to_string(cpu_id); + if (!std::filesystem::exists(cpu_fpath)) { continue; } + + int this_cpu_core = 0; + int this_cpu_package = 0; + uint64_t this_cpu_freq_min = 0; + uint64_t this_cpu_freq_max = 0; + std::ifstream(cpu_fpath + "/topology/core_id") >> this_cpu_core; + std::ifstream(cpu_fpath + "/topology/physical_package_id") >> this_cpu_package; + std::ifstream(cpu_fpath + "/cpufreq/scaling_min_freq") >> this_cpu_freq_min; + std::ifstream(cpu_fpath + "/cpufreq/scaling_max_freq") >> this_cpu_freq_max; + host_cores_selected.insert(std::make_pair(this_cpu_package, this_cpu_core)); + cpu_freq_min = cpu_freq_min == 0 + ? (this_cpu_freq_min * 1000ull) + : std::min(this_cpu_freq_min * 1000ull, cpu_freq_min); + cpu_freq_max = std::max(this_cpu_freq_max * 1000ull, cpu_freq_max); + } + host_cores_used = host_cores_selected.size(); + if (host_processors_used != 0) { + props.emplace_back("host_processors_used", std::to_string(host_processors_used)); + } + if (host_cores_used != 0) { + props.emplace_back("host_cores_used", std::to_string(host_cores_used)); + } + if (cpu_freq_min != 0) { props.emplace_back("host_cpu_freq_min", std::to_string(cpu_freq_min)); } + if (cpu_freq_max != 0) { props.emplace_back("host_cpu_freq_max", std::to_string(cpu_freq_max)); } + + return props; +} + +inline auto cuda_info() +{ + std::vector> props; +#ifndef BUILD_CPU_ONLY + int dev, driver = 0, runtime = 0; + cudaDriverGetVersion(&driver); + cudaRuntimeGetVersion(&runtime); + + cudaDeviceProp device_prop; + auto err_code = cudaGetDevice(&dev); + if (err_code == cudaErrorNoDevice || err_code == cudaErrorInvalidDevice) { + throw std::runtime_error{"cuda_info: call to cudaGetDevice failed with code " + + std::to_string(err_code) + + ". If you are running on a CPU-only machine, please " + "use the CPU package instead."}; + } else if (err_code != cudaSuccess) { + throw std::runtime_error{"cuda_info: call to cudaGetDevice failed with code " + + std::to_string(err_code)}; + } + err_code = cudaGetDeviceProperties(&device_prop, dev); + if (err_code != cudaSuccess) { + throw std::runtime_error{"cuda_info: call to cudaGetDeviceProperties failed with code " + + std::to_string(err_code)}; + } + int clockRate = 0; + int memoryClockRate = 0; + err_code = cudaDeviceGetAttribute(&clockRate, cudaDevAttrClockRate, dev); + err_code = cudaDeviceGetAttribute(&memoryClockRate, cudaDevAttrMemoryClockRate, dev); + + props.emplace_back("gpu_name", std::string(device_prop.name)); + props.emplace_back("gpu_sm_count", std::to_string(device_prop.multiProcessorCount)); + props.emplace_back("gpu_sm_freq", std::to_string(clockRate * 1e3)); + props.emplace_back("gpu_mem_freq", std::to_string(memoryClockRate * 1e3)); + props.emplace_back("gpu_mem_bus_width", std::to_string(device_prop.memoryBusWidth)); + props.emplace_back("gpu_mem_global_size", std::to_string(device_prop.totalGlobalMem)); + props.emplace_back("gpu_mem_shared_size", std::to_string(device_prop.sharedMemPerMultiprocessor)); + props.emplace_back("gpu_driver_version", + std::to_string(driver / 1000) + "." + std::to_string((driver % 100) / 10)); + props.emplace_back("gpu_runtime_version", + std::to_string(runtime / 1000) + "." + std::to_string((runtime % 100) / 10)); + props.emplace_back("gpu_hostNativeAtomicSupported", + std::to_string(device_prop.hostNativeAtomicSupported)); + props.emplace_back("gpu_pageableMemoryAccess", std::to_string(device_prop.pageableMemoryAccess)); + props.emplace_back("gpu_pageableMemoryAccessUsesHostPageTables", + std::to_string(device_prop.pageableMemoryAccessUsesHostPageTables)); + props.emplace_back("gpu_gpuDirectRDMASupported", + std::to_string(device_prop.gpuDirectRDMASupported)); +#endif + return props; +} + +struct nvtx_case { +#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND + private: + std::string case_name_; + std::array iter_name_{0}; + nvtxDomainHandle_t domain_; + int64_t iteration_ = 0; + nvtxEventAttributes_t case_attrib_{0}; + nvtxEventAttributes_t iter_attrib_{0}; +#endif + + public: + struct nvtx_lap { +#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND + private: + nvtxDomainHandle_t domain_; + + public: + nvtx_lap(nvtxDomainHandle_t domain, nvtxEventAttributes_t* attr) : domain_(domain) + { + nvtxDomainRangePushEx(domain_, attr); + } + nvtx_lap() = delete; + ~nvtx_lap() noexcept { nvtxDomainRangePop(domain_); } +#endif + }; + +#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND + explicit nvtx_case(std::string case_name) + : case_name_(std::move(case_name)), domain_(nvtxDomainCreateA("algo benchmark")) + { + case_attrib_.version = NVTX_VERSION; + iter_attrib_.version = NVTX_VERSION; + case_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + iter_attrib_.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; + case_attrib_.colorType = NVTX_COLOR_ARGB; + iter_attrib_.colorType = NVTX_COLOR_ARGB; + case_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; + iter_attrib_.messageType = NVTX_MESSAGE_TYPE_ASCII; + case_attrib_.message.ascii = case_name_.c_str(); + auto c = std::hash{}(case_name_); + case_attrib_.color = c | 0xA0A0A0; + nvtxDomainRangePushEx(domain_, &case_attrib_); + } + + ~nvtx_case() + { + nvtxDomainRangePop(domain_); + nvtxDomainDestroy(domain_); + } +#else + explicit nvtx_case(std::string) {} +#endif + + [[nodiscard]] auto lap() -> nvtx_case::nvtx_lap + { +#ifdef ANN_BENCH_NVTX3_HEADERS_FOUND + auto i = iteration_++; + uint32_t c = (i % 5); + uint32_t r = 150 + c * 20; + uint32_t g = 200 + c * 10; + uint32_t b = 220 + c * 5; + std::snprintf(iter_name_.data(), iter_name_.size(), "Lap %zd", i); + iter_attrib_.message.ascii = iter_name_.data(); + iter_attrib_.color = (r << 16) + (g << 8) + b; + return nvtx_lap{domain_, &iter_attrib_}; +#else + return nvtx_lap{}; +#endif + } +}; + +/** + * A progress tracker that allows syncing threads multiple times and resets the global + * progress once the threads are done. + */ +struct progress_barrier { + progress_barrier() = default; + ~progress_barrier() noexcept + { + { + // Lock makes sure the notified threads see the updates to `done_`. + std::unique_lock lk(mutex_); + done_.store(true, std::memory_order_relaxed); + cv_.notify_all(); + } + // This is the only place where the order of the updates to thread_progress_ and done_ is + // important. They are not guarded by the mutex, and `done_` must not be reset to `true` by + // other threads after the `total_progress_` is zero. + // Hence the default memory order (std::memory_order_seq_cst). + auto rem = total_progress_.fetch_sub(thread_progress_); + if (rem == thread_progress_) { + // the last thread to exit clears the progress state. + done_.store(false); + } + } + + /** + * Advance the progress counter by `n` and return the previous `progress` value. + * + * This can be used to track which thread arrives on the call site first. + * + * @return the previous progress counter value (before incrementing it by `n`). + */ + auto arrive(int n) + { + thread_progress_ += n; + // Lock makes sure the notified threads see the updates to `total_progress_`. + std::unique_lock lk(mutex_); + auto prev = total_progress_.fetch_add(n, std::memory_order_relaxed); + cv_.notify_all(); + return prev; + } + + /** + * Wait till the progress counter reaches `n` or finishes abnormally. + * + * @return the latest observed value of the progress counter. + */ + auto wait(int limit) + { + int cur = total_progress_.load(std::memory_order_relaxed); + if (cur >= limit) { return cur; } + auto done = done_.load(std::memory_order_relaxed); + if (done) { return cur; } + std::unique_lock lk(mutex_); + while (cur < limit && !done) { + using namespace std::chrono_literals; + cv_.wait_for(lk, 10ms); + cur = total_progress_.load(std::memory_order_relaxed); + done = done_.load(std::memory_order_relaxed); + } + return cur; + } + + private: + static inline std::atomic total_progress_; + static inline std::atomic done_; + static inline std::mutex mutex_; + static inline std::condition_variable cv_; + int thread_progress_{0}; +}; + +inline auto split(const std::string& s, char delimiter) -> std::vector +{ + std::vector tokens; + std::string token; + std::istringstream iss(s); + while (getline(iss, token, delimiter)) { + if (!token.empty()) { tokens.push_back(token); } + } + return tokens; +} + +inline auto file_exists(const std::string& filename) -> bool +{ + struct stat statbuf; + if (stat(filename.c_str(), &statbuf) != 0) { return false; } + return S_ISREG(statbuf.st_mode); +} + +inline auto dir_exists(const std::string& dir) -> bool +{ + struct stat statbuf; + if (stat(dir.c_str(), &statbuf) != 0) { return false; } + return S_ISDIR(statbuf.st_mode); +} + +inline auto create_dir(const std::string& dir) -> bool +{ + const auto path = split(dir, '/'); + + std::string cwd; + if (!dir.empty() && dir[0] == '/') { cwd += '/'; } + + for (const auto& p : path) { + cwd += p + "/"; + if (!dir_exists(cwd)) { + int ret = mkdir(cwd.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + if (ret != 0) { return false; } + } + } + return true; +} + +inline void make_sure_parent_dir_exists(const std::string& file_path) +{ + const auto pos = file_path.rfind('/'); + if (pos != std::string::npos) { + auto dir = file_path.substr(0, pos); + if (!dir_exists(dir)) { create_dir(dir); } + } +} + +inline auto combine_path(const std::string& dir, const std::string& path) +{ + std::filesystem::path p_dir(dir); + std::filesystem::path p_suf(path); + return (p_dir / p_suf).string(); +} + +template +void log_with_level(const char* level, const Ts&... vs) +{ + char buf[20]; + auto now = std::chrono::system_clock::now(); + auto now_tt = std::chrono::system_clock::to_time_t(now); + size_t millis = + std::chrono::duration_cast(now.time_since_epoch()).count() % + 1000000ULL; + std::strftime(buf, sizeof(buf), "%H:%M:%S", std::localtime(&now_tt)); + printf("[%s] [%s.%06zu] ", level, buf, millis); + if constexpr (sizeof...(Ts) == 1) { + printf("%s", vs...); + } else { + printf(vs...); + } + printf("\n"); + fflush(stdout); +} + +template +void log_info(Ts&&... vs) +{ + log_with_level("I", std::forward(vs)...); +} + +template +void log_warn(Ts&&... vs) +{ + log_with_level("W", std::forward(vs)...); +} + +template +void log_error(Ts&&... vs) +{ + log_with_level("E", std::forward(vs)...); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h new file mode 100644 index 0000000000..3ad01dc2b4 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#undef WARP_SIZE +#ifdef CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE +#include "cuvs_wrapper.h" +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT +#include "cuvs_ivf_flat_wrapper.h" +extern template class cuvs::bench::cuvs_ivf_flat; +extern template class cuvs::bench::cuvs_ivf_flat; +extern template class cuvs::bench::cuvs_ivf_flat; +#endif +#if defined(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA) || \ + defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN) +#include "cuvs_ivf_pq_wrapper.h" +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_PQ +extern template class cuvs::bench::cuvs_ivf_pq; +extern template class cuvs::bench::cuvs_ivf_pq; +extern template class cuvs::bench::cuvs_ivf_pq; +#endif +#if defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) || \ + defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN) +#include "cuvs_cagra_wrapper.h" +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_CAGRA +extern template class cuvs::bench::cuvs_cagra; +extern template class cuvs::bench::cuvs_cagra; +extern template class cuvs::bench::cuvs_cagra; +extern template class cuvs::bench::cuvs_cagra; +#endif + +#ifdef CUVS_ANN_BENCH_USE_CUVS_MG +#include "cuvs_ivf_flat_wrapper.h" +#include "cuvs_mg_ivf_flat_wrapper.h" + +#include "cuvs_ivf_pq_wrapper.h" +#include "cuvs_mg_ivf_pq_wrapper.h" + +#include "cuvs_cagra_wrapper.h" +#include "cuvs_mg_cagra_wrapper.h" +#endif + +template +void parse_dynamic_batching_params(const nlohmann::json& conf, ParamT& param) +{ + if (!conf.value("dynamic_batching", false)) { return; } + param.dynamic_batching = true; + if (conf.contains("dynamic_batching_max_batch_size")) { + param.dynamic_batching_max_batch_size = conf.at("dynamic_batching_max_batch_size"); + } + param.dynamic_batching_conservative_dispatch = + conf.value("dynamic_batching_conservative_dispatch", false); + if (conf.contains("dynamic_batching_dispatch_timeout_ms")) { + param.dynamic_batching_dispatch_timeout_ms = conf.at("dynamic_batching_dispatch_timeout_ms"); + } + if (conf.contains("dynamic_batching_n_queues")) { + param.dynamic_batching_n_queues = conf.at("dynamic_batching_n_queues"); + } + param.dynamic_batching_k = + uint32_t(uint32_t(conf.at("k")) * float(conf.value("refine_ratio", 1.0f))); +} + +#if defined(CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT) || defined(CUVS_ANN_BENCH_USE_CUVS_MG) +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_ivf_flat::build_param& param) +{ + param.n_lists = conf.at("nlist"); + if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } + if (conf.contains("ratio")) { + param.kmeans_trainset_fraction = 1.0 / static_cast(conf.at("ratio")); + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_ivf_flat::search_param& param) +{ + param.ivf_flat_params.n_probes = conf.at("nprobe"); +} +#endif + +#if defined(CUVS_ANN_BENCH_USE_CUVS_IVF_PQ) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA) || \ + defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) || defined(CUVS_ANN_BENCH_USE_CUVS_MG) || \ + defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN) +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_ivf_pq::build_param& param) +{ + if (conf.contains("nlist")) { param.n_lists = conf.at("nlist"); } + if (conf.contains("niter")) { param.kmeans_n_iters = conf.at("niter"); } + if (conf.contains("ratio")) { + param.kmeans_trainset_fraction = 1.0 / static_cast(conf.at("ratio")); + } + if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } + if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } + if (conf.contains("codebook_kind")) { + std::string kind = conf.at("codebook_kind"); + if (kind == "cluster") { + param.codebook_kind = cuvs::neighbors::ivf_pq::codebook_gen::PER_CLUSTER; + } else if (kind == "subspace") { + param.codebook_kind = cuvs::neighbors::ivf_pq::codebook_gen::PER_SUBSPACE; + } else { + throw std::runtime_error("codebook_kind: '" + kind + + "', should be either 'cluster' or 'subspace'"); + } + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_ivf_pq::search_param& param) +{ + if (conf.contains("nprobe")) { param.pq_param.n_probes = conf.at("nprobe"); } + if (conf.contains("internalDistanceDtype")) { + std::string type = conf.at("internalDistanceDtype"); + if (type == "float") { + param.pq_param.internal_distance_dtype = CUDA_R_32F; + } else if (type == "half") { + param.pq_param.internal_distance_dtype = CUDA_R_16F; + } else { + throw std::runtime_error("internalDistanceDtype: '" + type + + "', should be either 'float' or 'half'"); + } + } else { + // set half as default type + param.pq_param.internal_distance_dtype = CUDA_R_16F; + } + + if (conf.contains("smemLutDtype")) { + std::string type = conf.at("smemLutDtype"); + if (type == "float") { + param.pq_param.lut_dtype = CUDA_R_32F; + } else if (type == "half") { + param.pq_param.lut_dtype = CUDA_R_16F; + } else if (type == "fp8") { + param.pq_param.lut_dtype = CUDA_R_8U; + } else { + throw std::runtime_error("smemLutDtype: '" + type + + "', should be either 'float', 'half' or 'fp8'"); + } + } else { + // set half as default + param.pq_param.lut_dtype = CUDA_R_16F; + } + + if (conf.contains("coarse_search_dtype")) { + std::string type = conf.at("coarse_search_dtype"); + if (type == "float") { + param.pq_param.coarse_search_dtype = CUDA_R_32F; + } else if (type == "half") { + param.pq_param.coarse_search_dtype = CUDA_R_16F; + } else if (type == "int8") { + param.pq_param.coarse_search_dtype = CUDA_R_8I; + } else { + throw std::runtime_error("coarse_search_dtype: '" + type + + "', should be either 'float', 'half' or 'int8'"); + } + } + + if (conf.contains("max_internal_batch_size")) { + param.pq_param.max_internal_batch_size = conf.at("max_internal_batch_size"); + } + + if (conf.contains("refine_ratio")) { + param.refine_ratio = conf.at("refine_ratio"); + if (param.refine_ratio < 1.0f) { throw std::runtime_error("refine_ratio should be >= 1.0"); } + } + + // enable dynamic batching + parse_dynamic_batching_params(conf, param); +} +#endif + +#if defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_HNSWLIB) || \ + defined(CUVS_ANN_BENCH_USE_CUVS_MG) || defined(CUVS_ANN_BENCH_USE_CUVS_CAGRA_DISKANN) +template +void parse_build_param(const nlohmann::json& conf, cuvs::neighbors::nn_descent::index_params& param) +{ + if (conf.contains("graph_degree")) { param.graph_degree = conf.at("graph_degree"); } + if (conf.contains("intermediate_graph_degree")) { + param.intermediate_graph_degree = conf.at("intermediate_graph_degree"); + } + // we allow niter shorthand for max_iterations + if (conf.contains("niter")) { param.max_iterations = conf.at("niter"); } + if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } + if (conf.contains("termination_threshold")) { + param.termination_threshold = conf.at("termination_threshold"); + } +} + +inline void parse_build_param(const nlohmann::json& conf, cuvs::neighbors::vpq_params& param) +{ + if (conf.contains("pq_bits")) { param.pq_bits = conf.at("pq_bits"); } + if (conf.contains("pq_dim")) { param.pq_dim = conf.at("pq_dim"); } + if (conf.contains("vq_n_centers")) { param.vq_n_centers = conf.at("vq_n_centers"); } + if (conf.contains("kmeans_n_iters")) { param.kmeans_n_iters = conf.at("kmeans_n_iters"); } + if (conf.contains("vq_kmeans_trainset_fraction")) { + param.vq_kmeans_trainset_fraction = conf.at("vq_kmeans_trainset_fraction"); + } + if (conf.contains("pq_kmeans_trainset_fraction")) { + param.pq_kmeans_trainset_fraction = conf.at("pq_kmeans_trainset_fraction"); + } +} + +nlohmann::json collect_conf_with_prefix(const nlohmann::json& conf, + const std::string& prefix, + bool remove_prefix = true) +{ + nlohmann::json out; + for (auto& i : conf.items()) { + if (i.key().compare(0, prefix.size(), prefix) == 0) { + auto new_key = remove_prefix ? i.key().substr(prefix.size()) : i.key(); + out[new_key] = i.value(); + } + } + return out; +} + +template +void parse_build_param(const nlohmann::json& conf, cuvs::neighbors::cagra::index_params& params) +{ + // NB: try to avoid setting parameters implicitly, because this parsing may be used after + // parameter heuristics are applied (e.g. in cuvs_cagra_hnswlib.cu). + if (conf.contains("graph_degree")) { params.graph_degree = conf.at("graph_degree"); } + if (conf.contains("intermediate_graph_degree")) { + params.intermediate_graph_degree = conf.at("intermediate_graph_degree"); + } else { + // Only update the intermediate graph degree if it's invalid. + params.intermediate_graph_degree = + std::max(params.graph_degree, params.intermediate_graph_degree); + } + + nlohmann::json comp_search_conf = collect_conf_with_prefix(conf, "compression_"); + if (!comp_search_conf.empty()) { + auto vpq_pams = params.compression.value_or(cuvs::neighbors::vpq_params{}); + parse_build_param(comp_search_conf, vpq_pams); + params.compression.emplace(vpq_pams); + } + + if (conf.contains("guarantee_connectivity")) { + params.guarantee_connectivity = conf.at("guarantee_connectivity"); + } + + // Override the graph_build_algo if requested explicitly + if (conf.contains("graph_build_algo")) { + if (conf.at("graph_build_algo") == "IVF_PQ") { + if (!std::holds_alternative( + params.graph_build_params)) { + params.graph_build_params = cuvs::neighbors::graph_build_params::ivf_pq_params{}; + } + } else if (conf.at("graph_build_algo") == "NN_DESCENT") { + if (!std::holds_alternative( + params.graph_build_params)) { + params.graph_build_params = cuvs::neighbors::graph_build_params::nn_descent_params{}; + } + } + } + + // Parse build-algo-specific parameters and use them to decide on the algo type + nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "ivf_pq_build_"); + nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "ivf_pq_search_"); + nlohmann::json nn_descent_conf = collect_conf_with_prefix(conf, "nn_descent_"); + + if (std::holds_alternative(params.graph_build_params)) { + if (!ivf_pq_build_conf.empty() || !ivf_pq_search_conf.empty()) { + params.graph_build_params = cuvs::neighbors::graph_build_params::ivf_pq_params{}; + } else if (!nn_descent_conf.empty()) { + params.graph_build_params = cuvs::neighbors::graph_build_params::nn_descent_params{}; + } else { + params.graph_build_params = cuvs::neighbors::graph_build_params::iterative_search_params{}; + } + } + + // Apply build-algo-specific parameters + std::visit( + [&](auto& arg) { + using U = std::decay_t; + if constexpr (std::is_same_v) { + parse_build_param(ivf_pq_build_conf, arg.build_params); + typename cuvs::bench::cuvs_ivf_pq::search_param sparam; + sparam.pq_param = arg.search_params; + sparam.refine_ratio = arg.refinement_rate; + parse_search_param(ivf_pq_search_conf, sparam); + arg.search_params = sparam.pq_param; + arg.refinement_rate = sparam.refine_ratio; + } else if constexpr (std::is_same_v) { + parse_build_param(nn_descent_conf, arg); + } + }, + params.graph_build_params); +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_cagra::build_param& param) +{ + if (conf.contains("num_dataset_splits")) { + param.num_dataset_splits = conf.at("num_dataset_splits"); + } + if (conf.contains("merge_type")) { + std::string mt = conf.at("merge_type"); + if (mt == "PHYSICAL") { + param.merge_type = cuvs::bench::CagraMergeType::kPhysical; + } else if (mt == "LOGICAL") { + param.merge_type = cuvs::bench::CagraMergeType::kLogical; + } else { + throw std::runtime_error("invalid value for merge_type"); + } + } + param.cagra_params = [conf](raft::matrix_extent extents, + cuvs::distance::DistanceType dist_type) { + // Delayed parsing/initialization of cagra_params - it's called once the dataset shape is known + auto cagra_params = cuvs::neighbors::cagra::index_params{}; + cagra_params.metric = dist_type; + if (conf.value("graph_build_algo", "") == "IVF_PQ") { + cagra_params.graph_build_params = + cuvs::neighbors::cagra::graph_build_params::ivf_pq_params(extents, dist_type); + } else if (conf.value("graph_build_algo", "") == "NN_DESCENT") { + cagra_params.graph_build_params = + cuvs::neighbors::cagra::graph_build_params::nn_descent_params( + conf.value("intermediate_graph_degree", cagra_params.intermediate_graph_degree), + dist_type); + } + ::parse_build_param(conf, cagra_params); + return cagra_params; + }; +} + +cuvs::bench::AllocatorType parse_allocator(std::string mem_type) +{ + if (mem_type == "device") { + return cuvs::bench::AllocatorType::kDevice; + } else if (mem_type == "host_pinned") { + return cuvs::bench::AllocatorType::kHostPinned; + } else if (mem_type == "host_huge_page") { + return cuvs::bench::AllocatorType::kHostHugePage; + } + THROW( + "Invalid value for memory type %s, must be one of [\"device\", \"host_pinned\", " + "\"host_huge_page\"", + mem_type.c_str()); +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_cagra::search_param& param) +{ + if (conf.contains("itopk")) { param.p.itopk_size = conf.at("itopk"); } + if (conf.contains("search_width")) { param.p.search_width = conf.at("search_width"); } + if (conf.contains("max_iterations")) { param.p.max_iterations = conf.at("max_iterations"); } + if (conf.contains("persistent")) { param.p.persistent = conf.at("persistent"); } + if (conf.contains("persistent_lifetime")) { + param.p.persistent_lifetime = conf.at("persistent_lifetime"); + } + if (conf.contains("persistent_device_usage")) { + param.p.persistent_device_usage = conf.at("persistent_device_usage"); + } + if (conf.contains("thread_block_size")) { + param.p.thread_block_size = conf.at("thread_block_size"); + } + if (conf.contains("algo")) { + if (conf.at("algo") == "single_cta") { + param.p.algo = cuvs::neighbors::cagra::search_algo::SINGLE_CTA; + } else if (conf.at("algo") == "multi_cta") { + param.p.algo = cuvs::neighbors::cagra::search_algo::MULTI_CTA; + } else if (conf.at("algo") == "multi_kernel") { + param.p.algo = cuvs::neighbors::cagra::search_algo::MULTI_KERNEL; + } else if (conf.at("algo") == "auto") { + param.p.algo = cuvs::neighbors::cagra::search_algo::AUTO; + } else { + std::string tmp = conf.at("algo"); + THROW("Invalid value for algo: %s", tmp.c_str()); + } + } + if (conf.contains("graph_memory_type")) { + param.graph_mem = parse_allocator(conf.at("graph_memory_type")); + } + if (conf.contains("internal_dataset_memory_type")) { + param.dataset_mem = parse_allocator(conf.at("internal_dataset_memory_type")); + } + // Same ratio as in IVF-PQ + param.refine_ratio = conf.value("refine_ratio", 1.0f); + + // enable dynamic batching + parse_dynamic_batching_params(conf, param); +} +#endif diff --git a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_utils.h b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_utils.h new file mode 100644 index 0000000000..3aef8a64a4 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_utils.h @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/util.hpp" +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace cuvs::bench { + +inline auto parse_metric_type(cuvs::bench::Metric metric) -> cuvs::distance::DistanceType +{ + if (metric == cuvs::bench::Metric::kInnerProduct) { + return cuvs::distance::DistanceType::InnerProduct; + } else if (metric == cuvs::bench::Metric::kEuclidean) { + // Even for L2 expanded RAFT IVF Flat uses unexpanded formula + return cuvs::distance::DistanceType::L2Expanded; + } else { + throw std::runtime_error("raft supports only metric type of inner product and L2"); + } +} + +/** Report a more verbose error with a backtrace when OOM occurs on RMM side. */ +inline auto rmm_oom_callback(std::size_t bytes, void*) -> bool +{ + auto cuda_status = cudaGetLastError(); + size_t free = 0; + size_t total = 0; + RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); + RAFT_FAIL( + "Failed to allocate %zu bytes using RMM memory resource. " + "NB: latest cuda status = %s, free memory = %zu, total memory = %zu.", + bytes, + cudaGetErrorName(cuda_status), + free, + total); +} + +/** + * This container keeps the part of raft state that should be shared among multiple copies of raft + * handles (in different CPU threads). + * An example of this is an RMM memory resource: if we had an RMM memory pool per thread, we'd + * quickly run out of memory. + */ +class shared_raft_resources { + public: + using pool_mr_type = rmm::mr::pool_memory_resource; + using mr_type = rmm::mr::failure_callback_resource_adaptor; + using large_mr_type = rmm::mr::managed_memory_resource; + + shared_raft_resources() + try + : orig_resource_{rmm::mr::get_current_device_resource()}, + pool_resource_(orig_resource_, 1024 * 1024 * 1024ull), + resource_(&pool_resource_, rmm_oom_callback, nullptr), + large_mr_() { + rmm::mr::set_current_device_resource(&resource_); + } catch (const std::exception& e) { + auto cuda_status = cudaGetLastError(); + size_t free = 0; + size_t total = 0; + RAFT_CUDA_TRY_NO_THROW(cudaMemGetInfo(&free, &total)); + RAFT_FAIL( + "Failed to initialize shared raft resources (NB: latest cuda status = %s, free memory = %zu, " + "total memory = %zu): %s", + cudaGetErrorName(cuda_status), + free, + total, + e.what()); + } + + shared_raft_resources(shared_raft_resources&&) = delete; + auto operator=(shared_raft_resources&&) -> shared_raft_resources& = delete; + shared_raft_resources(const shared_raft_resources& res) = delete; + auto operator=(const shared_raft_resources& other) -> shared_raft_resources& = delete; + + ~shared_raft_resources() noexcept { rmm::mr::set_current_device_resource(orig_resource_); } + + auto get_large_memory_resource() noexcept + { + return static_cast(&large_mr_); + } + + private: + rmm::mr::device_memory_resource* orig_resource_; + pool_mr_type pool_resource_; + mr_type resource_; + large_mr_type large_mr_; +}; + +/** + * This struct is used by multiple raft benchmark wrappers. It serves as a thread-safe keeper of + * shared and private GPU resources (see below). + * + * - Accessing the same `configured_raft_resources` from concurrent threads is not safe. + * - Accessing the copies of `configured_raft_resources` from concurrent threads is safe. + * - There must be at most one "original" `configured_raft_resources` at any time, but as many + * copies of it as needed (modifies the program static state). + */ +class configured_raft_resources { + public: + /** + * This constructor has the shared state passed unmodified but creates the local state anew. + * It's used by the copy constructor. + */ + explicit configured_raft_resources(const std::shared_ptr& shared_res) + : shared_res_{shared_res}, + res_{std::make_unique( + rmm::cuda_stream_view(get_stream_from_global_pool()))} + { + // set the large workspace resource to the raft handle, but without the deleter + // (this resource is managed by the shared_res). + raft::resource::set_large_workspace_resource( + *res_, + std::shared_ptr(shared_res_->get_large_memory_resource(), + raft::void_op{})); + } + + /** Default constructor creates all resources anew. */ + configured_raft_resources() : configured_raft_resources{std::make_shared()} + { + } + + configured_raft_resources(configured_raft_resources&&); + auto operator=(configured_raft_resources&&) -> configured_raft_resources&; + ~configured_raft_resources() = default; + configured_raft_resources(const configured_raft_resources& res) + : configured_raft_resources{res.shared_res_} + { + } + auto operator=(const configured_raft_resources& other) -> configured_raft_resources& + { + this->shared_res_ = other.shared_res_; + return *this; + } + + operator raft::resources&() noexcept { return *res_; } // NOLINT + operator const raft::resources&() const noexcept { return *res_; } // NOLINT + + /** Get the main stream */ + [[nodiscard]] auto get_sync_stream() const noexcept + { + return raft::resource::get_cuda_stream(*res_); + } + + private: + /** The resources shared among multiple raft handles / threads. */ + std::shared_ptr shared_res_; + /** + * Until we make the use of copies of raft::resources thread-safe, each benchmark wrapper must + * have its own copy of it. + */ + std::unique_ptr res_ = std::make_unique(); +}; + +inline configured_raft_resources::configured_raft_resources(configured_raft_resources&&) = default; +inline auto configured_raft_resources::operator=(configured_raft_resources&&) + -> configured_raft_resources& = default; + +/** A helper to refine the neighbors when the data is on device or on host. */ +template +void refine_helper(const raft::resources& res, + const DatasetT& dataset, + const QueriesT& queries, + const CandidatesT& candidates, + int k, + algo_base::index_type* neighbors, + float* distances, + cuvs::distance::DistanceType metric) +{ + using data_type = typename DatasetT::value_type; + using index_type = algo_base::index_type; + using extents_type = int64_t; // device-side refine requires this + + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + extents_type batch_size = queries.extent(0); + extents_type dim = queries.extent(1); + extents_type k0 = candidates.extent(1); + + if (raft::get_device_for_address(dataset.data_handle()) >= 0) { + auto dataset_device = raft::make_device_matrix_view( + dataset.data_handle(), dataset.extent(0), dataset.extent(1)); + auto queries_device = raft::make_device_matrix_view( + queries.data_handle(), batch_size, dim); + auto candidates_device = raft::make_device_matrix_view( + candidates.data_handle(), batch_size, k0); + auto neighbors_device = + raft::make_device_matrix_view(neighbors, batch_size, k); + auto distances_device = + raft::make_device_matrix_view(distances, batch_size, k); + + cuvs::neighbors::refine(res, + dataset_device, + queries_device, + candidates_device, + neighbors_device, + distances_device, + metric); + } else { + auto dataset_host = raft::make_host_matrix_view( + dataset.data_handle(), dataset.extent(0), dataset.extent(1)); + if (raft::get_device_for_address(queries.data_handle()) >= 0) { + // Queries & results are on the device + + auto queries_host = raft::make_host_matrix(batch_size, dim); + auto candidates_host = raft::make_host_matrix(batch_size, k0); + auto neighbors_host = raft::make_host_matrix(batch_size, k); + auto distances_host = raft::make_host_matrix(batch_size, k); + + auto stream = raft::resource::get_cuda_stream(res); + raft::copy(queries_host.data_handle(), queries.data_handle(), queries_host.size(), stream); + raft::copy( + candidates_host.data_handle(), candidates.data_handle(), candidates_host.size(), stream); + + raft::resource::sync_stream(res); // wait for the queries and candidates + cuvs::neighbors::refine(res, + dataset_host, + queries_host.view(), + candidates_host.view(), + neighbors_host.view(), + distances_host.view(), + metric); + + raft::copy(neighbors, neighbors_host.data_handle(), neighbors_host.size(), stream); + raft::copy(distances, distances_host.data_handle(), distances_host.size(), stream); + + } else { + // Queries & results are on the host - no device sync / copy needed + + auto queries_host = raft::make_host_matrix_view( + queries.data_handle(), batch_size, dim); + auto candidates_host = raft::make_host_matrix_view( + candidates.data_handle(), batch_size, k0); + auto neighbors_host = + raft::make_host_matrix_view(neighbors, batch_size, k); + auto distances_host = + raft::make_host_matrix_view(distances, batch_size, k); + + cuvs::neighbors::refine( + res, dataset_host, queries_host, candidates_host, neighbors_host, distances_host, metric); + } + } +} + +/** + * Construct a cuVS-compatible bitset filter object from a raw pointer to the bitset. + * + * @param[in] filter_bitset a pointer to a pre-generated bitset + * @param[in] n_rows the number of elements in the bitset / dataset. + * @return a shared pointer to the filter object (doesn't own the bitset data!) + */ +inline auto make_cuvs_filter(const void* filter_bitset, int64_t n_rows) + -> std::shared_ptr +{ + if (filter_bitset != nullptr) { + return std::make_shared>( + raft::core::bitset_view( + const_cast(reinterpret_cast(filter_bitset)), n_rows)); + } else { + return std::make_shared(); + } +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu new file mode 100644 index 0000000000..821b80cc72 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_benchmark.cu @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_param_parser.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +#ifdef CUVS_ANN_BENCH_USE_CUVS_MG +void add_distribution_mode(cuvs::neighbors::distribution_mode* dist_mode, + const nlohmann::json& conf) +{ + if (conf.contains("distribution_mode")) { + std::string distribution_mode = conf.at("distribution_mode"); + if (distribution_mode == "replicated") { + *dist_mode = cuvs::neighbors::distribution_mode::REPLICATED; + } else if (distribution_mode == "sharded") { + *dist_mode = cuvs::neighbors::distribution_mode::SHARDED; + } else { + throw std::runtime_error("invalid value for distribution_mode"); + } + } else { + // default + *dist_mode = cuvs::neighbors::distribution_mode::SHARDED; + } +}; + +void add_merge_mode(cuvs::neighbors::sharded_merge_mode* merge_mode, const nlohmann::json& conf) +{ + if (conf.contains("merge_mode")) { + std::string sharded_merge_mode = conf.at("merge_mode"); + if (sharded_merge_mode == "tree_merge") { + *merge_mode = cuvs::neighbors::sharded_merge_mode::TREE_MERGE; + } else if (sharded_merge_mode == "merge_on_root_rank") { + *merge_mode = cuvs::neighbors::sharded_merge_mode::MERGE_ON_ROOT_RANK; + } else { + throw std::runtime_error("invalid value for merge_mode"); + } + } else { + // default + *merge_mode = cuvs::neighbors::sharded_merge_mode::TREE_MERGE; + } +}; +#endif + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + [[maybe_unused]] cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v) { +#ifdef CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE + if (algo_name == "raft_brute_force" || algo_name == "cuvs_brute_force") { + a = std::make_unique>(metric, dim); + } +#endif + } + + if constexpr (std::is_same_v) {} + +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "raft_ivf_flat" || algo_name == "cuvs_ivf_flat") { + typename cuvs::bench::cuvs_ivf_flat::build_param param; + parse_build_param(conf, param); + a = std::make_unique>(metric, dim, param); + } + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_PQ + if (algo_name == "raft_ivf_pq" || algo_name == "cuvs_ivf_pq") { + typename cuvs::bench::cuvs_ivf_pq::build_param param; + parse_build_param(conf, param); + a = std::make_unique>(metric, dim, param); + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_CAGRA + if (algo_name == "raft_cagra" || algo_name == "cuvs_cagra") { + typename cuvs::bench::cuvs_cagra::build_param param; + parse_build_param(conf, param); + a = std::make_unique>(metric, dim, param); + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_MG + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "raft_mg_ivf_flat" || algo_name == "cuvs_mg_ivf_flat") { + typename cuvs::bench::cuvs_mg_ivf_flat::build_param param; + parse_build_param(conf, param); + add_distribution_mode(¶m.mode, conf); + a = std::make_unique>(metric, dim, param); + } + } + + if (algo_name == "raft_mg_ivf_pq" || algo_name == "cuvs_mg_ivf_pq") { + typename cuvs::bench::cuvs_mg_ivf_pq::build_param param; + parse_build_param(conf, param); + add_distribution_mode(¶m.mode, conf); + a = std::make_unique>(metric, dim, param); + } + + if (algo_name == "raft_mg_cagra" || algo_name == "cuvs_mg_cagra") { + typename cuvs::bench::cuvs_mg_cagra::build_param param; + parse_build_param(conf, param); + add_distribution_mode(¶m.mode, conf); + a = std::make_unique>(metric, dim, param); + } + +#endif + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ +#ifdef CUVS_ANN_BENCH_USE_CUVS_BRUTE_FORCE + if (algo_name == "raft_brute_force" || algo_name == "cuvs_brute_force") { + auto param = std::make_unique::search_param>(); + return param; + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_FLAT + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "raft_ivf_flat" || algo_name == "cuvs_ivf_flat") { + auto param = + std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_IVF_PQ + if (algo_name == "raft_ivf_pq" || algo_name == "cuvs_ivf_pq") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_CAGRA + if (algo_name == "raft_cagra" || algo_name == "cuvs_cagra") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } +#endif +#ifdef CUVS_ANN_BENCH_USE_CUVS_MG + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "raft_mg_ivf_flat" || algo_name == "cuvs_mg_ivf_flat") { + auto param = + std::make_unique::search_param>(); + parse_search_param(conf, *param); + add_merge_mode(¶m->merge_mode, conf); + return param; + } + } + + if (algo_name == "raft_mg_ivf_pq" || algo_name == "cuvs_mg_ivf_pq") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + add_merge_mode(¶m->merge_mode, conf); + return param; + } + + if (algo_name == "raft_mg_cagra" || algo_name == "cuvs_mg_cagra") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + add_merge_mode(¶m->merge_mode, conf); + return param; + } +#endif + + // else + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +}; // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(half); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/cuvs/cuvs_brute_force_knn.cu b/cpp/bench/ann/src/cuvs/cuvs_brute_force_knn.cu new file mode 100644 index 0000000000..55d5b8c704 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_brute_force_knn.cu @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace cuvs::neighbors::brute_force { + +struct print_metric { + cuvs::distance::DistanceType value; +}; + +struct RandomKNNInputs { + int num_queries; + int num_db_vecs; + int dim; + int k; + cuvs::distance::DistanceType metric; + bool row_major; +}; + +inline auto operator<<(std::ostream& os, const print_metric& p) -> std::ostream& +{ + switch (p.value) { + case cuvs::distance::DistanceType::L2Expanded: os << "L2Expanded"; break; + case cuvs::distance::DistanceType::L2SqrtExpanded: os << "L2SqrtExpanded"; break; + case cuvs::distance::DistanceType::CosineExpanded: os << "CosineExpanded"; break; + case cuvs::distance::DistanceType::L1: os << "L1"; break; + case cuvs::distance::DistanceType::L2Unexpanded: os << "L2Unexpanded"; break; + case cuvs::distance::DistanceType::L2SqrtUnexpanded: os << "L2SqrtUnexpanded"; break; + case cuvs::distance::DistanceType::InnerProduct: os << "InnerProduct"; break; + case cuvs::distance::DistanceType::Linf: os << "Linf"; break; + case cuvs::distance::DistanceType::Canberra: os << "Canberra"; break; + case cuvs::distance::DistanceType::LpUnexpanded: os << "LpUnexpanded"; break; + case cuvs::distance::DistanceType::CorrelationExpanded: os << "CorrelationExpanded"; break; + case cuvs::distance::DistanceType::JaccardExpanded: os << "JaccardExpanded"; break; + case cuvs::distance::DistanceType::HellingerExpanded: os << "HellingerExpanded"; break; + case cuvs::distance::DistanceType::Haversine: os << "Haversine"; break; + case cuvs::distance::DistanceType::BrayCurtis: os << "BrayCurtis"; break; + case cuvs::distance::DistanceType::JensenShannon: os << "JensenShannon"; break; + case cuvs::distance::DistanceType::HammingUnexpanded: os << "HammingUnexpanded"; break; + case cuvs::distance::DistanceType::KLDivergence: os << "KLDivergence"; break; + case cuvs::distance::DistanceType::RusselRaoExpanded: os << "RusselRaoExpanded"; break; + case cuvs::distance::DistanceType::DiceExpanded: os << "DiceExpanded"; break; + case cuvs::distance::DistanceType::Precomputed: os << "Precomputed"; break; + default: RAFT_FAIL("unreachable code"); + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const RandomKNNInputs& input) +{ + return os << "num_queries:" << input.num_queries << " num_vecs:" << input.num_db_vecs + << " dim:" << input.dim << " k:" << input.k << " metric:" << print_metric{input.metric} + << " row_major:" << input.row_major; +} + +template +class BruteForceKNNBenchmark { + public: + BruteForceKNNBenchmark(const RandomKNNInputs& params, const std::string& type_str) + : stream_(raft::resource::get_cuda_stream(handle_)), + params_(params), + type_str_(type_str), + database(params_.num_db_vecs * params_.dim, stream_), + search_queries(params_.num_queries * params_.dim, stream_), + cuvs_indices_(params_.num_queries * params_.k, stream_), + cuvs_distances_(params_.num_queries * params_.k, stream_) + { + raft::matrix::fill( + handle_, + raft::make_device_matrix_view(database.data(), params_.num_db_vecs, params_.dim), + T{0.0}); + raft::matrix::fill( + handle_, + raft::make_device_matrix_view(search_queries.data(), params_.num_queries, params_.dim), + T{0.0}); + raft::matrix::fill( + handle_, + raft::make_device_matrix_view(cuvs_distances_.data(), params_.num_queries, params_.k), + DistT{0.0}); + } + + void runBenchmark() + { + DistT metric_arg = 3.0; + rmm::device_uvector workspace(0, stream_); + + std::chrono::duration build_dur; + std::chrono::duration search_dur; + + auto indices = raft::make_device_matrix_view( + cuvs_indices_.data(), params_.num_queries, params_.k); + auto distances = raft::make_device_matrix_view( + cuvs_distances_.data(), params_.num_queries, params_.k); + raft::resource::sync_stream(handle_, stream_); + + if (params_.row_major) { + { + auto idx_warm = + cuvs::neighbors::brute_force::build(handle_, + raft::make_device_matrix_view( + database.data(), params_.num_db_vecs, params_.dim), + params_.metric, + metric_arg); + cuvs::neighbors::brute_force::search( + handle_, + idx_warm, + raft::make_device_matrix_view( + search_queries.data(), params_.num_queries, params_.dim), + indices, + distances, + cuvs::neighbors::filtering::none_sample_filter{}); + flush_l2_cache(); + raft::resource::sync_stream(handle_, stream_); + } + + auto start = std::chrono::high_resolution_clock::now(); + auto idx = + cuvs::neighbors::brute_force::build(handle_, + raft::make_device_matrix_view( + database.data(), params_.num_db_vecs, params_.dim), + params_.metric, + metric_arg); + raft::resource::sync_stream(handle_, stream_); + auto end = std::chrono::high_resolution_clock::now(); + build_dur = end - start; + + start = std::chrono::high_resolution_clock::now(); + cuvs::neighbors::brute_force::search( + handle_, + idx, + raft::make_device_matrix_view( + search_queries.data(), params_.num_queries, params_.dim), + indices, + distances, + cuvs::neighbors::filtering::none_sample_filter{}); + raft::resource::sync_stream(handle_, stream_); + end = std::chrono::high_resolution_clock::now(); + search_dur = end - start; + + } else { + { + auto idx_warm = + cuvs::neighbors::brute_force::build(handle_, + raft::make_device_matrix_view( + database.data(), params_.num_db_vecs, params_.dim), + params_.metric, + metric_arg); + cuvs::neighbors::brute_force::search( + handle_, + idx_warm, + raft::make_device_matrix_view( + search_queries.data(), params_.num_queries, params_.dim), + indices, + distances, + cuvs::neighbors::filtering::none_sample_filter{}); + flush_l2_cache(); + raft::resource::sync_stream(handle_, stream_); + } + + auto start = std::chrono::high_resolution_clock::now(); + auto idx = cuvs::neighbors::brute_force::build( + handle_, + raft::make_device_matrix_view( + database.data(), params_.num_db_vecs, params_.dim), + params_.metric, + metric_arg); + raft::resource::sync_stream(handle_, stream_); + auto end = std::chrono::high_resolution_clock::now(); + build_dur = end - start; + + start = std::chrono::high_resolution_clock::now(); + cuvs::neighbors::brute_force::search( + handle_, + idx, + raft::make_device_matrix_view( + search_queries.data(), params_.num_queries, params_.dim), + indices, + distances, + cuvs::neighbors::filtering::none_sample_filter{}); + raft::resource::sync_stream(handle_, stream_); + end = std::chrono::high_resolution_clock::now(); + search_dur = end - start; + } + + double total_dur = build_dur.count() + search_dur.count(); + double throughput = static_cast(params_.num_queries) / (total_dur / 1000.0); + ; + printResult(params_, build_dur.count(), search_dur.count(), total_dur, throughput); + } + + void setUp() + { + unsigned long long int seed = 1234ULL; + raft::random::RngState r(seed); + + // JensenShannon distance requires positive values + T min_val = params_.metric == cuvs::distance::DistanceType::JensenShannon ? T(0.0) : T(-1.0); + uniform(handle_, r, database.data(), params_.num_db_vecs * params_.dim, min_val, T(1.0)); + uniform(handle_, r, search_queries.data(), params_.num_queries * params_.dim, min_val, T(1.0)); + } + + private: + void flush_l2_cache() + { + int l2_cache_size = 0; + int device_id = 0; + RAFT_CUDA_TRY(cudaGetDevice(&device_id)); + RAFT_CUDA_TRY(cudaDeviceGetAttribute(&l2_cache_size, cudaDevAttrL2CacheSize, device_id)); + scratch_buf_ = rmm::device_buffer(l2_cache_size * 3, stream_); + RAFT_CUDA_TRY(cudaMemsetAsync(scratch_buf_.data(), 0, scratch_buf_.size(), stream_)); + }; + + void printResult(const RandomKNNInputs& params, + double build_time, + double search_time, + double total_time, + double throughput) + { + std::cout << std::left << std::setw(15) << type_str_ << std::setw(10) << params.num_queries + << std::setw(10) << params.num_db_vecs << std::setw(10) << params.dim << std::setw(10) + << params.k << std::setw(20) << print_metric{params.metric} << std::setw(15) + << (params.row_major ? "row" : "col") << std::right << std::setw(20) << std::fixed + << std::setprecision(3) << build_time << std::right << std::setw(20) << std::fixed + << std::setprecision(3) << search_time << std::right << std::setw(20) << std::fixed + << std::setprecision(3) << total_time << std::right << std::setw(20) << std::fixed + << std::setprecision(3) << throughput << "\n"; + } + raft::resources handle_; + cudaStream_t stream_ = 0; + RandomKNNInputs params_; + rmm::device_uvector database; + rmm::device_uvector search_queries; + rmm::device_uvector cuvs_indices_; + rmm::device_uvector cuvs_distances_; + rmm::device_buffer scratch_buf_; + std::string type_str_; +}; + +static std::vector getInputs() +{ + std::vector param_vec; + struct TestParams { + int num_queries; + int num_db_vecs; + int dim; + int k; + cuvs::distance::DistanceType metric; + bool row_major; + }; + + const std::vector params_group = raft::util::itertools::product( + {int(10), int(100), int(1024)}, + {int(1000000)}, + {int(32), int(256), int(1024)}, + {int(128), int(1024)}, + {cuvs::distance::DistanceType::InnerProduct, cuvs::distance::DistanceType::L2SqrtExpanded}, + {true, false}); + + param_vec.reserve(params_group.size()); + for (TestParams params : params_group) { + param_vec.push_back(RandomKNNInputs({params.num_queries, + params.num_db_vecs, + params.dim, + params.k, + params.metric, + params.row_major})); + } + return param_vec; +} + +void printHeader() +{ + std::cout << std::left << std::setw(15) << "Type" << std::setw(10) << "Queries" << std::setw(10) + << "Vectors" << std::setw(10) << "Dim" << std::setw(10) << "K" << std::setw(20) + << "Metric" << std::setw(15) << "Layout" << std::right << std::setw(20) + << "Build Time (ms)" << std::right << std::setw(20) << "Search Time (ms)" << std::right + << std::setw(20) << "Total Time (ms)" << std::right << std::setw(20) + << "Throughput (q/s)" + << "\n"; + std::cout << std::string(165, '-') << "\n"; +} + +void runBenchmarkForType() +{ + auto selected_inputs = getInputs(); + for (const auto& input : selected_inputs) { + { + BruteForceKNNBenchmark benchmark(input, "float"); + benchmark.setUp(); + benchmark.runBenchmark(); + } + { + BruteForceKNNBenchmark benchmark(input, "half"); + benchmark.setUp(); + benchmark.runBenchmark(); + } + } +} + +} // namespace cuvs::neighbors::brute_force + +int main() +{ + cuvs::neighbors::brute_force::printHeader(); + cuvs::neighbors::brute_force::runBenchmarkForType(); + return 0; +} diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann.cu new file mode 100644 index 0000000000..8719d83867 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann.cu @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_param_parser.h" +#include "cuvs_cagra_diskann_wrapper.h" + +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_cagra_diskann::search_param& param) +{ + if (conf.contains("L_search")) { param.L_search = conf.at("L_search"); } +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + [[maybe_unused]] cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "cuvs_cagra_diskann") { + typename cuvs::bench::cuvs_cagra_diskann::build_param param; + ::parse_build_param(conf, param); + a = std::make_unique>(metric, dim, param); + } + } + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "cuvs_cagra_diskann") { + auto param = + std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +/* +[NOTE] Dear developer, + +Please don't modify the content of the `main` function; this will make the behavior of the benchmark +executable differ depending on the cmake flags and will complicate the debugging. In particular, +don't try to setup an RMM memory resource here; it will anyway be modified by the memory resource +set on per-algorithm basis. For example, see `cuvs/cuvs_ann_bench_utils.h`. +*/ +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann_wrapper.h new file mode 100644 index 0000000000..9cf5b50930 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_diskann_wrapper.h @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cuvs_cagra_wrapper.h" +#include +#include + +#include +#include + +#include "../common/ann_types.hpp" +#include "../diskann/diskann_wrapper.h" +#include "cuvs_ann_bench_utils.h" +#include +#include + +#include +#include +#include + +namespace cuvs::bench { + +template +class cuvs_cagra_diskann : public algo, public algo_gpu { + public: + using build_param = typename cuvs_cagra::build_param; + using search_param_base = typename algo::search_param; + using search_param = typename diskann_memory::search_param; + + cuvs_cagra_diskann(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return cagra_build_.get_sync_stream(); + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kHost; + return property; + } + + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override + { + return std::make_unique>(*this); + } + + private: + raft::resources handle_{}; + build_param build_param_; + search_param search_param_; + cuvs_cagra cagra_build_; + std::shared_ptr> diskann_memory_search_; +}; + +template +cuvs_cagra_diskann::cuvs_cagra_diskann(Metric metric, int dim, const build_param& param) + : algo(metric, dim), build_param_{param}, cagra_build_{metric, dim, param, 1} +{ + diskann_memory_search_ = std::make_shared>( + metric, + dim, + typename diskann_memory::build_param{static_cast(32), static_cast(32)}); +} + +template +void cuvs_cagra_diskann::build(const T* dataset, size_t nrow) +{ + cagra_build_.build(dataset, nrow); +} + +template +void cuvs_cagra_diskann::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + diskann_memory_search_->set_search_param(param, nullptr); +} + +template +void cuvs_cagra_diskann::save(const std::string& file) const +{ + // Write graph to first index file (format from MSFT DiskANN OSS) + std::ofstream index_of(file, std::ios::out | std::ios::binary); + if (!index_of) { RAFT_FAIL("Cannot open file %s", file.c_str()); } + + size_t file_offset = 0; + index_of.seekp(file_offset, index_of.beg); + uint32_t max_degree = 0; + size_t index_size = 24; + uint32_t start = static_cast(rand() % (cagra_build_.get_index()->graph().extent(0))); + size_t num_frozen_points = 0; + uint32_t max_observed_degree = 0; + + index_of.write((char*)&index_size, sizeof(uint64_t)); + index_of.write((char*)&max_observed_degree, sizeof(uint32_t)); + index_of.write((char*)&start, sizeof(uint32_t)); + index_of.write((char*)&num_frozen_points, sizeof(size_t)); + + auto d_graph = cagra_build_.get_index()->graph(); + auto h_graph = raft::make_host_matrix(d_graph.extent(0), d_graph.extent(1)); + raft::copy(h_graph.data_handle(), + d_graph.data_handle(), + d_graph.size(), + raft::resource::get_cuda_stream(handle_)); + raft::resource::sync_stream(handle_); + + size_t total_edges = 0; + size_t num_sparse = 0; + size_t num_single = 0; + + for (uint32_t i = 0; i < h_graph.extent(0); i++) { + uint32_t node_edges = 0; + for (; node_edges < h_graph.extent(1); node_edges++) { + if (h_graph(i, node_edges) == raft::upper_bound()) { break; } + } + + if (node_edges < 3) num_sparse++; + if (node_edges < 2) num_single++; + total_edges += node_edges; + + index_of.write((char*)&node_edges, sizeof(uint32_t)); + if constexpr (!std::is_same_v) { + RAFT_FAIL("serialization is only implemented for uint32_t graph"); + } + index_of.write((char*)&h_graph(i, 0), node_edges * sizeof(uint32_t)); + + max_degree = node_edges > max_degree ? (uint32_t)node_edges : max_degree; + index_size += (size_t)(sizeof(uint32_t) * (node_edges + 1)); + } + index_of.seekp(file_offset, index_of.beg); + index_of.write((char*)&index_size, sizeof(uint64_t)); + index_of.write((char*)&max_degree, sizeof(uint32_t)); + + RAFT_LOG_DEBUG( + "Wrote file out, index size:%lu, max_degree:%u, num_sparse:%ld, num_single:%ld, total " + "edges:%ld, avg degree:%f", + index_size, + max_degree, + num_sparse, + num_single, + total_edges, + (float)total_edges / (float)h_graph.extent(0)); + + index_of.close(); + if (!index_of) { RAFT_FAIL("Error writing output %s", file.c_str()); } + + // try allocating a buffer for the dataset on host + try { + const cuvs::neighbors::strided_dataset* strided_dataset = + dynamic_cast*>( + const_cast*>(&cagra_build_.get_index()->data())); + if (strided_dataset == nullptr) { + RAFT_LOG_DEBUG("dynamic_cast to strided_dataset failed"); + } else { + auto h_dataset = + raft::make_host_matrix(strided_dataset->n_rows(), strided_dataset->dim()); + raft::copy(h_dataset.data_handle(), + strided_dataset->view().data_handle(), + strided_dataset->n_rows() * strided_dataset->dim(), + raft::resource::get_cuda_stream(handle_)); + std::string dataset_base_file = file + ".data"; + std::ofstream dataset_of(dataset_base_file, std::ios::out | std::ios::binary); + if (!dataset_of) { RAFT_FAIL("Cannot open file %s", dataset_base_file.c_str()); } + size_t dataset_file_offset = 0; + int size = static_cast(cagra_build_.get_index()->size()); + int dim = static_cast(cagra_build_.get_index()->dim()); + dataset_of.seekp(dataset_file_offset, dataset_of.beg); + dataset_of.write((char*)&size, sizeof(int)); + dataset_of.write((char*)&dim, sizeof(int)); + for (int i = 0; i < size; i++) { + dataset_of.write((char*)(h_dataset.data_handle() + i * h_dataset.extent(1)), + dim * sizeof(T)); + } + dataset_of.close(); + if (!dataset_of) { RAFT_FAIL("Error writing output %s", dataset_base_file.c_str()); } + } + } catch (std::bad_alloc& e) { + RAFT_LOG_INFO("Failed to serialize dataset"); + } catch (raft::logic_error& e) { + RAFT_LOG_INFO("Failed to serialize dataset"); + } +} + +template +void cuvs_cagra_diskann::load(const std::string& file) +{ + diskann_memory_search_->load(file); +} + +template +void cuvs_cagra_diskann::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + diskann_memory_search_->search(queries, batch_size, k, neighbors, distances); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_float.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_float.cu new file mode 100644 index 0000000000..576f1d5bd2 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_float.cu @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_cagra_wrapper.h" + +namespace cuvs::bench { +template class cuvs_cagra; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_half.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_half.cu new file mode 100644 index 0000000000..b4a3235c41 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_half.cu @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_cagra_wrapper.h" + +namespace cuvs::bench { +template class cuvs_cagra; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu new file mode 100644 index 0000000000..d27a350dfa --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib.cu @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_param_parser.h" +#include "cuvs_cagra_hnswlib_wrapper.h" + +#include +#include +#include + +namespace cuvs::bench { + +template +auto parse_build_param(const nlohmann::json& conf) -> + typename cuvs::bench::cuvs_cagra_hnswlib::build_param +{ + typename cuvs::bench::cuvs_cagra_hnswlib::build_param param; + auto& hnsw_params = param.hnsw_index_params; + auto& cagra_params = param.cagra_build_params; + if (conf.contains("hierarchy")) { + if (conf.at("hierarchy") == "none") { + hnsw_params.hierarchy = cuvs::neighbors::hnsw::HnswHierarchy::NONE; + } else if (conf.at("hierarchy") == "cpu") { + hnsw_params.hierarchy = cuvs::neighbors::hnsw::HnswHierarchy::CPU; + } else if (conf.at("hierarchy") == "gpu") { + hnsw_params.hierarchy = cuvs::neighbors::hnsw::HnswHierarchy::GPU; + } else { + THROW("Invalid value for hierarchy: %s", conf.at("hierarchy").get().c_str()); + } + } else { + hnsw_params.hierarchy = cuvs::neighbors::hnsw::HnswHierarchy::GPU; + } + if (conf.contains("ef_construction")) { + hnsw_params.ef_construction = conf.at("ef_construction"); + } + if (conf.contains("num_threads")) { hnsw_params.num_threads = conf.at("num_threads"); } + + // Reuse the CAGRA wrapper params parser + ::parse_build_param(conf, cagra_params); + // If the users provides parameter M, we can use the CAGRA-HNSW heuristics to find optimal + // parameters for the dataset and HNSW reference. + if (conf.contains("M")) { + // Postpone the parsing of the CAGRA build params until the dataset extents are known. + // We the default parameters depend on the dataset extents; and we still would like to be able + // to override them. + cagra_params.cagra_params = [conf, hnsw_params](raft::matrix_extent extents, + cuvs::distance::DistanceType dist_type) { + auto ps = cuvs::neighbors::hnsw::to_cagra_params( + extents, conf.at("M"), hnsw_params.ef_construction, dist_type); + ps.metric = dist_type; + // NB: above, we only provide the defaults. Below we parse the explicit parameters as usual. + ::parse_build_param(conf, ps); + return ps; + }; + } + return param; +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_cagra_hnswlib::search_param& param) +{ + param.hnsw_search_param.ef = conf.at("ef"); + if (conf.contains("num_threads")) { + param.hnsw_search_param.num_threads = conf.at("num_threads"); + } +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + [[maybe_unused]] cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "raft_cagra_hnswlib" || algo_name == "cuvs_cagra_hnswlib") { + auto params = parse_build_param(conf); + a = std::make_unique>(metric, dim, params); + } + } + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "raft_cagra_hnswlib" || algo_name == "cuvs_cagra_hnswlib") { + auto param = + std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(half); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +/* +[NOTE] Dear developer, + +Please don't modify the content of the `main` function; this will make the behavior of the benchmark +executable differ depending on the cmake flags and will complicate the debugging. In particular, +don't try to setup an RMM memory resource here; it will anyway be modified by the memory resource +set on per-algorithm basis. For example, see `cuvs/cuvs_ann_bench_utils.h`. +*/ +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h new file mode 100644 index 0000000000..600061dead --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_hnswlib_wrapper.h @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cuvs_cagra_wrapper.h" +#include +#include + +#include +#include + +namespace cuvs::bench { + +template +class cuvs_cagra_hnswlib : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + + struct build_param { + using cagra_wrapper_params = typename cuvs_cagra::build_param; + cagra_wrapper_params cagra_build_params; + cuvs::neighbors::hnsw::index_params hnsw_index_params; + }; + + struct search_param : public search_param_base { + cuvs::neighbors::hnsw::search_params hnsw_search_param; + }; + + cuvs_cagra_hnswlib(Metric metric, int dim, const build_param& param, int concurrent_searches = 1) + : algo(metric, dim), build_param_{param} + { + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + + [[nodiscard]] auto uses_stream() const noexcept -> bool override + { + // there's no need to synchronize with the GPU neither on build nor on search + return false; + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kHost; + return property; + } + + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override + { + return std::make_unique>(*this); + } + + private: + configured_raft_resources handle_{}; + build_param build_param_; + search_param search_param_; + std::shared_ptr> hnsw_index_; +}; + +template +void cuvs_cagra_hnswlib::build(const T* dataset, size_t nrow) +{ + // when the data set is on host, we can pass it directly to HNSW + bool dataset_is_on_host = raft::get_device_for_address(dataset) == -1; + + // re-use the CAGRA wrapper to parse build params + auto bps = build_param_.cagra_build_params; + // Not very conveniently, the CAGRA wrapper resolves parameters after the dataset shape is known, + // so it takes a lambda to do it. Even though we know the shape, we want to use the wrapper as-is, + // so we just modify that lambda. + bps.cagra_params = [dataset_is_on_host, orig_cagra_params = bps.cagra_params]( + auto dataset_extents, auto metric) { + auto params = orig_cagra_params(dataset_extents, metric); + params.attach_dataset_on_build = !dataset_is_on_host; + return params; + }; + cuvs_cagra cagra_wrapper{this->metric_, this->dim_, bps}; + + // build the CAGRA index + cagra_wrapper.build(dataset, nrow); + auto& cagra_index = *cagra_wrapper.get_index(); + + // pass the dataset directly to HNSW if it's on the host + std::optional> opt_dataset_view = std::nullopt; + if (dataset_is_on_host) { + opt_dataset_view.emplace( + raft::make_host_matrix_view(dataset, nrow, this->dim_)); + } + + // convert the index to HNSW format + hnsw_index_ = cuvs::neighbors::hnsw::from_cagra( + handle_, build_param_.hnsw_index_params, cagra_index, opt_dataset_view); +} + +template +void cuvs_cagra_hnswlib::set_search_param(const search_param_base& param_, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + search_param_ = dynamic_cast(param_); +} + +template +void cuvs_cagra_hnswlib::save(const std::string& file) const +{ + cuvs::neighbors::hnsw::serialize(handle_, file, *(hnsw_index_.get())); +} + +template +void cuvs_cagra_hnswlib::load(const std::string& file) +{ + cuvs::neighbors::hnsw::index* idx = nullptr; + cuvs::neighbors::hnsw::deserialize(handle_, + build_param_.hnsw_index_params, + file, + this->dim_, + parse_metric_type(this->metric_), + &idx); + hnsw_index_ = std::shared_ptr>(idx); +} + +template +void cuvs_cagra_hnswlib::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + // Only Latency mode is supported for now + auto queries_view = + raft::make_host_matrix_view(queries, batch_size, this->dim_); + auto neighbors_view = raft::make_host_matrix_view( + reinterpret_cast(neighbors), batch_size, k); + auto distances_view = raft::make_host_matrix_view(distances, batch_size, k); + + cuvs::neighbors::hnsw::search(handle_, + search_param_.hnsw_search_param, + *(hnsw_index_.get()), + queries_view, + neighbors_view, + distances_view); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_int8_t.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_int8_t.cu new file mode 100644 index 0000000000..e2c0735f5b --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_int8_t.cu @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_cagra_wrapper.h" + +namespace cuvs::bench { +template class cuvs_cagra; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_uint8_t.cu b/cpp/bench/ann/src/cuvs/cuvs_cagra_uint8_t.cu new file mode 100644 index 0000000000..0f38498850 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_uint8_t.cu @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_cagra_wrapper.h" + +namespace cuvs::bench { +template class cuvs_cagra; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h new file mode 100644 index 0000000000..eb4e3ec3a0 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_cagra_wrapper.h @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../../../../src/neighbors/detail/cagra/utils.hpp" +#include "../common/ann_types.hpp" +#include "../common/cuda_huge_page_resource.hpp" +#include "../common/cuda_pinned_resource.hpp" +#include "cuvs_ann_bench_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +// #include +// #include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +enum class AllocatorType { kHostPinned, kHostHugePage, kDevice }; +enum class CagraBuildAlgo { kAuto, kIvfPq, kNnDescent }; +enum class CagraMergeType { kPhysical, kLogical }; + +template +class cuvs_cagra : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + using algo::dim_; + using algo::metric_; + + struct search_param : public search_param_base { + cuvs::neighbors::cagra::search_params p; + float refine_ratio; + AllocatorType graph_mem = AllocatorType::kDevice; + AllocatorType dataset_mem = AllocatorType::kDevice; + [[nodiscard]] auto needs_dataset() const -> bool override { return true; } + /* Dynamic batching */ + bool dynamic_batching = false; + int64_t dynamic_batching_k; + int64_t dynamic_batching_max_batch_size = 4; + double dynamic_batching_dispatch_timeout_ms = 0.01; + size_t dynamic_batching_n_queues = 8; + bool dynamic_batching_conservative_dispatch = false; + }; + + struct build_param { + // The optimal defaults depend on the dataset shape and thus only available once the build + // function is called. + using dataset_dependent_params = std::function, cuvs::distance::DistanceType)>; + dataset_dependent_params cagra_params; + size_t num_dataset_splits = 1; + CagraMergeType merge_type = CagraMergeType::kPhysical; + }; + + cuvs_cagra(Metric metric, int dim, const build_param& param, int concurrent_searches = 1) + : algo(metric, dim), + index_params_(param), + + dataset_(std::make_shared>( + std::move(raft::make_device_matrix(handle_, 0, 0)))), + graph_(std::make_shared>( + std::move(raft::make_device_matrix(handle_, 0, 0)))), + input_dataset_v_( + std::make_shared>( + nullptr, 0, 0)) + + { + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void set_search_dataset(const T* dataset, size_t nrow) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + void search_base(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + + [[nodiscard]] auto uses_stream() const noexcept -> bool override + { + // If the algorithm uses persistent kernel, the CPU has to synchronize by the end of computing + // the result. Hence it guarantees the benchmark CUDA stream is empty by the end of the + // execution. Hence we inform the benchmark to not waste the time on recording & synchronizing + // the event. + return !search_params_.persistent; + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kDevice; + return property; + } + void save(const std::string& file) const override; + void load(const std::string&) override; + void save_to_hnswlib(const std::string& file) const; + std::unique_ptr> copy() override; + + auto get_index() const -> const cuvs::neighbors::cagra::index* { return index_.get(); } + + private: + // handle_ must go first to make sure it dies last and all memory allocated in pool + configured_raft_resources handle_{}; + raft::mr::cuda_pinned_resource mr_pinned_; + raft::mr::cuda_huge_page_resource mr_huge_page_; + AllocatorType graph_mem_{AllocatorType::kDevice}; + AllocatorType dataset_mem_{AllocatorType::kDevice}; + float refine_ratio_; + build_param index_params_; + bool need_dataset_update_{true}; + cuvs::neighbors::cagra::search_params search_params_; + std::shared_ptr> index_; + std::shared_ptr> graph_; + std::shared_ptr> dataset_; + std::shared_ptr> input_dataset_v_; + + std::shared_ptr> + dynamic_batcher_; + cuvs::neighbors::dynamic_batching::search_params dynamic_batcher_sp_{}; + int64_t dynamic_batching_max_batch_size_; + size_t dynamic_batching_n_queues_; + bool dynamic_batching_conservative_dispatch_; + + std::shared_ptr filter_; + std::vector>> sub_indices_; + + inline rmm::device_async_resource_ref get_mr(AllocatorType mem_type) + { + switch (mem_type) { + case (AllocatorType::kHostPinned): return &mr_pinned_; + case (AllocatorType::kHostHugePage): return &mr_huge_page_; + default: return rmm::mr::get_current_device_resource(); + } + } +}; + +template +void cuvs_cagra::build(const T* dataset, size_t nrow) +{ + auto dataset_extents = raft::make_extents(nrow, dim_); + auto params = index_params_.cagra_params(dataset_extents, parse_metric_type(metric_)); + + auto dataset_view_host = + raft::make_mdspan(dataset, dataset_extents); + auto dataset_view_device = + raft::make_mdspan(dataset, dataset_extents); + bool dataset_is_on_host = raft::get_device_for_address(dataset) == -1; + if (index_params_.num_dataset_splits <= 1) { + index_ = std::make_shared>(std::move( + dataset_is_on_host ? cuvs::neighbors::cagra::build(handle_, params, dataset_view_host) + : cuvs::neighbors::cagra::build(handle_, params, dataset_view_device))); + } else { + IdxT rows_per_split = + raft::ceildiv(nrow, static_cast(index_params_.num_dataset_splits)); + for (size_t i = 0; i < index_params_.num_dataset_splits; ++i) { + IdxT start = static_cast(i * rows_per_split); + if (start >= nrow) break; + IdxT rows = std::min(rows_per_split, static_cast(nrow) - start); + const T* sub_ptr = dataset + static_cast(start) * dim_; + auto sub_host = + raft::make_host_matrix_view(sub_ptr, rows, dim_); + auto sub_dev = + raft::make_device_matrix_view(sub_ptr, rows, dim_); + + auto sub_index = cuvs::neighbors::cagra::index(handle_, params.metric); + if (index_params_.merge_type == CagraMergeType::kPhysical) { + if (dataset_is_on_host) { + sub_index.update_dataset(handle_, sub_host); + } else { + sub_index.update_dataset(handle_, sub_dev); + } + } + if (index_params_.merge_type == CagraMergeType::kLogical) { + if (dataset_is_on_host) { + sub_index = cuvs::neighbors::cagra::build(handle_, params, sub_host); + } else { + sub_index = cuvs::neighbors::cagra::build(handle_, params, sub_dev); + } + } + auto sub_index_shared = + std::make_shared>(std::move(sub_index)); + sub_indices_.push_back(std::move(sub_index_shared)); + } + if (index_params_.merge_type == CagraMergeType::kPhysical) { + cuvs::neighbors::cagra::merge_params merge_params{params}; + merge_params.merge_strategy = cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL; + + std::vector*> indices; + indices.reserve(sub_indices_.size()); + for (auto& ptr : sub_indices_) { + indices.push_back(ptr.get()); + } + + index_ = std::make_shared>( + std::move(cuvs::neighbors::cagra::merge(handle_, merge_params, indices))); + } + } +} + +inline auto allocator_to_string(AllocatorType mem_type) -> std::string +{ + if (mem_type == AllocatorType::kDevice) { + return "device"; + } else if (mem_type == AllocatorType::kHostPinned) { + return "host_pinned"; + } else if (mem_type == AllocatorType::kHostHugePage) { + return "host_huge_page"; + } + return ""; +} + +template +void cuvs_cagra::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (index_) { filter_ = make_cuvs_filter(filter_bitset, index_->size()); } + auto sp = dynamic_cast(param); + bool needs_dynamic_batcher_update = + (dynamic_batching_max_batch_size_ != sp.dynamic_batching_max_batch_size) || + (dynamic_batching_n_queues_ != sp.dynamic_batching_n_queues) || + (dynamic_batching_conservative_dispatch_ != sp.dynamic_batching_conservative_dispatch); + dynamic_batching_max_batch_size_ = sp.dynamic_batching_max_batch_size; + dynamic_batching_n_queues_ = sp.dynamic_batching_n_queues; + dynamic_batching_conservative_dispatch_ = sp.dynamic_batching_conservative_dispatch; + search_params_ = sp.p; + refine_ratio_ = sp.refine_ratio; + if (sp.graph_mem != graph_mem_) { + // Move graph to correct memory space + graph_mem_ = sp.graph_mem; + RAFT_LOG_DEBUG("moving graph to new memory space: %s", allocator_to_string(graph_mem_).c_str()); + // We create a new graph and copy to it from existing graph + auto mr = get_mr(graph_mem_); + + // Create a new graph, then copy, and __only then__ replace the shared pointer. + auto old_graph = + index_->graph(); // view of graph_ if it exists, of an internal index member otherwise + auto new_graph = raft::make_device_mdarray(handle_, mr, old_graph.extents()); + raft::copy(new_graph.data_handle(), + old_graph.data_handle(), + old_graph.size(), + raft::resource::get_cuda_stream(handle_)); + raft::resource::sync_stream(handle_); + *graph_ = std::move(new_graph); + + // NB: update_graph() only stores a view in the index. We need to keep the graph object alive. + index_->update_graph(handle_, make_const_mdspan(graph_->view())); + needs_dynamic_batcher_update = true; + } + + if (sp.dataset_mem != dataset_mem_ || need_dataset_update_) { + dataset_mem_ = sp.dataset_mem; + + // First free up existing memory + *dataset_ = raft::make_device_matrix(handle_, 0, 0); + index_->update_dataset(handle_, make_const_mdspan(dataset_->view())); + + // Allocate space using the correct memory resource. + RAFT_LOG_DEBUG("moving dataset to new memory space: %s", + allocator_to_string(dataset_mem_).c_str()); + + auto mr = get_mr(dataset_mem_); + cuvs::neighbors::cagra::detail::copy_with_padding(handle_, *dataset_, *input_dataset_v_, mr); + + auto dataset_view = raft::make_device_strided_matrix_view( + dataset_->data_handle(), dataset_->extent(0), this->dim_, dataset_->extent(1)); + index_->update_dataset(handle_, dataset_view); + + need_dataset_update_ = false; + needs_dynamic_batcher_update = true; + } + + // dynamic batching + if (sp.dynamic_batching) { + if (!dynamic_batcher_ || needs_dynamic_batcher_update) { + dynamic_batcher_ = + std::make_shared>( + handle_, + cuvs::neighbors::dynamic_batching::index_params{ + {}, + sp.dynamic_batching_k, + sp.dynamic_batching_max_batch_size, + sp.dynamic_batching_n_queues, + sp.dynamic_batching_conservative_dispatch}, + *index_, + search_params_, + filter_.get()); + } + dynamic_batcher_sp_.dispatch_timeout_ms = sp.dynamic_batching_dispatch_timeout_ms; + } else { + if (dynamic_batcher_) { dynamic_batcher_.reset(); } + } +} + +template +void cuvs_cagra::set_search_dataset(const T* dataset, size_t nrow) +{ + if (index_params_.num_dataset_splits > 1 && + index_params_.merge_type == CagraMergeType::kLogical) { + bool dataset_is_on_host = raft::get_device_for_address(dataset) == -1; + IdxT rows_per_split = + raft::ceildiv(nrow, static_cast(index_params_.num_dataset_splits)); + for (size_t i = 0; i < sub_indices_.size(); ++i) { + IdxT start = static_cast(i * rows_per_split); + if (start >= nrow) break; + IdxT rows = std::min(rows_per_split, static_cast(nrow) - start); + const T* sub_ptr = dataset + static_cast(start) * dim_; + auto sub_host = + raft::make_host_matrix_view(sub_ptr, rows, dim_); + auto sub_dev = + raft::make_device_matrix_view(sub_ptr, rows, dim_); + auto sub_index = sub_indices_[i].get(); + if (index_params_.merge_type == CagraMergeType::kLogical) { + if (dataset_is_on_host) { + sub_index->update_dataset(handle_, sub_host); + } else { + sub_index->update_dataset(handle_, sub_dev); + } + } + } + need_dataset_update_ = false; + } else { + using ds_idx_type = decltype(index_->data().n_rows()); + bool is_vpq = + dynamic_cast*>(&index_->data()) || + dynamic_cast*>(&index_->data()); + // It can happen that we are re-using a previous algo object which already has + // the dataset set. Check if we need update. + if (static_cast(input_dataset_v_->extent(0)) != nrow || + input_dataset_v_->data_handle() != dataset) { + *input_dataset_v_ = + raft::make_device_matrix_view(dataset, nrow, this->dim_); + need_dataset_update_ = !is_vpq; // ignore update if this is a VPQ dataset. + } + } +} + +template +void cuvs_cagra::save(const std::string& file) const +{ + if (index_params_.num_dataset_splits > 1 && + index_params_.merge_type == CagraMergeType::kLogical) { + for (size_t i = 0; i < sub_indices_.size(); ++i) { + std::string subfile = file + (i == 0 ? "" : ".subidx." + std::to_string(i)); + cuvs::neighbors::cagra::serialize(handle_, subfile, *sub_indices_[i], false); + } + std::ofstream f(file + ".submeta", std::ios::out); + f << sub_indices_.size(); + f.close(); + } else { + using ds_idx_type = decltype(index_->data().n_rows()); + bool is_vpq = + dynamic_cast*>(&index_->data()) || + dynamic_cast*>(&index_->data()); + cuvs::neighbors::cagra::serialize(handle_, file, *index_, is_vpq); + } +} + +template +void cuvs_cagra::save_to_hnswlib(const std::string& file) const +{ + cuvs::neighbors::cagra::serialize_to_hnswlib(handle_, file, *index_); +} + +template +void cuvs_cagra::load(const std::string& file) +{ + std::ifstream meta(file + ".submeta", std::ios::in); + if (index_params_.num_dataset_splits > 1 && + index_params_.merge_type == CagraMergeType::kLogical && meta.good()) { + // Load multiple sub-indices for logical merge + size_t count; + meta >> count; + meta.close(); + sub_indices_.clear(); + for (size_t i = 0; i < count; ++i) { + std::string subfile = file + (i == 0 ? "" : ".subidx." + std::to_string(i)); + auto sub_index = std::make_shared>(handle_); + cuvs::neighbors::cagra::deserialize(handle_, subfile, sub_index.get()); + sub_indices_.push_back(std::move(sub_index)); + } + } else { + index_ = std::make_shared>(handle_); + cuvs::neighbors::cagra::deserialize(handle_, file, index_.get()); + } +} + +template +std::unique_ptr> cuvs_cagra::copy() +{ + return std::make_unique>(std::cref(*this)); // use copy constructor +} + +template +void cuvs_cagra::search_base( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + + auto queries_view = raft::make_device_matrix_view(queries, batch_size, dim_); + auto neighbors_view = + raft::make_device_matrix_view(neighbors, batch_size, k); + auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); + + if (dynamic_batcher_) { + cuvs::neighbors::dynamic_batching::search(handle_, + dynamic_batcher_sp_, + *dynamic_batcher_, + queries_view, + neighbors_view, + distances_view); + } else { + if (index_params_.num_dataset_splits <= 1 || + index_params_.merge_type == CagraMergeType::kPhysical) { + cuvs::neighbors::cagra::search( + handle_, search_params_, *index_, queries_view, neighbors_view, distances_view, *filter_); + } else { + if (index_params_.merge_type == CagraMergeType::kLogical) { + // TODO: index merge must happen outside of search, otherwise what are we benchmarking? + cuvs::neighbors::cagra::merge_params merge_params{cuvs::neighbors::cagra::index_params{}}; + merge_params.merge_strategy = cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_LOGICAL; + + // Create wrapped indices for composite merge + std::vector>> + wrapped_indices; + wrapped_indices.reserve(sub_indices_.size()); + for (auto& ptr : sub_indices_) { + auto index_wrapper = + cuvs::neighbors::cagra::make_index_wrapper(ptr.get()); + wrapped_indices.push_back(index_wrapper); + } + + raft::resources composite_handle(handle_); + size_t n_streams = wrapped_indices.size(); + raft::resource::set_cuda_stream_pool(composite_handle, + std::make_shared(n_streams)); + + auto merged_index = + cuvs::neighbors::composite::merge(composite_handle, merge_params, wrapped_indices); + cuvs::neighbors::filtering::none_sample_filter empty_filter; + merged_index->search(composite_handle, + search_params_, + queries_view, + neighbors_view, + distances_view, + empty_filter); + } + } + } +} + +template +void cuvs_cagra::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + + auto k0 = static_cast(refine_ratio_ * k); + const bool disable_refinement = k0 <= static_cast(k); + const raft::resources& res = handle_; + // NOTE: caching mem_type to reduce mutex locks + // raft::get_device_for_address call cuda API to get the pointer properties, + // this means it locks the context mutex for a very small amount of time. + // In the event of thread contention (such as thousands threads), this time can actually increase. + // Hence we try to bypass this check for repeated search calls. + thread_local MemoryType mem_type = MemoryType::kDevice; + thread_local algo_base::index_type* prev_neighbors = nullptr; + if (prev_neighbors != neighbors) { + prev_neighbors = neighbors; + mem_type = + raft::get_device_for_address(neighbors) >= 0 ? MemoryType::kDevice : MemoryType::kHostPinned; + } + + // If dynamic batching is used and there's no sync between benchmark laps, multiple sequential + // requests can group together. The data is copied asynchronously, and if the same intermediate + // buffer is used for multiple requests, they can override each other's data. Hence, we need to + // allocate as much space as required by the maximum number of sequential requests. + auto max_dyn_grouping = dynamic_batcher_ ? raft::div_rounding_up_safe( + dynamic_batching_max_batch_size_, batch_size) * + dynamic_batching_n_queues_ + : 1; + auto tmp_buf_size = + ((disable_refinement ? 0 : (sizeof(float) + sizeof(algo_base::index_type)))) * batch_size * k0; + auto& tmp_buf = get_tmp_buffer_from_global_pool(tmp_buf_size * max_dyn_grouping); + thread_local static int64_t group_id = 0; + auto* candidates_ptr = reinterpret_cast( + reinterpret_cast(tmp_buf.data(mem_type)) + tmp_buf_size * group_id); + group_id = (group_id + 1) % max_dyn_grouping; + auto* candidate_dists_ptr = + reinterpret_cast(candidates_ptr + (disable_refinement ? 0 : batch_size * k0)); + + if (disable_refinement) { + search_base(queries, batch_size, k, neighbors, distances); + } else { + search_base(queries, batch_size, k0, candidates_ptr, candidate_dists_ptr); + + if (mem_type == MemoryType::kHostPinned && uses_stream()) { + // If the algorithm uses a stream to synchronize (non-persistent kernel), but the data is in + // the pinned host memory, we need to synchronize before the refinement operation to wait for + // the data being available for the host. + raft::resource::sync_stream(res); + } + + auto candidate_ixs = + raft::make_device_matrix_view( + candidates_ptr, batch_size, k0); + auto queries_v = + raft::make_device_matrix_view(queries, batch_size, dim_); + refine_helper( + res, *input_dataset_v_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); + } +} +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_flat.cu b/cpp/bench/ann/src/cuvs/cuvs_ivf_flat.cu new file mode 100644 index 0000000000..f38e16c9fc --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_flat.cu @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_ivf_flat_wrapper.h" + +namespace cuvs::bench { +template class cuvs_ivf_flat; +template class cuvs_ivf_flat; +template class cuvs_ivf_flat; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_flat_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_ivf_flat_wrapper.h new file mode 100644 index 0000000000..a9dec026e4 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_flat_wrapper.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +class cuvs_ivf_flat : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + + struct search_param : public search_param_base { + cuvs::neighbors::ivf_flat::search_params ivf_flat_params; + }; + + using build_param = cuvs::neighbors::ivf_flat::index_params; + + cuvs_ivf_flat(Metric metric, int dim, const build_param& param) + : algo(metric, dim), index_params_(param), dimension_(dim) + { + index_params_.metric = parse_metric_type(metric); + index_params_.conservative_memory_allocation = true; + RAFT_CUDA_TRY(cudaGetDevice(&device_)); + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kDevice; + return property; + } + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override; + + private: + // handle_ must go first to make sure it dies last and all memory allocated in pool + configured_raft_resources handle_{}; + build_param index_params_; + cuvs::neighbors::ivf_flat::search_params search_params_; + std::shared_ptr> index_; + int device_; + int dimension_; + + std::shared_ptr filter_; +}; + +template +void cuvs_ivf_flat::build(const T* dataset, size_t nrow) +{ + // Create a CUDA stream pool with 1 stream (besides main stream) for kernel/copy overlapping. + size_t n_streams = 1; + raft::resource::set_cuda_stream_pool(handle_, std::make_shared(n_streams)); + index_ = std::make_shared>( + std::move(cuvs::neighbors::ivf_flat::build( + handle_, + index_params_, + raft::make_host_matrix_view(dataset, nrow, dimension_)))); + // Note: internally the IVF-Flat build works with simple pointers, and accepts both host and + // device pointer. Therefore, although we provide here a host_mdspan, this works with device + // pointer too. +} + +template +void cuvs_ivf_flat::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + filter_ = make_cuvs_filter(filter_bitset, index_->size()); + auto sp = dynamic_cast(param); + search_params_ = sp.ivf_flat_params; + assert(search_params_.n_probes <= index_params_.n_lists); +} + +template +void cuvs_ivf_flat::save(const std::string& file) const +{ + cuvs::neighbors::ivf_flat::serialize(handle_, file, *index_); + return; +} + +template +void cuvs_ivf_flat::load(const std::string& file) +{ + index_ = + std::make_shared>(handle_, index_params_, this->dim_); + + cuvs::neighbors::ivf_flat::deserialize(handle_, file, index_.get()); + return; +} + +template +std::unique_ptr> cuvs_ivf_flat::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +template +void cuvs_ivf_flat::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + + IdxT* neighbors_idx_t; + std::optional> neighbors_storage{std::nullopt}; + if constexpr (sizeof(IdxT) == sizeof(algo_base::index_type)) { + neighbors_idx_t = reinterpret_cast(neighbors); + } else { + neighbors_storage.emplace(batch_size * k, raft::resource::get_cuda_stream(handle_)); + neighbors_idx_t = neighbors_storage->data(); + } + cuvs::neighbors::ivf_flat::search( + handle_, + search_params_, + *index_, + raft::make_device_matrix_view(queries, batch_size, index_->dim()), + raft::make_device_matrix_view(neighbors_idx_t, batch_size, k), + raft::make_device_matrix_view(distances, batch_size, k), + *filter_); + if constexpr (sizeof(IdxT) != sizeof(algo_base::index_type)) { + raft::linalg::unaryOp(neighbors, + neighbors_idx_t, + batch_size * k, + raft::cast_op(), + raft::resource::get_cuda_stream(handle_)); + } +} +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_pq.cu b/cpp/bench/ann/src/cuvs/cuvs_ivf_pq.cu new file mode 100644 index 0000000000..2df4609666 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_pq.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_ivf_pq_wrapper.h" + +namespace cuvs::bench { +template class cuvs_ivf_pq; +template class cuvs_ivf_pq; +template class cuvs_ivf_pq; +template class cuvs_ivf_pq; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_ivf_pq_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_ivf_pq_wrapper.h new file mode 100644 index 0000000000..5bef85b7e5 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_ivf_pq_wrapper.h @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_utils.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace cuvs::bench { + +template +class cuvs_ivf_pq : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + using algo::dim_; + + struct search_param : public search_param_base { + cuvs::neighbors::ivf_pq::search_params pq_param; + float refine_ratio = 1.0f; + [[nodiscard]] auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } + /* Dynamic batching */ + bool dynamic_batching = false; + int64_t dynamic_batching_k; + int64_t dynamic_batching_max_batch_size = 128; + double dynamic_batching_dispatch_timeout_ms = 0.01; + size_t dynamic_batching_n_queues = 3; + bool dynamic_batching_conservative_dispatch = true; + }; + + using build_param = cuvs::neighbors::ivf_pq::index_params; + + cuvs_ivf_pq(Metric metric, int dim, const build_param& param) + : algo(metric, dim), index_params_(param), dimension_(dim) + { + index_params_.metric = parse_metric_type(metric); + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + void set_search_dataset(const T* dataset, size_t nrow) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + void search_base(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kDevice; + return property; + } + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override; + + private: + // handle_ must go first to make sure it dies last and all memory allocated in pool + configured_raft_resources handle_{}; + build_param index_params_; + cuvs::neighbors::ivf_pq::search_params search_params_; + std::shared_ptr> index_; + int dimension_; + float refine_ratio_ = 1.0; + raft::device_matrix_view dataset_; + + std::shared_ptr> dynamic_batcher_; + cuvs::neighbors::dynamic_batching::search_params dynamic_batcher_sp_{}; + + std::shared_ptr filter_; +}; + +template +void cuvs_ivf_pq::save(const std::string& file) const +{ + cuvs::neighbors::ivf_pq::serialize(handle_, file, *index_); +} + +template +void cuvs_ivf_pq::load(const std::string& file) +{ + index_ = std::make_shared>(handle_, index_params_, dim_); + cuvs::neighbors::ivf_pq::deserialize(handle_, file, index_.get()); +} + +template +void cuvs_ivf_pq::build(const T* dataset, size_t nrow) +{ + // Create a CUDA stream pool with 1 stream (besides main stream) for kernel/copy overlapping. + size_t n_streams = 1; + raft::resource::set_cuda_stream_pool(handle_, std::make_shared(n_streams)); + auto dataset_v = raft::make_device_matrix_view(dataset, IdxT(nrow), dim_); + std::make_shared>( + std::move(cuvs::neighbors::ivf_pq::build(handle_, index_params_, dataset_v))) + .swap(index_); +} + +template +std::unique_ptr> cuvs_ivf_pq::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +template +void cuvs_ivf_pq::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + filter_ = make_cuvs_filter(filter_bitset, index_->size()); + auto sp = dynamic_cast(param); + search_params_ = sp.pq_param; + refine_ratio_ = sp.refine_ratio; + assert(search_params_.n_probes <= index_params_.n_lists); + + if (sp.dynamic_batching) { + dynamic_batcher_ = std::make_shared>( + handle_, + cuvs::neighbors::dynamic_batching::index_params{{}, + sp.dynamic_batching_k, + sp.dynamic_batching_max_batch_size, + sp.dynamic_batching_n_queues, + sp.dynamic_batching_conservative_dispatch}, + *index_, + search_params_, + filter_.get()); + dynamic_batcher_sp_.dispatch_timeout_ms = sp.dynamic_batching_dispatch_timeout_ms; + } else { + dynamic_batcher_.reset(); + } +} + +template +void cuvs_ivf_pq::set_search_dataset(const T* dataset, size_t nrow) +{ + dataset_ = raft::make_device_matrix_view(dataset, nrow, index_->dim()); +} + +template +void cuvs_ivf_pq::search_base( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + + IdxT* neighbors_idx_t; + std::optional> neighbors_storage{std::nullopt}; + if constexpr (sizeof(IdxT) == sizeof(algo_base::index_type)) { + neighbors_idx_t = reinterpret_cast(neighbors); + } else { + neighbors_storage.emplace(batch_size * k, raft::resource::get_cuda_stream(handle_)); + neighbors_idx_t = neighbors_storage->data(); + } + + auto queries_view = + raft::make_device_matrix_view(queries, batch_size, dimension_); + auto neighbors_view = + raft::make_device_matrix_view(neighbors_idx_t, batch_size, k); + auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); + + if (dynamic_batcher_) { + cuvs::neighbors::dynamic_batching::search(handle_, + dynamic_batcher_sp_, + *dynamic_batcher_, + queries_view, + neighbors_view, + distances_view); + } else { + cuvs::neighbors::ivf_pq::search( + handle_, search_params_, *index_, queries_view, neighbors_view, distances_view, *filter_); + } + + if constexpr (sizeof(IdxT) != sizeof(algo_base::index_type)) { + raft::linalg::unaryOp(neighbors, + neighbors_idx_t, + batch_size * k, + raft::cast_op(), + raft::resource::get_cuda_stream(handle_)); + } +} + +template +void cuvs_ivf_pq::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + auto k0 = static_cast(refine_ratio_ * k); + const bool disable_refinement = k0 <= static_cast(k); + const raft::resources& res = handle_; + + if (disable_refinement) { + search_base(queries, batch_size, k, neighbors, distances); + } else { + auto queries_v = raft::make_device_matrix_view( + queries, batch_size, dimension_); + auto candidate_ixs = + raft::make_device_matrix(res, batch_size, k0); + auto candidate_dists = + raft::make_device_matrix(res, batch_size, k0); + search_base( + queries, batch_size, k0, candidate_ixs.data_handle(), candidate_dists.data_handle()); + refine_helper( + res, dataset_, queries_v, candidate_ixs, k, neighbors, distances, index_->metric()); + } +} +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_cagra.cu b/cpp/bench/ann/src/cuvs/cuvs_mg_cagra.cu new file mode 100644 index 0000000000..801caa85f2 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_cagra.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_mg_cagra_wrapper.h" + +namespace cuvs::bench { +template class cuvs_mg_cagra; +template class cuvs_mg_cagra; +template class cuvs_mg_cagra; +template class cuvs_mg_cagra; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_cagra_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_mg_cagra_wrapper.h new file mode 100644 index 0000000000..7e76900865 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_cagra_wrapper.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2024-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "cuvs_ann_bench_utils.h" +#include "cuvs_cagra_wrapper.h" +#include +#include + +namespace cuvs::bench { +using namespace cuvs::neighbors; + +enum class AllocatorType; +enum class CagraBuildAlgo; + +template +class cuvs_mg_cagra : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + using algo::dim_; + using algo::metric_; + + struct build_param : public cuvs::bench::cuvs_cagra::build_param { + cuvs::neighbors::distribution_mode mode; + }; + + struct search_param : public cuvs::bench::cuvs_cagra::search_param { + cuvs::neighbors::sharded_merge_mode merge_mode; + }; + + cuvs_mg_cagra(Metric metric, int dim, const build_param& param, int concurrent_searches = 1) + : algo(metric, dim), index_params_(param), clique_() + { + clique_.set_memory_pool(80); + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void set_search_dataset(const T* dataset, size_t nrow) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + void search_base(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + auto stream = raft::resource::get_cuda_stream(clique_); + return stream; + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + void save(const std::string& file) const override; + void load(const std::string&) override; + void save_to_hnswlib(const std::string& file) const; + std::unique_ptr> copy() override; + + private: + raft::device_resources_snmg clique_; + float refine_ratio_; + build_param index_params_; + cuvs::neighbors::mg_search_params search_params_; + std::shared_ptr, T, IdxT>> + index_; +}; + +template +void cuvs_mg_cagra::build(const T* dataset, size_t nrow) +{ + auto dataset_extents = raft::make_extents(nrow, dim_); + auto params = index_params_.cagra_params(dataset_extents, parse_metric_type(metric_)); + + cuvs::neighbors::mg_index_params build_params = params; + build_params.mode = index_params_.mode; + + auto dataset_view = + raft::make_host_matrix_view(dataset, nrow, dim_); + auto idx = cuvs::neighbors::cagra::build(clique_, build_params, dataset_view); + index_ = + std::make_shared, T, IdxT>>( + std::move(idx)); +} + +inline auto allocator_to_string(AllocatorType mem_type) -> std::string; + +template +void cuvs_mg_cagra::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + cagra::search_params* search_params_ptr_ = static_cast(&search_params_); + *search_params_ptr_ = sp.p; + search_params_.merge_mode = sp.merge_mode; + refine_ratio_ = sp.refine_ratio; +} + +template +void cuvs_mg_cagra::set_search_dataset(const T* dataset, size_t nrow) +{ +} + +template +void cuvs_mg_cagra::save(const std::string& file) const +{ + cuvs::neighbors::cagra::serialize(clique_, *index_, file); +} + +template +void cuvs_mg_cagra::load(const std::string& file) +{ + index_ = + std::make_shared, T, IdxT>>( + std::move(cuvs::neighbors::cagra::deserialize(clique_, file))); +} + +template +std::unique_ptr> cuvs_mg_cagra::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +template +void cuvs_mg_cagra::search_base( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(std::is_integral_v); + static_assert(std::is_integral_v); + + auto queries_view = + raft::make_host_matrix_view(queries, batch_size, dim_); + auto neighbors_view = + raft::make_host_matrix_view(neighbors, batch_size, k); + auto distances_view = + raft::make_host_matrix_view(distances, batch_size, k); + + cuvs::neighbors::cagra::search( + clique_, *index_, search_params_, queries_view, neighbors_view, distances_view); +} + +template +void cuvs_mg_cagra::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + auto k0 = static_cast(refine_ratio_ * k); + const bool disable_refinement = k0 <= static_cast(k); + + if (disable_refinement) { + search_base(queries, batch_size, k, neighbors, distances); + } else { + throw std::runtime_error("refinement not supported"); + } +} +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat.cu b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat.cu new file mode 100644 index 0000000000..20cdc41e36 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_mg_ivf_flat_wrapper.h" + +namespace cuvs::bench { +template class cuvs_mg_ivf_flat; +// template class cuvs_mg_ivf_flat; +template class cuvs_mg_ivf_flat; +template class cuvs_mg_ivf_flat; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat_wrapper.h new file mode 100644 index 0000000000..d42e3279bc --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_flat_wrapper.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cuvs_ann_bench_utils.h" +#include "cuvs_ivf_flat_wrapper.h" +#include +#include + +namespace cuvs::bench { +using namespace cuvs::neighbors; + +template +class cuvs_mg_ivf_flat : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + using algo::dim_; + + using build_param = cuvs::neighbors::mg_index_params; + + struct search_param : public cuvs::bench::cuvs_ivf_flat::search_param { + cuvs::neighbors::sharded_merge_mode merge_mode; + }; + + cuvs_mg_ivf_flat(Metric metric, int dim, const build_param& param) + : algo(metric, dim), index_params_(param), clique_() + { + index_params_.metric = parse_metric_type(metric); + + clique_.set_memory_pool(80); + } + + void build(const T* dataset, size_t nrow) final; + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + auto stream = raft::resource::get_cuda_stream(clique_); + return stream; + } + + [[nodiscard]] auto uses_stream() const noexcept -> bool override { return false; } + + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override; + + private: + raft::device_resources_snmg clique_; + build_param index_params_; + cuvs::neighbors::mg_search_params search_params_; + std::shared_ptr, T, IdxT>> + index_; +}; + +template +void cuvs_mg_ivf_flat::build(const T* dataset, size_t nrow) +{ + auto dataset_view = + raft::make_host_matrix_view(dataset, IdxT(nrow), IdxT(dim_)); + auto idx = cuvs::neighbors::ivf_flat::build(clique_, index_params_, dataset_view); + index_ = + std::make_shared, T, IdxT>>( + std::move(idx)); +} + +template +void cuvs_mg_ivf_flat::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + // search_params_ = sp.ivf_flat_params; + ivf_flat::search_params* search_params_ptr_ = + static_cast(&search_params_); + *search_params_ptr_ = sp.ivf_flat_params; + search_params_.merge_mode = sp.merge_mode; + assert(search_params_.n_probes <= index_params_.n_lists); +} + +template +void cuvs_mg_ivf_flat::save(const std::string& file) const +{ + cuvs::neighbors::ivf_flat::serialize(clique_, *index_, file); +} + +template +void cuvs_mg_ivf_flat::load(const std::string& file) +{ + index_ = + std::make_shared, T, IdxT>>( + std::move(cuvs::neighbors::ivf_flat::deserialize(clique_, file))); +} + +template +std::unique_ptr> cuvs_mg_ivf_flat::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +template +void cuvs_mg_ivf_flat::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + auto queries_view = raft::make_host_matrix_view( + queries, IdxT(batch_size), IdxT(dim_)); + auto neighbors_view = raft::make_host_matrix_view( + neighbors, IdxT(batch_size), IdxT(k)); + auto distances_view = raft::make_host_matrix_view( + distances, IdxT(batch_size), IdxT(k)); + + cuvs::neighbors::ivf_flat::search( + clique_, *index_, search_params_, queries_view, neighbors_view, distances_view); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq.cu b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq.cu new file mode 100644 index 0000000000..a74bab6f5e --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq.cu @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "cuvs_mg_ivf_pq_wrapper.h" + +namespace cuvs::bench { +template class cuvs_mg_ivf_pq; +template class cuvs_mg_ivf_pq; +template class cuvs_mg_ivf_pq; +template class cuvs_mg_ivf_pq; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq_wrapper.h new file mode 100644 index 0000000000..f3ce1549c9 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_mg_ivf_pq_wrapper.h @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "cuvs_ann_bench_utils.h" +#include "cuvs_ivf_pq_wrapper.h" +#include +#include + +namespace cuvs::bench { +using namespace cuvs::neighbors; + +template +class cuvs_mg_ivf_pq : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + using algo::dim_; + + using build_param = cuvs::neighbors::mg_index_params; + + struct search_param : public cuvs::bench::cuvs_ivf_pq::search_param { + cuvs::neighbors::sharded_merge_mode merge_mode; + }; + + cuvs_mg_ivf_pq(Metric metric, int dim, const build_param& param) + : algo(metric, dim), index_params_(param), clique_() + { + index_params_.metric = parse_metric_type(metric); + + clique_.set_memory_pool(80); + } + + void build(const T* dataset, size_t nrow) final; + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + auto stream = raft::resource::get_cuda_stream(clique_); + return stream; + } + + [[nodiscard]] auto uses_stream() const noexcept -> bool override { return false; } + + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override; + + private: + raft::device_resources_snmg clique_; + build_param index_params_; + cuvs::neighbors::mg_search_params search_params_; + std::shared_ptr, T, IdxT>> index_; +}; + +template +void cuvs_mg_ivf_pq::build(const T* dataset, size_t nrow) +{ + auto dataset_view = + raft::make_host_matrix_view(dataset, IdxT(nrow), IdxT(dim_)); + auto idx = cuvs::neighbors::ivf_pq::build(clique_, index_params_, dataset_view); + index_ = + std::make_shared, T, IdxT>>( + std::move(idx)); +} + +template +void cuvs_mg_ivf_pq::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + ivf_pq::search_params* search_params_ptr_ = static_cast(&search_params_); + *search_params_ptr_ = sp.pq_param; + search_params_.merge_mode = sp.merge_mode; + assert(search_params_.n_probes <= index_params_.n_lists); +} + +template +void cuvs_mg_ivf_pq::save(const std::string& file) const +{ + cuvs::neighbors::ivf_pq::serialize(clique_, *index_, file); +} + +template +void cuvs_mg_ivf_pq::load(const std::string& file) +{ + index_ = + std::make_shared, T, IdxT>>( + std::move(cuvs::neighbors::ivf_pq::deserialize(clique_, file))); +} + +template +std::unique_ptr> cuvs_mg_ivf_pq::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +template +void cuvs_mg_ivf_pq::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + auto queries_view = raft::make_host_matrix_view( + queries, IdxT(batch_size), IdxT(dim_)); + auto neighbors_view = raft::make_host_matrix_view( + neighbors, IdxT(batch_size), IdxT(k)); + auto distances_view = raft::make_host_matrix_view( + distances, IdxT(batch_size), IdxT(k)); + + cuvs::neighbors::ivf_pq::search( + clique_, *index_, search_params_, queries_view, neighbors_view, distances_view); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana.cu b/cpp/bench/ann/src/cuvs/cuvs_vamana.cu new file mode 100644 index 0000000000..524a61e0b4 --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana.cu @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "cuvs_vamana_wrapper.h" + +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_vamana::build_param& param) +{ + if (conf.contains("graph_degree")) { param.graph_degree = conf.at("graph_degree"); } + if (conf.contains("visited_size")) { param.visited_size = conf.at("visited_size"); } + if (conf.contains("alpha")) { param.alpha = conf.at("alpha"); } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::cuvs_vamana::search_param& param) +{ + if (conf.contains("L_search")) { param.L_search = conf.at("L_search"); } +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + [[maybe_unused]] cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v or std::is_same_v) { + if (algo_name == "cuvs_vamana") { + typename cuvs::bench::cuvs_vamana::build_param param; + parse_build_param(conf, param); + a = std::make_unique>(metric, dim, param); + } + } + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "cuvs_vamana") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +/* +[NOTE] Dear developer, + +Please don't modify the content of the `main` function; this will make the behavior of the benchmark +executable differ depending on the cmake flags and will complicate the debugging. In particular, +don't try to setup an RMM memory resource here; it will anyway be modified by the memory resource +set on per-algorithm basis. For example, see `cuvs/cuvs_ann_bench_utils.h`. +*/ +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h new file mode 100644 index 0000000000..882483853a --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../diskann/diskann_wrapper.h" +#include "cuvs_ann_bench_utils.h" +#include +#include + +#include +#include +#include + +namespace cuvs::bench { + +template +class cuvs_vamana : public algo, public algo_gpu { + public: + using build_param = cuvs::neighbors::vamana::index_params; + using search_param_base = typename algo::search_param; + using search_param = typename diskann_memory::search_param; + + cuvs_vamana(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kDevice; + property.query_memory_type = MemoryType::kHost; + return property; + } + + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override { return std::make_unique>(*this); } + + private: + configured_raft_resources handle_{}; + build_param vamana_index_params_; + std::shared_ptr> vamana_index_; + std::shared_ptr> diskann_memory_search_; +}; + +template +cuvs_vamana::cuvs_vamana(Metric metric, int dim, const build_param& param) + : algo(metric, dim) +{ + this->vamana_index_params_ = param; + diskann_memory_search_ = std::make_shared>( + metric, dim, typename diskann_memory::build_param{param.graph_degree, param.visited_size}); +} + +template +void cuvs_vamana::build(const T* dataset, size_t nrow) +{ + auto dataset_view_host = raft::make_mdspan( + dataset, raft::make_extents(nrow, this->dim_)); + auto dataset_view_device = raft::make_mdspan( + dataset, raft::make_extents(nrow, this->dim_)); + bool dataset_is_on_host = raft::get_device_for_address(dataset) == -1; + + vamana_index_ = std::make_shared>(std::move( + dataset_is_on_host + ? cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_host) + : cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_device))); +} + +template +void cuvs_vamana::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + diskann_memory_search_->set_search_param(param, nullptr); +} + +template +void cuvs_vamana::save(const std::string& file) const +{ + cuvs::neighbors::vamana::serialize(handle_, file, *vamana_index_); +} + +template +void cuvs_vamana::load(const std::string& file) +{ + diskann_memory_search_->load(file); +} + +template +void cuvs_vamana::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + diskann_memory_search_->search(queries, batch_size, k, neighbors, distances); +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/cuvs/cuvs_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_wrapper.h new file mode 100644 index 0000000000..69bdaeb12e --- /dev/null +++ b/cpp/bench/ann/src/cuvs/cuvs_wrapper.h @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "cuvs_ann_bench_utils.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace raft_temp { + +inline auto parse_metric_type(cuvs::bench::Metric metric) -> cuvs::distance::DistanceType +{ + switch (metric) { + case cuvs::bench::Metric::kInnerProduct: return cuvs::distance::DistanceType::InnerProduct; + case cuvs::bench::Metric::kEuclidean: return cuvs::distance::DistanceType::L2Expanded; + default: throw std::runtime_error("raft supports only metric type of inner product and L2"); + } +} +} // namespace raft_temp + +namespace cuvs::bench { + +// brute force KNN - RAFT +template +class cuvs_gpu : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + + struct search_param : public search_param_base { + [[nodiscard]] auto needs_dataset() const -> bool override { return true; } + }; + + cuvs_gpu(Metric metric, int dim); + + void build(const T*, size_t) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const final; + + // to enable dataset access from GPU memory + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kDevice; + property.query_memory_type = MemoryType::kDevice; + return property; + } + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return handle_.get_sync_stream(); + } + void set_search_dataset(const T* dataset, size_t nrow) override; + void save(const std::string& file) const override; + void load(const std::string&) override; + std::unique_ptr> copy() override; + + protected: + // handle_ must go first to make sure it dies last and all memory allocated in pool + configured_raft_resources handle_{}; + std::shared_ptr> index_; + cuvs::distance::DistanceType metric_type_; + int device_; + const T* dataset_; + size_t nrow_; + + std::shared_ptr filter_; +}; + +template +cuvs_gpu::cuvs_gpu(Metric metric, int dim) + : algo(metric, dim), metric_type_(raft_temp::parse_metric_type(metric)) +{ + static_assert(std::is_same_v || std::is_same_v, + "raft bfknn only supports float/double"); + RAFT_CUDA_TRY(cudaGetDevice(&device_)); +} + +template +void cuvs_gpu::build(const T* dataset, size_t nrow) +{ + auto dataset_view = raft::make_device_matrix_view(dataset, nrow, this->dim_); + index_ = std::make_shared>( + std::move(cuvs::neighbors::brute_force::build(handle_, dataset_view, metric_type_))); +} + +template +void cuvs_gpu::set_search_param(const search_param_base&, const void* filter_bitset) +{ + filter_ = make_cuvs_filter(filter_bitset, index_->size()); +} + +template +void cuvs_gpu::set_search_dataset(const T* dataset, size_t nrow) +{ + dataset_ = dataset; + nrow_ = nrow; + // Wrap the dataset with an index. + auto dataset_view = raft::make_device_matrix_view(dataset, nrow, this->dim_); + index_ = std::make_shared>( + std::move(cuvs::neighbors::brute_force::build(handle_, dataset_view, metric_type_))); +} + +template +void cuvs_gpu::save(const std::string& file) const +{ + // The index is just the dataset with metadata (shape). The dataset already exist on disk, + // therefore we do not need to save it here. + // We create an empty file because the benchmark logic requires an index file to be created. + std::ofstream of(file); + of.close(); +} + +template +void cuvs_gpu::load(const std::string& file) +{ + // We do not have serialization of brute force index. We can simply wrap the + // dataset into a brute force index, like it is done in set_search_dataset. +} + +template +void cuvs_gpu::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + auto queries_view = + raft::make_device_matrix_view(queries, batch_size, this->dim_); + + auto neighbors_view = + raft::make_device_matrix_view(neighbors, batch_size, k); + auto distances_view = raft::make_device_matrix_view(distances, batch_size, k); + + cuvs::neighbors::brute_force::search( + handle_, *index_, queries_view, neighbors_view, distances_view, *filter_); +} + +template +std::unique_ptr> cuvs_gpu::copy() +{ + return std::make_unique>(*this); // use copy constructor +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/diskann/diskann_benchmark.cpp b/cpp/bench/ann/src/diskann/diskann_benchmark.cpp new file mode 100644 index 0000000000..3c1e81c3ad --- /dev/null +++ b/cpp/bench/ann/src/diskann/diskann_benchmark.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "../common/conf.hpp" +#include "diskann_wrapper.h" + +#define JSON_DIAGNOSTICS 1 +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::diskann_memory::build_param& param) +{ + param.R = conf.at("R"); + param.L_build = conf.at("L_build"); + if (conf.contains("alpha")) { param.num_threads = conf.at("alpha"); } + if (conf.contains("num_threads")) { param.num_threads = conf.at("num_threads"); } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::diskann_ssd::build_param& param) +{ + param.R = conf.at("R"); + param.L_build = conf.at("L_build"); + if (conf.contains("alpha")) { param.num_threads = conf.at("alpha"); } + if (conf.contains("num_threads")) { param.num_threads = conf.at("num_threads"); } + if (conf.contains("QD")) { param.QD = conf.at("QD"); } + param.dataset_base_file = cuvs::bench::configuration::singleton().get_dataset_conf().base_file; + for (const auto& index : cuvs::bench::configuration::singleton().get_indices()) { + // The nlohmann::json operator== comparator compares each key and value by content. Reference: + // https://json.nlohmann.me/api/basic_json/operator_eq/ + if (index.build_param == conf) { + param.index_file = index.file; + break; + } + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::diskann_memory::search_param& param) +{ + param.L_search = conf.at("L_search"); +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::diskann_ssd::search_param& param) +{ + param.L_search = conf.at("L_search"); + if (conf.contains("num_nodes_to_cache")) { + param.num_nodes_to_cache = conf.at("num_nodes_to_cache"); + } + if (conf.contains("beam_width")) { param.beam_width = conf.at("beam_width"); } +} + +template class Algo> +std::unique_ptr> make_algo(cuvs::bench::Metric metric, + int dim, + const nlohmann::json& conf) +{ + typename Algo::build_param param; + parse_build_param(conf, param); + return std::make_unique>(metric, dim, param); +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "diskann_memory") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "diskann_ssd") { + a = make_algo(metric, dim, conf); + } + } + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +std::unique_ptr::search_param> create_search_param( + const std::string& algo_name, const nlohmann::json& conf) +{ + if (algo_name == "diskann_memory") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } else if (algo_name == "diskann_ssd") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +}; // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/diskann/diskann_wrapper.h b/cpp/bench/ann/src/diskann/diskann_wrapper.h new file mode 100644 index 0000000000..6e7e0dc1c7 --- /dev/null +++ b/cpp/bench/ann/src/diskann/diskann_wrapper.h @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/util.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace cuvs::bench { + +diskann::Metric parse_metric_to_diskann(cuvs::bench::Metric metric) +{ + if (metric == cuvs::bench::Metric::kInnerProduct) { + return diskann::Metric::INNER_PRODUCT; + } else if (metric == cuvs::bench::Metric::kEuclidean) { + return diskann::Metric::L2; + } else { + throw std::runtime_error("currently only inner product and L2 supported for benchmarking"); + } +} + +// TODO (tarangj): Remaining features are tracked at https://github.com/rapidsai/cuvs/issues/656 +template +class diskann_memory : public algo { + public: + struct build_param { + uint32_t R; + uint32_t L_build; + uint32_t build_pq_bytes = 0; + float alpha = 1.2; + int num_threads = omp_get_max_threads(); + }; + + using search_param_base = typename algo::search_param; + struct search_param : public search_param_base { + uint32_t L_search; + }; + + diskann_memory(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) override; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* indices, + float* distances) const override; + + void save(const std::string& index_file) const override; + void load(const std::string& index_file) override; + diskann_memory(const diskann_memory& other) = default; + std::unique_ptr> copy() override { return std::make_unique>(*this); } + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kHost; + return property; + } + + private: + std::shared_ptr diskann_index_write_params_{nullptr}; + uint32_t max_points_; + uint32_t build_pq_bytes_ = 0; + int num_threads_; + uint32_t L_search_; + Mode bench_mode_; + std::string index_path_prefix_; + std::shared_ptr> mem_index_{nullptr}; + void initialize_index_(size_t max_points); +}; + +template +diskann_memory::diskann_memory(Metric metric, int dim, const build_param& param) + : algo(metric, dim) +{ + assert(this->dim_ > 0); + num_threads_ = param.num_threads; + diskann_index_write_params_ = std::make_shared( + diskann::IndexWriteParametersBuilder(param.L_build, param.R) + .with_filter_list_size(0) + .with_alpha(param.alpha) + .with_saturate_graph(false) + .with_num_threads(param.num_threads) + .build()); +} + +template +void diskann_memory::initialize_index_(size_t max_points) +{ + this->mem_index_ = std::make_shared>(parse_metric_to_diskann(this->metric_), + this->dim_, + max_points, + diskann_index_write_params_, + nullptr, + 0, + false, + false, + false, + build_pq_bytes_ > 0, + build_pq_bytes_, + false, + false); +} +template +void diskann_memory::build(const T* dataset, size_t nrow) +{ + initialize_index_(nrow); + mem_index_->build(dataset, nrow, std::vector()); +} + +template +void diskann_memory::set_search_param(const search_param_base& param, const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + L_search_ = sp.L_search; +} + +template +void diskann_memory::search( + const T* queries, int batch_size, int k, algo_base::index_type* indices, float* distances) const +{ + for (int i = 0; i < batch_size; i++) { + mem_index_->search(queries + i * this->dim_, + static_cast(k), + L_search_, + reinterpret_cast(indices + i * k), + distances + i * k); + } +} + +template +void diskann_memory::save(const std::string& index_file) const +{ + this->mem_index_->save(index_file.c_str()); +} + +template +void diskann_memory::load(const std::string& index_file) +{ + index_path_prefix_ = index_file; + + bench_mode_ = (cuvs::bench::benchmark_n_threads > 1) ? Mode::kThroughput : Mode::kLatency; + + initialize_index_(0); + + int load_threads = (bench_mode_ == Mode::kThroughput) ? cuvs::bench::benchmark_n_threads : 1; + this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 2000); +} + +template +class diskann_ssd : public algo { + public: + struct build_param { + uint32_t R; + uint32_t L_build; + uint32_t build_pq_bytes = 0; + float alpha = 1.2; + int num_threads = omp_get_max_threads(); + uint32_t QD = 192; + std::string dataset_base_file = ""; + std::string index_file = ""; + }; + using search_param_base = typename algo::search_param; + + struct search_param : public search_param_base { + uint32_t L_search; + uint32_t num_nodes_to_cache = 10000; + int beam_width = 2; + }; + + diskann_ssd(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) override; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + + void save(const std::string& index_file) const override; + void load(const std::string& index_file) override; + diskann_ssd(const diskann_ssd& other) = default; + std::unique_ptr> copy() override { return std::make_unique>(*this); } + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHostMmap; + property.query_memory_type = MemoryType::kHost; + return property; + } + + private: + std::string index_build_params_str; + int beam_width_; + uint32_t num_nodes_to_cache_; + + // in-memory index params + uint32_t build_pq_bytes_ = 0; + uint32_t max_points_; + uint32_t L_search_ = 384; + Mode bench_mode_; + std::string base_file_; + std::string index_path_prefix_; + std::shared_ptr reader = nullptr; + std::shared_ptr> p_flash_index_; +}; + +template +diskann_ssd::diskann_ssd(Metric metric, int dim, const build_param& param) : algo(metric, dim) +{ + // Currently set the indexing RAM budget and the search RAM budget to max value to avoid sharding + uint32_t build_dram_budget = std::numeric_limits::max(); + uint32_t search_dram_budget = std::numeric_limits::max(); + index_build_params_str = + std::string(std::to_string(param.R)) + " " + std::string(std::to_string(param.L_build)) + " " + + std::string(std::to_string(search_dram_budget)) + " " + + std::string(std::to_string(build_dram_budget)) + " " + + std::string(std::to_string(param.num_threads)) + " " + std::string(std::to_string(false)) + + " " + std::string(std::to_string(false)) + " " + std::string(std::to_string(0)) + " " + + std::string(std::to_string(param.QD)); + base_file_ = param.dataset_base_file; + index_path_prefix_ = param.index_file; +} + +template +void diskann_ssd::build(const T* dataset, size_t nrow) +{ + diskann::build_disk_index(base_file_.c_str(), + index_path_prefix_.c_str(), + index_build_params_str.c_str(), + parse_metric_to_diskann(this->metric_), + false, + std::string(""), + false, + std::string(""), + std::string(""), + 0, + 0); +} + +template +void diskann_ssd::set_search_param(const search_param_base& param, const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + L_search_ = sp.L_search; + num_nodes_to_cache_ = sp.num_nodes_to_cache; + beam_width_ = sp.beam_width; +} + +template +void diskann_ssd::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + for (int64_t i = 0; i < (int64_t)batch_size; i++) { + p_flash_index_->cached_beam_search(queries + (i * this->dim_), + static_cast(k), + L_search_, + reinterpret_cast(neighbors + i * k), + distances + i * k, + beam_width_, + false, + nullptr); + } +} + +template +void diskann_ssd::save(const std::string& index_file) const +{ + // Nothing to do here. Index already saved in build stage, but an empty file has to be created + // with the index filename. + std::ofstream of(index_file); + of.close(); +} + +template +void diskann_ssd::load(const std::string& index_file) +{ + index_path_prefix_ = index_file; + + bench_mode_ = (cuvs::bench::benchmark_n_threads > 1) ? Mode::kThroughput : Mode::kLatency; + + int load_threads = (bench_mode_ == Mode::kThroughput) ? cuvs::bench::benchmark_n_threads : 1; + + reader.reset(new LinuxAlignedFileReader()); + p_flash_index_ = + std::make_shared>(reader, parse_metric_to_diskann(this->metric_)); + int result = p_flash_index_->load(load_threads, index_path_prefix_.c_str()); + std::vector node_list; + p_flash_index_->cache_bfs_levels(num_nodes_to_cache_, node_list); + p_flash_index_->load_cache_list(node_list); + node_list.clear(); + node_list.shrink_to_fit(); +} +}; // namespace cuvs::bench diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp new file mode 100644 index 0000000000..c65f7110f8 --- /dev/null +++ b/cpp/bench/ann/src/faiss/faiss_cpu_benchmark.cpp @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "faiss_cpu_wrapper.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_base_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu::build_param& param) +{ + param.nlist = conf.at("nlist"); + if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_ivf_flat::build_param& param) +{ + parse_base_build_param(conf, param); +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_ivfpq::build_param& param) +{ + parse_base_build_param(conf, param); + param.m = conf.at("M"); + if (conf.contains("usePrecomputed")) { + param.use_precomputed = conf.at("usePrecomputed"); + } else { + param.use_precomputed = false; + } + if (conf.contains("bitsPerCode")) { + param.bits_per_code = conf.at("bitsPerCode"); + } else { + param.bits_per_code = 8; + } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_ivfsq::build_param& param) +{ + parse_base_build_param(conf, param); + param.quantizer_type = conf.at("quantizer_type"); +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_hnsw_flat::build_param& param) +{ + param.M = conf.at("M"); + if (conf.contains("efConstruction")) { + param.efConstruction = conf.at("efConstruction"); + } else { + param.efConstruction = 40; + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu::search_param& param) +{ + param.nprobe = conf.at("nprobe"); + if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } + if (conf.contains("parallel_mode")) { param.parallel_mode = conf.at("parallel_mode"); } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_cpu_hnsw_flat::search_param& param) +{ + faiss::SearchParametersHNSW p; + if (conf.contains("efSearch")) { p.efSearch = conf.at("efSearch"); } + if (conf.contains("check_relative_distance")) { + p.check_relative_distance = conf.at("check_relative_distance"); + } + if (conf.contains("bounded_queue")) { p.bounded_queue = conf.at("bounded_queue"); } + param.p = p; +} + +template class Algo> +auto make_algo(cuvs::bench::Metric metric, int dim, const nlohmann::json& conf) + -> std::unique_ptr> +{ + typename Algo::build_param param; + parse_build_param(conf, param); + return std::make_unique>(metric, dim, param); +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + std::unique_ptr> a; + + if constexpr (std::is_same_v) { + cuvs::bench::Metric metric = parse_metric(distance); + if (algo_name == "faiss_cpu_ivf_flat") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_cpu_ivf_pq") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_cpu_ivf_sq") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_cpu_flat") { + a = std::make_unique>(metric, dim); + } else if (algo_name == "faiss_cpu_hnsw_flat") { + a = make_algo(metric, dim, conf); + } + } + + if constexpr (std::is_same_v) {} + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "faiss_cpu_ivf_flat" || algo_name == "faiss_cpu_ivf_pq" || + algo_name == "faiss_cpu_ivf_sq") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } else if (algo_name == "faiss_cpu_flat") { + auto param = std::make_unique::search_param>(); + return param; + } else if (algo_name == "faiss_cpu_hnsw_flat") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + // else + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h new file mode 100644 index 0000000000..b5094f111f --- /dev/null +++ b/cpp/bench/ann/src/faiss/faiss_cpu_wrapper.h @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/util.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace { + +auto parse_metric_type(cuvs::bench::Metric metric) -> faiss::MetricType +{ + if (metric == cuvs::bench::Metric::kInnerProduct) { + return faiss::METRIC_INNER_PRODUCT; + } else if (metric == cuvs::bench::Metric::kEuclidean) { + return faiss::METRIC_L2; + } else { + throw std::runtime_error("faiss supports only metric type of inner product and L2"); + } +} +} // namespace + +namespace cuvs::bench { + +template +class faiss_cpu : public algo { + public: + using search_param_base = typename algo::search_param; + struct search_param : public search_param_base { + int nprobe; + float refine_ratio = 1.0; + /** + * parallel_mode determines how queries are parallelized with OpenMP. + * + * Options: + * 0: Split over queries. + * 1: Parallelize over inverted lists. + * 2: Parallelize over both queries and inverted lists. + * 3: Split over queries with finer granularity. + */ + int parallel_mode = 0; + }; + + struct build_param { + int nlist = 1; + int ratio = 2; + }; + + faiss_cpu(Metric metric, int dim, const build_param& param) + : algo(metric, dim), + metric_type_(parse_metric_type(metric)), + nlist_{param.nlist}, + training_sample_fraction_{1.0 / double(param.ratio)} + { + static_assert(std::is_same_v, "faiss support only float type"); + } + + void build(const T* dataset, size_t nrow) final; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + + void init_quantizer(int dim) + { + if (this->metric_type_ == faiss::MetricType::METRIC_L2) { + this->quantizer_ = std::make_shared(dim); + } else if (this->metric_type_ == faiss::MetricType::METRIC_INNER_PRODUCT) { + this->quantizer_ = std::make_shared(dim); + } + } + + // TODO(snanditale): if the number of results is less than k, the remaining elements of + // 'neighbors' will be filled with (size_t)-1 + virtual void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + // to enable building big dataset which is larger than memory + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + + protected: + template + void save_(const std::string& file) const; // NOLINT + + template + void load_(const std::string& file); // NOLINT + + std::shared_ptr index_; + std::shared_ptr quantizer_; + std::shared_ptr index_refine_; + faiss::MetricType metric_type_; + int nlist_; + double training_sample_fraction_; +}; + +template +void faiss_cpu::build(const T* dataset, size_t nrow) +{ + auto index_ivf = dynamic_cast(index_.get()); + if (index_ivf != nullptr) { + // set the min/max training size for clustering to use the whole provided training set. + double trainset_size = training_sample_fraction_ * static_cast(nrow); + double points_per_centroid = trainset_size / static_cast(nlist_); + int max_ppc = std::ceil(points_per_centroid); + int min_ppc = std::floor(points_per_centroid); + if (min_ppc < index_ivf->cp.min_points_per_centroid) { + log_warn( + "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " + "points per cluster (n_lists = %d). This is smaller than the FAISS default " + "min_points_per_centroid = %d.", + static_cast(trainset_size), + nrow, + training_sample_fraction_, + min_ppc, + nlist_, + index_ivf->cp.min_points_per_centroid); + } + index_ivf->cp.max_points_per_centroid = max_ppc; + index_ivf->cp.min_points_per_centroid = min_ppc; + } + faiss::IndexHNSWFlat* hnsw_index = dynamic_cast(index_.get()); + index_->train(nrow, dataset); // faiss::IndexFlat::train() will do nothing + assert(index_->is_trained); + index_->add(nrow, dataset); + index_refine_ = std::make_shared(this->index_.get(), dataset); +} + +template +void faiss_cpu::set_search_param(const search_param_base& param, const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast(param); + int nprobe = sp.nprobe; + assert(nprobe <= nlist_); + auto index_ivf = dynamic_cast(index_.get()); + if (index_ivf != nullptr) { + index_ivf->nprobe = nprobe; + index_ivf->parallel_mode = sp.parallel_mode; + } + + if (sp.refine_ratio > 1.0) { this->index_refine_.get()->k_factor = sp.refine_ratio; } +} + +template +void faiss_cpu::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(sizeof(size_t) == sizeof(faiss::idx_t), + "sizes of size_t and faiss::idx_t are different"); + + index_->search(batch_size, queries, k, distances, reinterpret_cast(neighbors)); +} + +template +template +void faiss_cpu::save_(const std::string& file) const +{ + faiss::write_index(index_.get(), file.c_str()); +} + +template +template +void faiss_cpu::load_(const std::string& file) +{ + index_ = std::shared_ptr(dynamic_cast(faiss::read_index(file.c_str()))); +} + +template +class faiss_cpu_ivf_flat : public faiss_cpu { + public: + using typename faiss_cpu::build_param; + + faiss_cpu_ivf_flat(Metric metric, int dim, const build_param& param) + : faiss_cpu(metric, dim, param) + { + this->init_quantizer(dim); + this->index_ = std::make_shared( + this->quantizer_.get(), dim, param.nlist, this->metric_type_); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override { this->template load_(file); } + + std::unique_ptr> copy() + { + return std::make_unique>(*this); // use copy constructor + } +}; + +template +class faiss_cpu_ivfpq : public faiss_cpu { + public: + struct build_param : public faiss_cpu::build_param { + int m; + int bits_per_code; + bool use_precomputed; + }; + + faiss_cpu_ivfpq(Metric metric, int dim, const build_param& param) + : faiss_cpu(metric, dim, param) + { + this->init_quantizer(dim); + this->index_ = std::make_shared( + this->quantizer_.get(), dim, param.nlist, param.m, param.bits_per_code, this->metric_type_); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override { this->template load_(file); } + + std::unique_ptr> copy() + { + return std::make_unique>(*this); // use copy constructor + } +}; + +// TODO(snanditale): Enable this in cmake +// ref: https://github.com/rapidsai/raft/issues/1876 +template +class faiss_cpu_ivfsq : public faiss_cpu { + public: + struct build_param : public faiss_cpu::build_param { + std::string quantizer_type; + }; + + faiss_cpu_ivfsq(Metric metric, int dim, const build_param& param) + : faiss_cpu(metric, dim, param) + { + faiss::ScalarQuantizer::QuantizerType qtype; + if (param.quantizer_type == "fp16") { + qtype = faiss::ScalarQuantizer::QT_fp16; + } else if (param.quantizer_type == "int8") { + qtype = faiss::ScalarQuantizer::QT_8bit; + } else { + throw std::runtime_error("faiss_cpu_ivfsq supports only fp16 and int8 but got " + + param.quantizer_type); + } + + this->init_quantizer(dim); + this->index_ = std::make_shared( + this->quantizer_.get(), dim, param.nlist, qtype, this->metric_type_, true); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override + { + this->template load_(file); + } + + std::unique_ptr> copy() + { + return std::make_unique>(*this); // use copy constructor + } +}; + +template +class faiss_cpu_flat : public faiss_cpu { + public: + faiss_cpu_flat(Metric metric, int dim) + : faiss_cpu(metric, dim, typename faiss_cpu::build_param{}) + { + this->index_ = std::make_shared(dim, this->metric_type_); + } + + // class faiss_cpu is more like a IVF class, so need special treating here + void set_search_param(const typename algo::search_param& param, + const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto search_param = dynamic_cast::search_param&>(param); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override { this->template load_(file); } + + std::unique_ptr> copy() + { + return std::make_unique>(*this); // use copy constructor + } +}; + +template +class faiss_cpu_hnsw_flat : public faiss_cpu { + public: + struct build_param : public faiss_cpu::build_param { + int M; + int efConstruction; + }; + struct search_param : public faiss_cpu::search_param { + faiss::SearchParametersHNSW p; + }; + faiss_cpu_hnsw_flat(Metric metric, int dim, const build_param& param) + : faiss_cpu(metric, dim, param) + { + this->index_ = std::make_shared(dim, param.M, this->metric_type_); + faiss::IndexHNSWFlat* hnsw_index = static_cast(this->index_.get()); + hnsw_index->hnsw.efConstruction = param.efConstruction; + } + + void set_search_param(const typename algo::search_param& param, + const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = static_cast::search_param&>(param); + this->search_params_ = std::make_shared(sp.p); + }; + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override { this->template load_(file); } + + std::unique_ptr> copy() + { + return std::make_unique>(*this); // use copy constructor + } + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override + { + static_assert(sizeof(size_t) == sizeof(faiss::idx_t), + "sizes of size_t and faiss::idx_t are different"); + + this->index_->search(batch_size, + queries, + k, + distances, + reinterpret_cast(neighbors), + search_params_.get()); + } + + private: + std::shared_ptr search_params_; +}; + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu new file mode 100644 index 0000000000..fbe042979d --- /dev/null +++ b/cpp/bench/ann/src/faiss/faiss_gpu_benchmark.cu @@ -0,0 +1,308 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" + +#undef WARP_SIZE +#include "faiss_gpu_wrapper.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { +nlohmann::json collect_conf_with_prefix(const nlohmann::json& conf, + const std::string& prefix, + bool remove_prefix = true) +{ + nlohmann::json out; + for (auto& i : conf.items()) { + if (i.key().compare(0, prefix.size(), prefix) == 0) { + auto new_key = remove_prefix ? i.key().substr(prefix.size()) : i.key(); + out[new_key] = i.value(); + } + } + return out; +} +} // namespace + +namespace cuvs::bench { + +template +void parse_base_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu::build_param& param) +{ + param.nlist = conf.at("nlist"); + if (conf.contains("ratio")) { param.ratio = conf.at("ratio"); } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_ivf_flat::build_param& param) +{ + parse_base_build_param(conf, param); + if (conf.contains("use_cuvs")) { + param.use_cuvs = conf.at("use_cuvs"); + } else { + param.use_cuvs = false; + } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_ivfpq::build_param& param) +{ + parse_base_build_param(conf, param); + param.m = conf.at("M"); + if (conf.contains("usePrecomputed")) { + param.use_precomputed = conf.at("usePrecomputed"); + } else { + param.use_precomputed = false; + } + if (conf.contains("useFloat16")) { + param.use_float16 = conf.at("useFloat16"); + } else { + param.use_float16 = false; + } + if (conf.contains("use_cuvs")) { + param.use_cuvs = conf.at("use_cuvs"); + } else { + param.use_cuvs = false; + } + if (conf.contains("bitsPerCode")) { + param.bitsPerCode = conf.at("bitsPerCode"); + } else { + param.bitsPerCode = 8; + } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_ivfsq::build_param& param) +{ + parse_base_build_param(conf, param); + param.quantizer_type = conf.at("quantizer_type"); +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_cagra::build_param& param) +{ + if (conf.contains("graph_degree")) { + param.graph_degree = conf.at("graph_degree"); + } else { + param.graph_degree = 64; + } + if (conf.contains("intermediate_graph_degree")) { + param.intermediate_graph_degree = conf.at("intermediate_graph_degree"); + } else { + param.intermediate_graph_degree = 128; + } + if (conf.contains("cagra_build_algo")) { param.cagra_build_algo = conf.at("cagra_build_algo"); } + if (conf.contains("nn_descent_niter")) { + param.nn_descent_niter = conf.at("nn_descent_niter"); + } else { + param.nn_descent_niter = 20; + } + nlohmann::json ivf_pq_build_conf = collect_conf_with_prefix(conf, "b_"); + if (!ivf_pq_build_conf.empty()) { + faiss::gpu::IVFPQBuildCagraConfig ivf_pq_build_p; + + if (ivf_pq_build_conf.contains("nlist")) { + ivf_pq_build_p.n_lists = ivf_pq_build_conf.at("nlist"); + } + if (ivf_pq_build_conf.contains("niter")) { + ivf_pq_build_p.kmeans_n_iters = ivf_pq_build_conf.at("niter"); + } + if (ivf_pq_build_conf.contains("ratio")) { + ivf_pq_build_p.kmeans_trainset_fraction = 1.0 / static_cast(conf.at("ratio")); + } + if (ivf_pq_build_conf.contains("pq_bits")) { + ivf_pq_build_p.pq_bits = ivf_pq_build_conf.at("pq_bits"); + } + if (ivf_pq_build_conf.contains("pq_dim")) { + ivf_pq_build_p.pq_dim = ivf_pq_build_conf.at("pq_dim"); + } + param.ivf_pq_build_params = std::make_shared(ivf_pq_build_p); + } + nlohmann::json ivf_pq_search_conf = collect_conf_with_prefix(conf, "s_"); + if (!ivf_pq_search_conf.empty()) { + faiss::gpu::IVFPQSearchCagraConfig ivf_pq_search_p; + if (ivf_pq_search_conf.contains("nprobe")) { + ivf_pq_search_p.n_probes = ivf_pq_search_conf.at("nprobe"); + } + if (ivf_pq_search_conf.contains("internalDistanceDtype")) { + std::string type = ivf_pq_search_conf.at("internalDistanceDtype"); + if (type == "float") { + ivf_pq_search_p.internal_distance_dtype = CUDA_R_32F; + } else if (type == "half") { + ivf_pq_search_p.internal_distance_dtype = CUDA_R_16F; + } else { + throw std::runtime_error("internalDistanceDtype: '" + type + + "', should be either 'float' or 'half'"); + } + } else { + // set half as default type + ivf_pq_search_p.internal_distance_dtype = CUDA_R_16F; + } + + if (ivf_pq_search_conf.contains("smemLutDtype")) { + std::string type = ivf_pq_search_conf.at("smemLutDtype"); + if (type == "float") { + ivf_pq_search_p.lut_dtype = CUDA_R_32F; + } else if (type == "half") { + ivf_pq_search_p.lut_dtype = CUDA_R_16F; + } else if (type == "fp8") { + ivf_pq_search_p.lut_dtype = CUDA_R_8U; + } else { + throw std::runtime_error("smemLutDtype: '" + type + + "', should be either 'float', 'half' or 'fp8'"); + } + } else { + // set half as default + ivf_pq_search_p.lut_dtype = CUDA_R_16F; + } + param.ivf_pq_search_params = + std::make_shared(ivf_pq_search_p); + } +} + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_cagra_hnsw::build_param& param) +{ + typename cuvs::bench::faiss_gpu_cagra::build_param p; + parse_build_param(conf, p); + param.p = p; + if (conf.contains("base_level_only")) { param.base_level_only = conf.at("base_level_only"); } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu::search_param& param) +{ + if (conf.contains("nprobe")) { param.nprobe = conf.at("nprobe"); } + if (conf.contains("refine_ratio")) { param.refine_ratio = conf.at("refine_ratio"); } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_cagra::search_param& param) +{ + if (conf.contains("itopk")) { param.p.itopk_size = conf.at("itopk"); } + if (conf.contains("search_width")) { param.p.search_width = conf.at("search_width"); } + if (conf.contains("max_iterations")) { param.p.max_iterations = conf.at("max_iterations"); } + if (conf.contains("algo")) { + if (conf.at("algo") == "single_cta") { + param.p.algo = faiss::gpu::search_algo::SINGLE_CTA; + } else if (conf.at("algo") == "multi_cta") { + param.p.algo = faiss::gpu::search_algo::MULTI_CTA; + } else if (conf.at("algo") == "multi_kernel") { + param.p.algo = faiss::gpu::search_algo::MULTI_KERNEL; + } else if (conf.at("algo") == "auto") { + param.p.algo = faiss::gpu::search_algo::AUTO; + } else { + std::string tmp = conf.at("algo"); + THROW("Invalid value for algo: %s", tmp.c_str()); + } + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::faiss_gpu_cagra_hnsw::search_param& param) +{ + if (conf.contains("efSearch")) { param.p.efSearch = conf.at("efSearch"); } +} + +template class Algo> +auto make_algo(cuvs::bench::Metric metric, int dim, const nlohmann::json& conf) + -> std::unique_ptr> +{ + typename Algo::build_param param; + parse_build_param(conf, param); + return std::make_unique>(metric, dim, param); +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + std::unique_ptr> a; + + if constexpr (std::is_same_v) { + cuvs::bench::Metric metric = parse_metric(distance); + if (algo_name == "faiss_gpu_ivf_flat") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_gpu_ivf_pq") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_gpu_ivf_sq") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_gpu_flat") { + a = std::make_unique>(metric, dim); + } else if (algo_name == "faiss_gpu_cagra") { + a = make_algo(metric, dim, conf); + } else if (algo_name == "faiss_gpu_cagra_hnsw") { + a = make_algo(metric, dim, conf); + } + } + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "faiss_gpu_ivf_flat" || algo_name == "faiss_gpu_ivf_pq" || + algo_name == "faiss_gpu_ivf_sq") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } else if (algo_name == "faiss_gpu_flat") { + auto param = std::make_unique::search_param>(); + return param; + } else if (algo_name == "faiss_gpu_cagra") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } else if (algo_name == "faiss_gpu_cagra_hnsw") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + // else + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h new file mode 100644 index 0000000000..b4bd384e13 --- /dev/null +++ b/cpp/bench/ann/src/faiss/faiss_gpu_wrapper.h @@ -0,0 +1,675 @@ +/* + * Copyright (c) 2023-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/util.hpp" +#include "../cuvs/cuvs_ann_bench_utils.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + +auto parse_metric_faiss(cuvs::bench::Metric metric) -> faiss::MetricType +{ + if (metric == cuvs::bench::Metric::kInnerProduct) { + return faiss::METRIC_INNER_PRODUCT; + } else if (metric == cuvs::bench::Metric::kEuclidean) { + return faiss::METRIC_L2; + } else { + throw std::runtime_error("faiss supports only metric type of inner product and L2"); + } +} + +// note BLAS library can still use multi-threading, and +// setting environment variable like OPENBLAS_NUM_THREADS can control it +class omp_single_thread_scope { + public: + omp_single_thread_scope() + { + max_threads_ = omp_get_max_threads(); + omp_set_num_threads(1); + } + ~omp_single_thread_scope() + { + // the best we can do + omp_set_num_threads(max_threads_); + } + + private: + int max_threads_; +}; + +} // namespace + +namespace cuvs::bench { + +template +class faiss_gpu : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + struct search_param : public search_param_base { + int nprobe = 1; + float refine_ratio = 1.0; + [[nodiscard]] auto needs_dataset() const -> bool override { return refine_ratio > 1.0f; } + }; + + struct build_param { + int nlist = 1; + int ratio = 2; + }; + + faiss_gpu(Metric metric, int dim, const build_param& param) + : algo(metric, dim), + gpu_resource_{std::make_shared()}, + metric_type_(parse_metric_faiss(metric)), + nlist_{param.nlist}, + training_sample_fraction_{1.0 / double(param.ratio)} + { + static_assert(std::is_same_v, "faiss support only float type"); + cudaGetDevice(&device_); + // Disable Faiss' generic temporary memory reservation. All such allocations happen through the + // pool memory resource. + gpu_resource_->noTempMemory(); + } + + virtual void build(const T* dataset, size_t nrow); + + virtual void set_search_param(const search_param_base& param, const void* filter_bitset) {} + + void set_search_dataset(const T* dataset, size_t nrow) override { dataset_ = dataset; } + + // TODO(snanditale): if the number of results is less than k, the remaining elements of + // 'neighbors' will be filled with (size_t)-1 + virtual void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const; + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return gpu_resource_->getDefaultStream(device_); + } + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + // to enable building big dataset which is larger than GPU memory + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + + protected: + template + void save_(const std::string& file) const; // NOLINT + + template + void load_(const std::string& file); // NOLINT + + /** [NOTE Multithreading] + * + * `gpu_resource_` is a shared resource: + * 1. It uses a shared_ptr under the hood, so the copies of it refer to the same + * resource implementation instance + * 2. GpuIndex is probably keeping a reference to it, as it's passed to the constructor + * + * To avoid copying the index (database) in each thread, we make both the index and + * the gpu_resource shared. + * This means faiss GPU streams are possibly shared among the CPU threads; + * the throughput search mode may be inaccurate. + * + * WARNING: we haven't investigated whether faiss::gpu::GpuIndex or + * faiss::gpu::StandardGpuResources are thread-safe. + * + */ + + // simply owning a configured_raft_resource object takes care of setting the pool memory resource + configured_raft_resources handle_{}; + mutable std::shared_ptr gpu_resource_; + std::shared_ptr index_; + std::shared_ptr index_refine_{nullptr}; + faiss::MetricType metric_type_; + int nlist_; + int device_ = 0; + double training_sample_fraction_; + std::shared_ptr search_params_; + std::shared_ptr refine_search_params_{nullptr}; + const T* dataset_; + float refine_ratio_ = 1.0; +}; + +template +void faiss_gpu::build(const T* dataset, size_t nrow) +{ + omp_single_thread_scope omp_single_thread; + auto index_ivf = dynamic_cast(index_.get()); + if (index_ivf != nullptr) { + // set the min/max training size for clustering to use the whole provided training set. + double trainset_size = training_sample_fraction_ * static_cast(nrow); + double points_per_centroid = trainset_size / static_cast(nlist_); + int max_ppc = std::ceil(points_per_centroid); + int min_ppc = std::floor(points_per_centroid); + if (min_ppc < index_ivf->cp.min_points_per_centroid) { + log_warn( + "The suggested training set size %zu (data size %zu, training sample ratio %f) yields %d " + "points per cluster (n_lists = %d). This is smaller than the FAISS default " + "min_points_per_centroid = %d.", + static_cast(trainset_size), + nrow, + training_sample_fraction_, + min_ppc, + nlist_, + index_ivf->cp.min_points_per_centroid); + } + index_ivf->cp.max_points_per_centroid = max_ppc; + index_ivf->cp.min_points_per_centroid = min_ppc; + } + index_->train(nrow, dataset); // faiss::gpu::GpuIndexFlat::train() will do nothing + assert(index_->is_trained); + auto index_cagra = dynamic_cast(index_.get()); + if (index_cagra == nullptr) { index_->add(nrow, dataset); } +} + +template +void faiss_gpu::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + ASSERT( + cuvs::bench::benchmark_n_threads == 1, + "Throughput mode disabled. Underlying StandardGpuResources object might not be thread-safe."); + using IdxT = faiss::idx_t; + static_assert(sizeof(size_t) == sizeof(faiss::idx_t), + "sizes of size_t and faiss::idx_t are different"); + + if (refine_ratio_ > 1.0) { + if (raft::get_device_for_address(queries) >= 0) { + uint32_t k0 = static_cast(refine_ratio_ * k); + auto distances_tmp = raft::make_device_matrix( + gpu_resource_->getRaftHandle(device_), batch_size, k0); + auto candidates = + raft::make_device_matrix(gpu_resource_->getRaftHandle(device_), batch_size, k0); + index_->search(batch_size, + queries, + k0, + distances_tmp.data_handle(), + candidates.data_handle(), + this->search_params_.get()); + gpu_resource_->getRaftHandle(device_).sync_stream(); + + auto queries_host = raft::make_host_matrix(batch_size, index_->d); + auto candidates_host = raft::make_host_matrix(batch_size, k0); + auto neighbors_host = raft::make_host_matrix(batch_size, k); + auto distances_host = raft::make_host_matrix(batch_size, k); + auto dataset_v = raft::make_host_matrix_view( + this->dataset_, index_->ntotal, index_->d); + + raft::device_resources handle_ = gpu_resource_->getRaftHandle(device_); + + raft::copy(queries_host.data_handle(), queries, queries_host.size(), handle_.get_stream()); + raft::copy(candidates_host.data_handle(), + candidates.data_handle(), + candidates_host.size(), + handle_.get_stream()); + + // wait for the queries to copy to host in 'stream` + handle_.sync_stream(); + + cuvs::neighbors::refine(handle_, + dataset_v, + queries_host.view(), + candidates_host.view(), + neighbors_host.view(), + distances_host.view(), + parse_metric_type(this->metric_)); + handle_.sync_stream(); + + raft::copy( + neighbors, neighbors_host.data_handle(), neighbors_host.size(), handle_.get_stream()); + raft::copy( + distances, distances_host.data_handle(), distances_host.size(), handle_.get_stream()); + } else { + index_refine_->search(batch_size, + queries, + k, + distances, + reinterpret_cast(neighbors), + this->refine_search_params_.get()); + } + } else { + index_->search(batch_size, + queries, + k, + distances, + reinterpret_cast(neighbors), + this->search_params_.get()); + } +} + +template +template +void faiss_gpu::save_(const std::string& file) const +{ + omp_single_thread_scope omp_single_thread; + + auto cpu_index = std::make_unique(); + auto hnsw_index = dynamic_cast(cpu_index.get()); + if (hnsw_index) { hnsw_index->base_level_only = true; } + static_cast(index_.get())->copyTo(cpu_index.get()); + faiss::write_index(cpu_index.get(), file.c_str()); +} + +template +template +void faiss_gpu::load_(const std::string& file) +{ + omp_single_thread_scope omp_single_thread; + + std::unique_ptr cpu_index(dynamic_cast(faiss::read_index(file.c_str()))); + assert(cpu_index); + + try { + dynamic_cast(index_.get())->copyFrom(cpu_index.get()); + + } catch (const std::exception& e) { + std::cout << "Error loading index file: " << std::string(e.what()) << std::endl; + } +} + +template +class faiss_gpu_ivf_flat : public faiss_gpu { + public: + struct build_param : public faiss_gpu::build_param { + bool use_cuvs; + }; + using typename faiss_gpu::search_param_base; + + faiss_gpu_ivf_flat(Metric metric, int dim, const build_param& param) + : faiss_gpu(metric, dim, param) + { + faiss::gpu::GpuIndexIVFFlatConfig config; + config.device = this->device_; + config.use_cuvs = param.use_cuvs; + this->index_ = std::make_shared( + this->gpu_resource_.get(), dim, param.nlist, this->metric_type_, config); + } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast::search_param&>(param); + int nprobe = sp.nprobe; + assert(nprobe <= this->nlist_); + + faiss::IVFSearchParameters faiss_search_params; + faiss_search_params.nprobe = nprobe; + this->search_params_ = std::make_shared(faiss_search_params); + this->refine_ratio_ = sp.refine_ratio; + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override + { + this->template load_(file); + } + std::unique_ptr> copy() override + { + return std::make_unique>(*this); + } +}; + +template +class faiss_gpu_ivfpq : public faiss_gpu { + public: + struct build_param : public faiss_gpu::build_param { + int m; + bool use_float16; + bool use_precomputed; + bool use_cuvs; + int bitsPerCode; + }; + using typename faiss_gpu::search_param_base; + + faiss_gpu_ivfpq(Metric metric, int dim, const build_param& param) + : faiss_gpu(metric, dim, param) + { + faiss::gpu::GpuIndexIVFPQConfig config; + config.useFloat16LookupTables = param.use_float16; + config.usePrecomputedTables = param.use_precomputed; + config.use_cuvs = param.use_cuvs; + if (param.use_cuvs) { config.interleavedLayout = param.use_cuvs; } + config.device = this->device_; + + this->index_ = std::make_shared(this->gpu_resource_.get(), + dim, + param.nlist, + param.m, + param.bitsPerCode, + this->metric_type_, + config); + } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast::search_param&>(param); + int nprobe = sp.nprobe; + assert(nprobe <= this->nlist_); + this->refine_ratio_ = sp.refine_ratio; + faiss::IVFPQSearchParameters faiss_search_params; + faiss_search_params.nprobe = nprobe; + + this->search_params_ = std::make_shared(faiss_search_params); + + if (sp.refine_ratio > 1.0) { + this->index_refine_ = + std::make_shared(this->index_.get(), this->dataset_); + this->index_refine_.get()->k_factor = sp.refine_ratio; + faiss::IndexRefineSearchParameters faiss_refine_search_params; + faiss_refine_search_params.k_factor = this->index_refine_.get()->k_factor; + faiss_refine_search_params.base_index_params = this->search_params_.get(); + this->refine_search_params_ = + std::make_unique(faiss_refine_search_params); + } + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override + { + this->template load_(file); + } + std::unique_ptr> copy() override { return std::make_unique>(*this); }; +}; + +// TODO(snanditale): Enable this in cmake +// ref: https://github.com/rapidsai/raft/issues/1876 +template +class faiss_gpu_ivfsq : public faiss_gpu { + public: + struct build_param : public faiss_gpu::build_param { + std::string quantizer_type; + }; + using typename faiss_gpu::search_param_base; + + faiss_gpu_ivfsq(Metric metric, int dim, const build_param& param) + : faiss_gpu(metric, dim, param) + { + faiss::ScalarQuantizer::QuantizerType qtype; + if (param.quantizer_type == "fp16") { + qtype = faiss::ScalarQuantizer::QT_fp16; + } else if (param.quantizer_type == "int8") { + qtype = faiss::ScalarQuantizer::QT_8bit; + } else { + throw std::runtime_error("faiss_gpu_ivfsq supports only fp16 and int8 but got " + + param.quantizer_type); + } + + faiss::gpu::GpuIndexIVFScalarQuantizerConfig config; + config.device = this->device_; + this->index_ = std::make_shared( + this->gpu_resource_.get(), dim, param.nlist, qtype, this->metric_type_, true, config); + } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast::search_param&>(param); + int nprobe = sp.nprobe; + assert(nprobe <= this->nlist_); + + faiss::IVFSearchParameters faiss_search_params; + faiss_search_params.nprobe = nprobe; + + this->search_params_ = std::make_shared(faiss_search_params); + this->refine_ratio_ = sp.refine_ratio; + if (sp.refine_ratio > 1.0) { + this->index_refine_ = + std::make_shared(this->index_.get(), this->dataset_); + this->index_refine_.get()->k_factor = sp.refine_ratio; + } + } + + void save(const std::string& file) const override + { + this->template save_( + file); + } + void load(const std::string& file) override + { + this->template load_( + file); + } + std::unique_ptr> copy() override { return std::make_unique>(*this); }; +}; + +template +class faiss_gpu_flat : public faiss_gpu { + public: + using typename faiss_gpu::search_param_base; + + faiss_gpu_flat(Metric metric, int dim) + : faiss_gpu(metric, dim, typename faiss_gpu::build_param{}) + { + faiss::gpu::GpuIndexFlatConfig config; + config.device = this->device_; + this->index_ = std::make_shared( + this->gpu_resource_.get(), dim, this->metric_type_, config); + } + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = dynamic_cast::search_param&>(param); + int nprobe = sp.nprobe; + assert(nprobe <= this->nlist_); + + this->search_params_ = std::make_shared(); + } + + void save(const std::string& file) const override + { + this->template save_(file); + } + void load(const std::string& file) override + { + this->template load_(file); + } + std::unique_ptr> copy() override { return std::make_unique>(*this); }; +}; + +template +class faiss_gpu_cagra : public faiss_gpu { + public: + struct build_param : public faiss_gpu::build_param { + size_t intermediate_graph_degree; + /// Degree of output graph. + size_t graph_degree; + /// ANN algorithm to build knn graph. + std::string cagra_build_algo; + /// Number of Iterations to run if building with NN_DESCENT + size_t nn_descent_niter; + + std::shared_ptr ivf_pq_build_params = nullptr; + + std::shared_ptr ivf_pq_search_params = nullptr; + }; + using typename faiss_gpu::search_param_base; + struct search_param : public faiss_gpu::search_param { + faiss::gpu::SearchParametersCagra p; + }; + + faiss_gpu_cagra(Metric metric, int dim, const build_param& param) + : faiss_gpu(metric, dim, param) + { + faiss::gpu::GpuIndexCagraConfig config; + config.graph_degree = param.graph_degree; + config.intermediate_graph_degree = param.intermediate_graph_degree; + config.device = this->device_; + config.store_dataset = true; + if (param.cagra_build_algo == "IVF_PQ") { + config.build_algo = faiss::gpu::graph_build_algo::IVF_PQ; + this->ivf_pq_build_params_ = param.ivf_pq_build_params; + config.ivf_pq_params = this->ivf_pq_build_params_; + this->ivf_pq_search_params_ = param.ivf_pq_search_params; + config.ivf_pq_search_params = this->ivf_pq_search_params_; + config.refine_rate = 1.0; + } else { + config.build_algo = faiss::gpu::graph_build_algo::NN_DESCENT; + } + config.nn_descent_niter = param.nn_descent_niter; + + this->index_ = std::make_shared( + this->gpu_resource_.get(), dim, parse_metric_faiss(this->metric_), config); + } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = static_cast::search_param&>(param); + this->search_params_ = std::make_shared(sp.p); + } + + void save(const std::string& file) const override + { + omp_single_thread_scope omp_single_thread; + + auto cpu_hnsw_index = std::make_unique(); + // Only add the base HNSW layer to serialize the CAGRA index. + cpu_hnsw_index->base_level_only = true; + static_cast(this->index_.get())->copyTo(cpu_hnsw_index.get()); + faiss::write_index(cpu_hnsw_index.get(), file.c_str()); + } + void load(const std::string& file) override + { + this->template load_(file); + } + std::unique_ptr> copy() override { return std::make_unique>(*this); }; + + std::shared_ptr faiss_index() { return this->index_; } + + private: + std::shared_ptr ivf_pq_build_params_; + std::shared_ptr ivf_pq_search_params_; +}; + +template +class faiss_gpu_cagra_hnsw : public faiss_gpu { + public: + struct build_param : public faiss_gpu::build_param { + typename faiss_gpu_cagra::build_param p; + bool base_level_only = true; + }; + using typename faiss_gpu::search_param_base; + struct search_param : public faiss_gpu::search_param { + faiss::SearchParametersHNSW p; + }; + + faiss_gpu_cagra_hnsw(Metric metric, int dim, const build_param& param) + : faiss_gpu(metric, dim, param) + { + this->build_index_ = std::make_shared>(metric, dim, param.p); + this->search_index_ = std::make_shared( + dim, int(param.p.graph_degree / 2), this->metric_type_); + this->search_index_->base_level_only = param.base_level_only; + } + + void build(const T* dataset, size_t nrow) override + { + this->build_index_->build(dataset, nrow); + static_cast((build_index_->faiss_index()).get()) + ->copyTo(search_index_.get()); + } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto sp = static_cast::search_param&>(param); + this->search_params_ = std::make_shared(sp.p); + } + + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override + { + search_index_->search(batch_size, + queries, + k, + distances, + reinterpret_cast(neighbors), + this->search_params_.get()); + } + + void save(const std::string& file) const override + { + faiss::write_index(search_index_.get(), file.c_str()); + } + void load(const std::string& file) override + { + omp_single_thread_scope omp_single_thread; + this->search_index_.reset(static_cast(faiss::read_index(file.c_str()))); + } + + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + if (this->gpu_resource_ == nullptr) { return 0; } + return this->gpu_resource_->getDefaultStream(this->device_); + } + std::unique_ptr> copy() override + { + auto new_instance = std::make_unique>(*this); + new_instance->build_index_ = nullptr; + return new_instance; + }; + + private: + std::shared_ptr> build_index_; + std::shared_ptr search_index_; +}; +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu new file mode 100644 index 0000000000..6b1a1eaf74 --- /dev/null +++ b/cpp/bench/ann/src/ggnn/ggnn_benchmark.cu @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "ggnn_wrapper.cuh" + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::ggnn::build_param& param) +{ + param.k = conf.at("k"); + + if (conf.contains("k_build")) { param.k_build = conf.at("k_build"); } + if (conf.contains("segment_size")) { param.segment_size = conf.at("segment_size"); } + if (conf.contains("num_layers")) { param.num_layers = conf.at("num_layers"); } + if (conf.contains("tau")) { param.tau = conf.at("tau"); } + if (conf.contains("refine_iterations")) { + param.refine_iterations = conf.at("refine_iterations"); + } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::ggnn::search_param& param) +{ + param.tau = conf.at("tau"); + + if (conf.contains("block_dim")) { param.block_dim = conf.at("block_dim"); } + if (conf.contains("max_iterations")) { param.max_iterations = conf.at("max_iterations"); } + if (conf.contains("cache_size")) { param.cache_size = conf.at("cache_size"); } + if (conf.contains("sorted_size")) { param.sorted_size = conf.at("sorted_size"); } +} + +template class Algo> +auto make_algo(cuvs::bench::Metric metric, int dim, const nlohmann::json& conf) + -> std::unique_ptr> +{ + typename Algo::build_param param; + parse_build_param(conf, param); + return std::make_unique>(metric, dim, param); +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "ggnn") { a = make_algo(metric, dim, conf); } + } + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if constexpr (std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "ggnn") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + } + // else + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +} // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh new file mode 100644 index 0000000000..b1c6c7b26d --- /dev/null +++ b/cpp/bench/ann/src/ggnn/ggnn_wrapper.cuh @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/util.hpp" + +// #include + +#include + +#include +#include + +namespace cuvs::bench { + +template +class ggnn_impl; + +template +class ggnn : public algo, public algo_gpu { + public: + struct build_param { + int k_build{24}; // KBuild + int segment_size{32}; // S + int num_layers{4}; // L + float tau{0.5}; + int refine_iterations{2}; + int k; // GGNN requires to know k during building + }; + + using search_param_base = typename algo::search_param; + struct search_param : public search_param_base { + float tau; + int block_dim{32}; + int max_iterations{400}; + int cache_size{512}; + int sorted_size{256}; + [[nodiscard]] auto needs_dataset() const -> bool override { return true; } + }; + + ggnn(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) override { impl_->build(dataset, nrow); } + + void set_search_param(const search_param_base& param, const void* filter_bitset) override + { + impl_->set_search_param(param, filter_bitset); + } + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override + { + impl_->search(queries, batch_size, k, neighbors, distances); + } + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override + { + return dynamic_cast(impl_.get())->get_sync_stream(); + } + + void save(const std::string& file) const override { impl_->save(file); } + void load(const std::string& file) override { impl_->load(file); } + std::unique_ptr> copy() override { return std::make_unique>(*this); }; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + return impl_->get_preference(); + } + + void set_search_dataset(const T* dataset, size_t nrow) override + { + impl_->set_search_dataset(dataset, nrow); + }; + + private: + std::shared_ptr> impl_; +}; + +template +ggnn::ggnn(Metric metric, int dim, const build_param& param) : algo(metric, dim) +{ + // ggnn/src/sift1m.cu + if (metric == Metric::kEuclidean && dim == 128 && param.k_build == 24 && param.k == 10 && + param.segment_size == 32) { + impl_ = std::make_shared>(metric, dim, param); + } + // ggnn/src/deep1b_multi_gpu.cu, and adapt it deep1B + else if (metric == Metric::kEuclidean && dim == 96 && param.k_build == 24 && param.k == 10 && + param.segment_size == 32) { + impl_ = std::make_shared>(metric, dim, param); + } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 24 && param.k == 10 && + param.segment_size == 32) { + impl_ = std::make_shared>(metric, dim, param); + } else if (metric == Metric::kInnerProduct && dim == 96 && param.k_build == 96 && param.k == 10 && + param.segment_size == 64) { + impl_ = std::make_shared>(metric, dim, param); + } + // ggnn/src/glove200.cu, adapt it to glove100 + else if (metric == Metric::kInnerProduct && dim == 100 && param.k_build == 96 && param.k == 10 && + param.segment_size == 64) { + impl_ = std::make_shared>(metric, dim, param); + } else { + throw std::runtime_error( + "ggnn: not supported combination of metric, dim and build param; " + "see Ggnn's constructor in ggnn_wrapper.cuh for available combinations"); + } +} + +template +class ggnn_impl : public algo, public algo_gpu { + public: + using search_param_base = typename algo::search_param; + + ggnn_impl(Metric metric, int dim, const typename ggnn::build_param& param); + + void build(const T* dataset, size_t nrow) override; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + void search(const T* queries, + int batch_size, + int k, + algo_base::index_type* neighbors, + float* distances) const override; + [[nodiscard]] auto get_sync_stream() const noexcept -> cudaStream_t override { return stream_; } + + void save(const std::string& file) const override; + void load(const std::string& file) override; + std::unique_ptr> copy() override + { + auto r = std::make_unique>(*this); + // set the thread-local stream to the copied handle. + r->stream_ = cuvs::bench::get_stream_from_global_pool(); + return r; + }; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kDevice; + property.query_memory_type = MemoryType::kDevice; + return property; + } + + void set_search_dataset(const T* dataset, size_t nrow) override; + + private: + using algo::metric_; + using algo::dim_; + + using ggnngpu_instance = GGNNGPUInstance; + std::shared_ptr ggnn_; + typename ggnn::build_param build_param_; + typename ggnn::search_param search_param_; + cudaStream_t stream_; + const T* base_dataset_ = nullptr; + size_t base_n_rows_ = 0; + std::optional graph_file_ = std::nullopt; + + void load_impl() + { + if (base_dataset_ == nullptr) { return; } + if (base_n_rows_ == 0) { return; } + int device; + cudaGetDevice(&device); + ggnn_ = std::make_shared( + device, base_n_rows_, build_param_.num_layers, true, build_param_.tau); + ggnn_->set_base_data(base_dataset_); + ggnn_->set_stream(get_sync_stream()); + if (graph_file_.has_value()) { + auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); + auto& ggnn_device = ggnn_->ggnn_shards.at(0); + ggnn_->set_stream(get_sync_stream()); + + ggnn_host.load(graph_file_.value()); + ggnn_host.uploadAsync(ggnn_device); + cudaStreamSynchronize(ggnn_device.stream); + } + } +}; + +template +ggnn_impl::ggnn_impl(Metric metric, + int dim, + const typename ggnn::build_param& param) + : algo(metric, dim), build_param_(param), stream_(cuvs::bench::get_stream_from_global_pool()) +{ + if (metric_ == Metric::kInnerProduct) { + if (measure != Cosine) { throw std::runtime_error("mis-matched metric"); } + } else if (metric_ == Metric::kEuclidean) { + if (measure != Euclidean) { throw std::runtime_error("mis-matched metric"); } + } else { + throw std::runtime_error( + "ggnn supports only metric type of InnerProduct, Cosine and Euclidean"); + } + + if (dim != D) { throw std::runtime_error("mis-matched dim"); } +} + +template +void ggnn_impl::build(const T* dataset, size_t nrow) +{ + base_dataset_ = dataset; + base_n_rows_ = nrow; + graph_file_ = std::nullopt; + load_impl(); + ggnn_->build(0); + for (int i = 0; i < build_param_.refine_iterations; ++i) { + ggnn_->refine(); + } +} + +template +void ggnn_impl::set_search_dataset(const T* dataset, size_t nrow) +{ + if (base_dataset_ != dataset || base_n_rows_ != nrow) { + base_dataset_ = dataset; + base_n_rows_ = nrow; + load_impl(); + } +} + +template +void ggnn_impl::set_search_param(const search_param_base& param, + const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + search_param_ = dynamic_cast::search_param&>(param); +} + +template +void ggnn_impl::search( + const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const +{ + static_assert(sizeof(size_t) == sizeof(int64_t), "sizes of size_t and GGNN's KeyT are different"); + if (k != KQuery) { + throw std::runtime_error( + "k = " + std::to_string(k) + + ", but this GGNN instance only supports k = " + std::to_string(KQuery)); + } + + ggnn_->set_stream(get_sync_stream()); + cudaMemcpyToSymbol(c_tau_query, &search_param_.tau, sizeof(float)); + + const int block_dim = search_param_.block_dim; + const int max_iterations = search_param_.max_iterations; + const int cache_size = search_param_.cache_size; + const int sorted_size = search_param_.sorted_size; + // default value + if (block_dim == 32 && max_iterations == 400 && cache_size == 512 && sorted_size == 256) { + ggnn_->template queryLayer<32, 400, 512, 256, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } + // ggnn/src/sift1m.cu + else if (block_dim == 32 && max_iterations == 200 && cache_size == 256 && sorted_size == 64) { + ggnn_->template queryLayer<32, 200, 256, 64, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } + // ggnn/src/sift1m.cu + else if (block_dim == 32 && max_iterations == 400 && cache_size == 448 && sorted_size == 64) { + ggnn_->template queryLayer<32, 400, 448, 64, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } + // ggnn/src/glove200.cu + else if (block_dim == 128 && max_iterations == 2000 && cache_size == 2048 && sorted_size == 32) { + ggnn_->template queryLayer<128, 2000, 2048, 32, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } + // for glove100 + else if (block_dim == 64 && max_iterations == 400 && cache_size == 512 && sorted_size == 32) { + ggnn_->template queryLayer<64, 400, 512, 32, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } else if (block_dim == 128 && max_iterations == 2000 && cache_size == 1024 && + sorted_size == 32) { + ggnn_->template queryLayer<128, 2000, 1024, 32, false>( + queries, batch_size, reinterpret_cast(neighbors), distances); + } else { + throw std::runtime_error("ggnn: not supported search param"); + } +} + +template +void ggnn_impl::save(const std::string& file) const +{ + auto& ggnn_host = ggnn_->ggnn_cpu_buffers.at(0); + auto& ggnn_device = ggnn_->ggnn_shards.at(0); + ggnn_->set_stream(get_sync_stream()); + + ggnn_host.downloadAsync(ggnn_device); + cudaStreamSynchronize(ggnn_device.stream); + ggnn_host.store(file); +} + +template +void ggnn_impl::load(const std::string& file) +{ + if (!graph_file_.has_value() || graph_file_.value() != file) { + graph_file_ = file; + load_impl(); + } +} + +} // namespace cuvs::bench diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp new file mode 100644 index 0000000000..90dc4bf5cc --- /dev/null +++ b/cpp/bench/ann/src/hnswlib/hnswlib_benchmark.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "../common/ann_types.hpp" +#include "hnswlib_wrapper.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +void parse_build_param(const nlohmann::json& conf, + typename cuvs::bench::hnsw_lib::build_param& param) +{ + param.ef_construction = conf.at("efConstruction"); + param.m = conf.at("M"); + if (conf.contains("num_threads")) { param.num_threads = conf.at("num_threads"); } +} + +template +void parse_search_param(const nlohmann::json& conf, + typename cuvs::bench::hnsw_lib::search_param& param) +{ + param.ef = conf.at("ef"); + if (conf.contains("num_threads")) { param.num_threads = conf.at("num_threads"); } +} + +template class Algo> +auto make_algo(cuvs::bench::Metric metric, int dim, const nlohmann::json& conf) + -> std::unique_ptr> +{ + typename Algo::build_param param; + parse_build_param(conf, param); + return std::make_unique>(metric, dim, param); +} + +template +auto create_algo(const std::string& algo_name, + const std::string& distance, + int dim, + const nlohmann::json& conf) -> std::unique_ptr> +{ + cuvs::bench::Metric metric = parse_metric(distance); + std::unique_ptr> a; + + if constexpr (std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v) { + if (algo_name == "hnswlib") { a = make_algo(metric, dim, conf); } + } + + if (!a) { throw std::runtime_error("invalid algo: '" + algo_name + "'"); } + return a; +} + +template +auto create_search_param(const std::string& algo_name, const nlohmann::json& conf) + -> std::unique_ptr::search_param> +{ + if (algo_name == "hnswlib") { + auto param = std::make_unique::search_param>(); + parse_search_param(conf, *param); + return param; + } + // else + throw std::runtime_error("invalid algo: '" + algo_name + "'"); +} + +}; // namespace cuvs::bench + +REGISTER_ALGO_INSTANCE(float); +REGISTER_ALGO_INSTANCE(half); +REGISTER_ALGO_INSTANCE(std::int8_t); +REGISTER_ALGO_INSTANCE(std::uint8_t); + +#ifdef ANN_BENCH_BUILD_MAIN +#include "../common/benchmark.hpp" +int main(int argc, char** argv) { return cuvs::bench::run_main(argc, argv); } +#endif diff --git a/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h new file mode 100644 index 0000000000..0170e3351c --- /dev/null +++ b/cpp/bench/ann/src/hnswlib/hnswlib_wrapper.h @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "../common/ann_types.hpp" +#include "../common/thread_pool.hpp" +#include "../common/util.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::bench { + +template +struct hnsw_dist_t { + using type = void; +}; + +template <> +struct hnsw_dist_t { + using type = float; +}; + +template <> +struct hnsw_dist_t { + using type = float; +}; + +template <> +struct hnsw_dist_t { + using type = int; +}; + +template <> +struct hnsw_dist_t { + using type = int; +}; + +template +class hnsw_lib : public algo { + public: + // https://github.com/nmslib/hnswlib/blob/master/ALGO_PARAMS.md + struct build_param { + int m; + int ef_construction; + int num_threads = omp_get_num_procs(); + }; + + using search_param_base = typename algo::search_param; + struct search_param : public search_param_base { + int ef; + int num_threads = 1; + }; + + hnsw_lib(Metric metric, int dim, const build_param& param); + + void build(const T* dataset, size_t nrow) override; + + void set_search_param(const search_param_base& param, const void* filter_bitset) override; + void search(const T* query, + int batch_size, + int k, + algo_base::index_type* indices, + float* distances) const override; + + void save(const std::string& path_to_index) const override; + void load(const std::string& path_to_index) override; + auto copy() -> std::unique_ptr> override { return std::make_unique>(*this); }; + + [[nodiscard]] auto get_preference() const -> algo_property override + { + algo_property property; + property.dataset_memory_type = MemoryType::kHost; + property.query_memory_type = MemoryType::kHost; + return property; + } + + void set_base_layer_only() { appr_alg_->base_layer_only = true; } + + private: + void get_search_knn_results(const T* query, + int k, + algo_base::index_type* indices, + float* distances) const; + + std::shared_ptr::type>> appr_alg_; + std::shared_ptr::type>> space_; + + using algo::metric_; + using algo::dim_; + int ef_construction_; + int m_; + int num_threads_; + std::shared_ptr thread_pool_; + Mode bench_mode_; +}; + +template +hnsw_lib::hnsw_lib(Metric metric, int dim, const build_param& param) : algo(metric, dim) +{ + assert(dim_ > 0); + static_assert(std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v, + "Only float, half, uint8, and int8 are supported"); + + ef_construction_ = param.ef_construction; + m_ = param.m; + num_threads_ = param.num_threads; +} + +template +void hnsw_lib::build(const T* dataset, size_t nrow) +{ + static_assert(std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v, + "Only float, half, uint8, and int8 are supported"); + if (metric_ == Metric::kInnerProduct) { + space_ = std::make_shared::type>>(dim_); + } else { + if constexpr (std::is_same_v || std::is_same_v) { + space_ = std::make_shared::type>>(dim_); + } else { + space_ = std::make_shared>(dim_); + } + } + + appr_alg_ = std::make_shared::type>>( + space_.get(), nrow, m_, ef_construction_); + + thread_pool_ = std::make_shared(num_threads_); + const size_t items_per_thread = nrow / (num_threads_ + 1); + + thread_pool_->submit( + [&](size_t i) { + if (i < items_per_thread && i % 10000 == 0) { + char buf[20]; + std::time_t now = std::time(nullptr); + std::strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", std::localtime(&now)); + printf("%s building %zu / %zu\n", buf, i, items_per_thread); + fflush(stdout); + } + + appr_alg_->addPoint(dataset + i * dim_, i); + }, + nrow); +} + +template +void hnsw_lib::set_search_param(const search_param_base& param_, const void* filter_bitset) +{ + if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } + auto param = dynamic_cast(param_); + appr_alg_->ef_ = param.ef; + num_threads_ = param.num_threads; + // bench_mode_ = param.metric_objective; + bench_mode_ = Mode::kLatency; // TODO(achirkin): pass the benchmark mode in the algo parameters + + // Create a pool if multiple query threads have been set and the pool hasn't been created already + bool create_pool = (bench_mode_ == Mode::kLatency && num_threads_ > 1 && !thread_pool_); + if (create_pool) { thread_pool_ = std::make_shared(num_threads_); } +} + +template +void hnsw_lib::search( + const T* query, int batch_size, int k, algo_base::index_type* indices, float* distances) const +{ + auto f = [&](int i) { + // hnsw can only handle a single vector at a time. + get_search_knn_results(query + i * dim_, k, indices + i * k, distances + i * k); + }; + if (bench_mode_ == Mode::kLatency && num_threads_ > 1) { + thread_pool_->submit(f, batch_size); + } else { + for (int i = 0; i < batch_size; i++) { + f(i); + } + } +} + +template +void hnsw_lib::save(const std::string& path_to_index) const +{ + appr_alg_->saveIndex(std::string(path_to_index)); +} + +template +void hnsw_lib::load(const std::string& path_to_index) +{ + if (metric_ == Metric::kInnerProduct) { + space_ = std::make_shared::type>>(dim_); + } else { + if constexpr (std::is_same_v || std::is_same_v) { + space_ = std::make_shared::type>>(dim_); + } else { + space_ = std::make_shared>(dim_); + } + } + + appr_alg_ = std::make_shared::type>>( + space_.get(), path_to_index); +} + +template +void hnsw_lib::get_search_knn_results(const T* query, + int k, + algo_base::index_type* indices, + float* distances) const +{ + auto result = appr_alg_->searchKnn(query, k); + assert(result.size() >= static_cast(k)); + + for (int i = k - 1; i >= 0; --i) { + indices[i] = result.top().second; + distances[i] = result.top().first; + result.pop(); + } +} + +}; // namespace cuvs::bench diff --git a/cpp/cmake/config.json b/cpp/cmake/config.json index f7cc50e513..a9f1b53007 100644 --- a/cpp/cmake/config.json +++ b/cpp/cmake/config.json @@ -1,22 +1,15 @@ { "parse": { "additional_commands": { - "CPMFindPackage": { + "ConfigureTest": { + "flags": ["NOCUDA", "FETCH_CODEBOOKS"], "kwargs": { "NAME": 1, - "GITHUB_REPOSITORY": "?", - "GIT_TAG": "?", - "VERSION": "?", - "GIT_SHALLOW": "?", - "OPTIONS": "*", - "FIND_PACKAGE_ARGUMENTS": "*" + "GPUS": "?", + "PERCENT": "?", + "ADDITIONAL_DEP": "?", + "PATH": "*" } - }, - "ConfigureTest": { - "flags": ["TEST_NAME", "TEST_SRC"] - }, - "ConfigureBench": { - "flags": ["BENCH_NAME", "BENCH_SRC"] } } }, diff --git a/cpp/cmake/modules/ConfigureCUDA.cmake b/cpp/cmake/modules/ConfigureCUDA.cmake index ea8a077b0c..d8576cd650 100644 --- a/cpp/cmake/modules/ConfigureCUDA.cmake +++ b/cpp/cmake/modules/ConfigureCUDA.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2018-2023, NVIDIA CORPORATION. +# Copyright (c) 2018-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,45 +13,62 @@ # ============================================================================= if(DISABLE_DEPRECATION_WARNINGS) - list(APPEND RAFT_CXX_FLAGS -Wno-deprecated-declarations) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations) + list(APPEND CUVS_CXX_FLAGS -Wno-deprecated-declarations -DRAFT_HIDE_DEPRECATION_WARNINGS) + list(APPEND CUVS_CUDA_FLAGS -Xcompiler=-Wno-deprecated-declarations + -DRAFT_HIDE_DEPRECATION_WARNINGS + ) +endif() + +if(DISABLE_OPENMP) + list(APPEND CUVS_CXX_FLAGS -Wno-unknown-pragmas) + list(APPEND CUVS_CUDA_FLAGS -Xcompiler=-Wno-unknown-pragmas) endif() # Be very strict when compiling with GCC as host compiler (and thus more lenient when compiling with # clang) if(CMAKE_COMPILER_IS_GNUCXX) - list(APPEND RAFT_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations) + list(APPEND CUVS_CXX_FLAGS -Wall -Werror -Wno-unknown-pragmas -Wno-error=deprecated-declarations + -Wno-reorder + ) + list(APPEND CUVS_CUDA_FLAGS + -Xcompiler=-Wall,-Werror,-Wno-error=deprecated-declarations,-Wno-reorder + ) # set warnings as errors if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 11.2.0) - list(APPEND RAFT_CUDA_FLAGS -Werror=all-warnings) + list(APPEND CUVS_CUDA_FLAGS -Werror=all-warnings) endif() endif() +# Allow invalid CUDA kernels in the short term +if(CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8.0) + list(APPEND CUVS_CUDA_FLAGS -static-global-template-stub=false) +endif() + if(CUDA_LOG_COMPILE_TIME) - list(APPEND RAFT_CUDA_FLAGS "--time=nvcc_compile_log.csv") + list(APPEND CUVS_CUDA_FLAGS "--time=nvcc_compile_log.csv") endif() -list(APPEND RAFT_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) -list(APPEND RAFT_CXX_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM") -list(APPEND RAFT_CUDA_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM") +list(APPEND CUVS_CUDA_FLAGS --expt-extended-lambda --expt-relaxed-constexpr) +list(APPEND CUVS_CXX_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM") +list(APPEND CUVS_CUDA_FLAGS "-DCUDA_API_PER_THREAD_DEFAULT_STREAM") # make sure we produce smallest binary size -list(APPEND RAFT_CUDA_FLAGS -Xfatbin=-compress-all) +include(${rapids-cmake-dir}/cuda/enable_fatbin_compression.cmake) +rapids_cuda_enable_fatbin_compression(VARIABLE CUVS_CUDA_FLAGS TUNE_FOR rapids) # Option to enable line info in CUDA device compilation to allow introspection when profiling / # memchecking if(CUDA_ENABLE_LINEINFO) - list(APPEND RAFT_CUDA_FLAGS -lineinfo) + list(APPEND CUVS_CUDA_FLAGS -lineinfo) endif() if(OpenMP_FOUND) - list(APPEND RAFT_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) + list(APPEND CUVS_CUDA_FLAGS -Xcompiler=${OpenMP_CXX_FLAGS}) endif() # Debug options if(CMAKE_BUILD_TYPE MATCHES Debug) - message(VERBOSE "RAFT: Building with debugging flags") - list(APPEND RAFT_CUDA_FLAGS -G -Xcompiler=-rdynamic) - list(APPEND RAFT_CUDA_FLAGS -Xptxas --suppress-stack-size-warning) + message(VERBOSE "cuVS: Building with debugging flags") + list(APPEND CUVS_CUDA_FLAGS -G -Xcompiler=-rdynamic --maxrregcount=64) + list(APPEND CUVS_CUDA_FLAGS -Xptxas --suppress-stack-size-warning) endif() diff --git a/cpp/cmake/modules/FindSVE.cmake b/cpp/cmake/modules/FindSVE.cmake new file mode 100644 index 0000000000..6533c7f069 --- /dev/null +++ b/cpp/cmake/modules/FindSVE.cmake @@ -0,0 +1,45 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# ============================================================================= +# FindSVE.cmake This module finds the SVE (Scalable Vector Extension) support in the compiler. +# ============================================================================= + +INCLUDE(CheckCXXSourceRuns) + +set(SVE_CODE + " + #include + + int main() + { + svfloat32_t a = svdup_f32(0); + return 0; + } +" +) + +# Check for SVE support +message("Checking for SVE support") +set(CMAKE_REQUIRED_FLAGS "-march=armv9-a+sve") +check_cxx_source_runs("${SVE_CODE}" CXX_HAS_SVE) + +if(CXX_HAS_SVE) + set(CXX_SVE_FOUND + TRUE + CACHE BOOL "SVE support found" + ) + set(CXX_SVE_FLAGS + "-march=armv9-a+sve" + CACHE STRING "Flags for SVE support" + ) +else() + set(CXX_SVE_FOUND + FALSE + CACHE BOOL "SVE support not found" + ) + set(CXX_SVE_FLAGS + "" + CACHE STRING "Flags for SVE support" + ) +endif() + +mark_as_advanced(CXX_SVE_FOUND CXX_SVE_FLAGS) diff --git a/cpp/cmake/patches/cutlass/build-export.patch b/cpp/cmake/patches/cutlass/build-export.patch new file mode 100644 index 0000000000..31bbd25102 --- /dev/null +++ b/cpp/cmake/patches/cutlass/build-export.patch @@ -0,0 +1,26 @@ +From e0a9597946257a01ae8444200f836ee51d5597ba Mon Sep 17 00:00:00 2001 +From: Kyle Edwards +Date: Wed, 20 Nov 2024 16:37:38 -0500 +Subject: [PATCH] Remove erroneous include directories + +These directories are left over from when CuTe was a separate +CMake project. Remove them. +--- + CMakeLists.txt | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 7419bdf5e..545384d82 100755 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -665,8 +665,6 @@ target_include_directories( + $ + $ + $ +- $ +- $ + ) + + # Mark CTK headers as system to supress warnings from them +-- +2.34.1 diff --git a/cpp/cmake/patches/cutlass_override.json b/cpp/cmake/patches/cutlass_override.json new file mode 100644 index 0000000000..7bf818987f --- /dev/null +++ b/cpp/cmake/patches/cutlass_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "cutlass" : { + "version": "3.5.1", + "git_url": "https://github.com/NVIDIA/cutlass.git", + "git_tag": "v${version}", + "patches" : [ + { + "file" : "${current_json_dir}/cutlass/build-export.patch", + "issue" : "Fix build directory export", + "fixed_in" : "" + } + ] + } + } +} diff --git a/cpp/cmake/patches/diskann.diff b/cpp/cmake/patches/diskann.diff new file mode 100644 index 0000000000..ee0c2cb165 --- /dev/null +++ b/cpp/cmake/patches/diskann.diff @@ -0,0 +1,250 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3d3d2b8..a007fa4 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -23,10 +23,6 @@ set(CMAKE_STANDARD 17) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + +-if(NOT MSVC) +- set(CMAKE_CXX_COMPILER g++) +-endif() +- + set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}") + + # Install nuget packages for dependencies. +@@ -145,62 +141,14 @@ if (MSVC) + "${DISKANN_MKL_LIB_PATH}/mkl_intel_thread.lib") + else() + # expected path for manual intel mkl installs +- set(POSSIBLE_OMP_PATHS "/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin/libiomp5.so;/usr/lib/x86_64-linux-gnu/libiomp5.so;/opt/intel/lib/intel64_lin/libiomp5.so") +- foreach(POSSIBLE_OMP_PATH ${POSSIBLE_OMP_PATHS}) +- if (EXISTS ${POSSIBLE_OMP_PATH}) +- get_filename_component(OMP_PATH ${POSSIBLE_OMP_PATH} DIRECTORY) +- endif() +- endforeach() +- +- if(NOT OMP_PATH) +- message(FATAL_ERROR "Could not find Intel OMP in standard locations; use -DOMP_PATH to specify the install location for your environment") +- endif() +- link_directories(${OMP_PATH}) +- +- set(POSSIBLE_MKL_LIB_PATHS "/opt/intel/oneapi/mkl/latest/lib/intel64/libmkl_core.so;/usr/lib/x86_64-linux-gnu/libmkl_core.so;/opt/intel/mkl/lib/intel64/libmkl_core.so") +- foreach(POSSIBLE_MKL_LIB_PATH ${POSSIBLE_MKL_LIB_PATHS}) +- if (EXISTS ${POSSIBLE_MKL_LIB_PATH}) +- get_filename_component(MKL_PATH ${POSSIBLE_MKL_LIB_PATH} DIRECTORY) +- endif() +- endforeach() +- +- set(POSSIBLE_MKL_INCLUDE_PATHS "/opt/intel/oneapi/mkl/latest/include;/usr/include/mkl;/opt/intel/mkl/include/;") +- foreach(POSSIBLE_MKL_INCLUDE_PATH ${POSSIBLE_MKL_INCLUDE_PATHS}) +- if (EXISTS ${POSSIBLE_MKL_INCLUDE_PATH}) +- set(MKL_INCLUDE_PATH ${POSSIBLE_MKL_INCLUDE_PATH}) +- endif() +- endforeach() +- if(NOT MKL_PATH) +- message(FATAL_ERROR "Could not find Intel MKL in standard locations; use -DMKL_PATH to specify the install location for your environment") +- elseif(NOT MKL_INCLUDE_PATH) +- message(FATAL_ERROR "Could not find Intel MKL in standard locations; use -DMKL_INCLUDE_PATH to specify the install location for headers for your environment") +- endif() +- if (EXISTS ${MKL_PATH}/libmkl_def.so.2) +- set(MKL_DEF_SO ${MKL_PATH}/libmkl_def.so.2) +- elseif(EXISTS ${MKL_PATH}/libmkl_def.so) +- set(MKL_DEF_SO ${MKL_PATH}/libmkl_def.so) +- else() +- message(FATAL_ERROR "Despite finding MKL, libmkl_def.so was not found in expected locations.") +- endif() +- link_directories(${MKL_PATH}) +- include_directories(${MKL_INCLUDE_PATH}) ++ find_package(MKL CONFIG REQUIRED) ++ include_directories($) ++ link_libraries($) + + # compile flags and link libraries + add_compile_options(-m64 -Wl,--no-as-needed) + if (NOT PYBIND) + link_libraries(mkl_intel_ilp64 mkl_intel_thread mkl_core iomp5 pthread m dl) +- else() +- # static linking for python so as to minimize customer dependency issues +- link_libraries( +- ${MKL_PATH}/libmkl_intel_ilp64.a +- ${MKL_PATH}/libmkl_intel_thread.a +- ${MKL_PATH}/libmkl_core.a +- ${MKL_DEF_SO} +- iomp5 +- pthread +- m +- dl +- ) + endif() + endif() + +@@ -286,7 +234,7 @@ if(MSVC) + set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY_RELEASE ${PROJECT_SOURCE_DIR}/x64/Release) + else() + set(ENV{TCMALLOC_LARGE_ALLOC_REPORT_THRESHOLD} 500000000000) +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mfma -msse2 -ftree-vectorize -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors -DUSE_AVX2") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mfma -msse2 -ftree-vectorize -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free -fopenmp -fopenmp-simd -funroll-loops -Wfatal-errors -DUSE_AVX2 -fno-finite-math-only -laio") + set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -g -DDEBUG") + if (NOT PYBIND) + set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -Ofast") +@@ -300,10 +248,6 @@ else() + endif() + + add_subdirectory(src) +-if (NOT PYBIND) +- add_subdirectory(apps) +- add_subdirectory(apps/utils) +-endif() + + if (UNIT_TEST) + enable_testing() +diff --git a/include/ann_exception.h b/include/ann_exception.h +index 6b81373..a385198 100644 +--- a/include/ann_exception.h ++++ b/include/ann_exception.h +@@ -2,6 +2,7 @@ + // Licensed under the MIT license. + + #pragma once ++#include + #include + #include + #include +diff --git a/include/distance.h b/include/distance.h +index f3b1de2..d4da72e 100644 +--- a/include/distance.h ++++ b/include/distance.h +@@ -77,6 +77,7 @@ class DistanceCosineInt8 : public Distance + DistanceCosineInt8() : Distance(diskann::Metric::COSINE) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const int8_t *a, const int8_t *b, uint32_t length) const; + }; + +@@ -86,6 +87,7 @@ class DistanceL2Int8 : public Distance + DistanceL2Int8() : Distance(diskann::Metric::L2) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const int8_t *a, const int8_t *b, uint32_t size) const; + }; + +@@ -96,6 +98,7 @@ class AVXDistanceL2Int8 : public Distance + AVXDistanceL2Int8() : Distance(diskann::Metric::L2) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const int8_t *a, const int8_t *b, uint32_t length) const; + }; + +@@ -105,6 +108,7 @@ class DistanceCosineFloat : public Distance + DistanceCosineFloat() : Distance(diskann::Metric::COSINE) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const float *a, const float *b, uint32_t length) const; + }; + +@@ -114,7 +118,7 @@ class DistanceL2Float : public Distance + DistanceL2Float() : Distance(diskann::Metric::L2) + { + } +- ++ using Distance::compare; + #ifdef _WINDOWS + DISKANN_DLLEXPORT virtual float compare(const float *a, const float *b, uint32_t size) const; + #else +@@ -128,6 +132,7 @@ class AVXDistanceL2Float : public Distance + AVXDistanceL2Float() : Distance(diskann::Metric::L2) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const float *a, const float *b, uint32_t length) const; + }; + +@@ -146,6 +151,7 @@ class SlowDistanceCosineUInt8 : public Distance + SlowDistanceCosineUInt8() : Distance(diskann::Metric::COSINE) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const uint8_t *a, const uint8_t *b, uint32_t length) const; + }; + +@@ -155,6 +161,7 @@ class DistanceL2UInt8 : public Distance + DistanceL2UInt8() : Distance(diskann::Metric::L2) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const uint8_t *a, const uint8_t *b, uint32_t size) const; + }; + +@@ -198,6 +205,7 @@ class AVXDistanceInnerProductFloat : public Distance + AVXDistanceInnerProductFloat() : Distance(diskann::Metric::INNER_PRODUCT) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const float *a, const float *b, uint32_t length) const; + }; + +@@ -213,6 +221,7 @@ class AVXNormalizedCosineDistanceFloat : public Distance + AVXNormalizedCosineDistanceFloat() : Distance(diskann::Metric::COSINE) + { + } ++ using Distance::compare; + DISKANN_DLLEXPORT virtual float compare(const float *a, const float *b, uint32_t length) const + { + // Inner product returns negative values to indicate distance. +diff --git a/include/utils.h b/include/utils.h +index d3af5c3..417af31 100644 +--- a/include/utils.h ++++ b/include/utils.h +@@ -29,6 +29,7 @@ typedef int FileHandle; + #include "types.h" + #include "tag_uint128.h" + #include ++#include + + #ifdef EXEC_ENV_OLS + #include "content_buf.h" +diff --git a/src/index.cpp b/src/index.cpp +index bf93344..9d8336c 100644 +--- a/src/index.cpp ++++ b/src/index.cpp +@@ -17,9 +17,7 @@ + #include "gperftools/malloc_extension.h" + #endif + +-#ifdef _WINDOWS + #include +-#endif + + #include "index.h" + +diff --git a/src/partition.cpp b/src/partition.cpp +index 570d45c..fb54cbf 100644 +--- a/src/partition.cpp ++++ b/src/partition.cpp +@@ -21,9 +21,7 @@ + #include "parameters.h" + #include "memory_mapper.h" + #include "partition.h" +-#ifdef _WINDOWS + #include +-#endif + + // block size for reading/ processing large files and matrices in blocks + #define BLOCK_SIZE 5000000 +diff --git a/src/pq_flash_index.cpp b/src/pq_flash_index.cpp +index d9ad506..145a978 100644 +--- a/src/pq_flash_index.cpp ++++ b/src/pq_flash_index.cpp +@@ -8,6 +8,7 @@ + #include "pq_scratch.h" + #include "pq_flash_index.h" + #include "cosine_similarity.h" ++#include + + #ifdef _WINDOWS + #include "windows_aligned_file_reader.h" diff --git a/cpp/cmake/patches/diskann_override.json b/cpp/cmake/patches/diskann_override.json new file mode 100644 index 0000000000..c83898548f --- /dev/null +++ b/cpp/cmake/patches/diskann_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "diskann" : { + "version": "0.7.0", + "git_url": "https://github.com/microsoft/DiskANN.git", + "git_tag": "main", + "patches" : [ + { + "file" : "${current_json_dir}/diskann.diff", + "issue" : "Correct compilation issues", + "fixed_in" : "" + } + ] + } + } +} diff --git a/cpp/cmake/patches/faiss/fix-missing-includes-for-thrust-copy.patch b/cpp/cmake/patches/faiss/fix-missing-includes-for-thrust-copy.patch new file mode 100644 index 0000000000..9f250b88be --- /dev/null +++ b/cpp/cmake/patches/faiss/fix-missing-includes-for-thrust-copy.patch @@ -0,0 +1,40 @@ +From 87c5331dfa6ec30fd9839ecb89af519e65f010e7 Mon Sep 17 00:00:00 2001 +From: Bradley Dice +Date: Mon, 29 Sep 2025 15:39:17 -0500 +Subject: [PATCH] Fix missing includes for thrust::copy + +--- + faiss/gpu/impl/BinaryCuvsCagra.cu | 3 +++ + faiss/gpu/impl/CuvsCagra.cu | 3 +++ + 2 files changed, 6 insertions(+) + +diff --git a/faiss/gpu/impl/BinaryCuvsCagra.cu b/faiss/gpu/impl/BinaryCuvsCagra.cu +index 0ca21dc5f..b331fdc8f 100644 +--- a/faiss/gpu/impl/BinaryCuvsCagra.cu ++++ b/faiss/gpu/impl/BinaryCuvsCagra.cu +@@ -32,6 +32,9 @@ + #include + #include + ++#include ++#include ++ + namespace faiss { + namespace gpu { + +diff --git a/faiss/gpu/impl/CuvsCagra.cu b/faiss/gpu/impl/CuvsCagra.cu +index 9ac4e1c5a..755817f43 100644 +--- a/faiss/gpu/impl/CuvsCagra.cu ++++ b/faiss/gpu/impl/CuvsCagra.cu +@@ -31,6 +31,9 @@ + #include + #include + ++#include ++#include ++ + namespace faiss { + namespace gpu { + +-- +2.49.0 diff --git a/cpp/cmake/patches/faiss_override.json b/cpp/cmake/patches/faiss_override.json new file mode 100644 index 0000000000..5af8bd8543 --- /dev/null +++ b/cpp/cmake/patches/faiss_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "faiss" : { + "version": "1.12.0", + "git_url": "https://github.com/facebookresearch/faiss.git", + "git_tag": "v1.12.0", + "patches" : [ + { + "file" : "${current_json_dir}/faiss/fix-missing-includes-for-thrust-copy.patch", + "issue" : "Fix missing Thrust includes. https://github.com/facebookresearch/faiss/pull/4597", + "fixed_in" : "" + } + ] + } + } +} diff --git a/cpp/cmake/patches/ggnn.diff b/cpp/cmake/patches/ggnn.diff new file mode 100644 index 0000000000..c2413110b6 --- /dev/null +++ b/cpp/cmake/patches/ggnn.diff @@ -0,0 +1,229 @@ +--- a/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh ++++ b/include/ggnn/cache/cuda_simple_knn_sym_cache.cuh +@@ -62,7 +62,7 @@ struct SimpleKNNSymCache { + const ValueT dist_half) + : dist_query(dist_query), dist_half(dist_half) {} + +- __device__ __forceinline__ DistQueryAndHalf() {} ++ DistQueryAndHalf() = default; + }; + + struct DistanceAndNorm { +@@ -98,8 +98,7 @@ struct SimpleKNNSymCache { + KeyT cache; + DistQueryAndHalf dist; + bool flag; +- +- __device__ __forceinline__ SyncTempStorage() {} ++ SyncTempStorage() = default; + }; + + public: +diff --git a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh +index 8cbaf0d..6eb72ac 100644 +--- a/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh ++++ b/include/ggnn/cuda_knn_ggnn_gpu_instance.cuh +@@ -41,7 +41,6 @@ limitations under the License. + #include "ggnn/sym/cuda_knn_sym_query_layer.cuh" + #include "ggnn/utils/cuda_knn_utils.cuh" + #include "ggnn/utils/cuda_knn_constants.cuh" +-#include "ggnn/utils/cuda_knn_dataset.cuh" + + template + __global__ void divide(ValueT* res, ValueT* input, ValueT N) { +@@ -98,9 +97,7 @@ struct GGNNGPUInstance { + typedef GGNNGraphDevice GGNNGraphDevice; + typedef GGNNGraphHost GGNNGraphHost; + +- const Dataset* dataset; + GGNNGraphBuffer* ggnn_buffer {nullptr}; +- GGNNQuery ggnn_query; + + // Graph Shards resident on the GPU + std::vector ggnn_shards; +@@ -117,13 +114,12 @@ struct GGNNGPUInstance { + // number of shards that need to be processed by this instance + const int num_parts; + +- GGNNGPUInstance(const int gpu_id, const Dataset* dataset, ++ GGNNGPUInstance(const int gpu_id, + const int N_shard, const int L, + const bool enable_construction, const float tau_build, + const int num_parts=1, const int num_cpu_buffers=1) : + N_shard{N_shard}, L{L}, tau_build{tau_build}, +- dataset{dataset}, gpu_id{gpu_id}, +- ggnn_query{dataset->N_query, D, KQuery, num_parts}, ++ gpu_id{gpu_id}, + num_parts{num_parts} + { + CHECK_LE(L, MAX_LAYER); +@@ -135,7 +131,6 @@ struct GGNNGPUInstance { + CHECK_EQ(current_gpu_id, gpu_id) << "cudaSetDevice() needs to be called in advance!"; + } + +- ggnn_query.loadQueriesAsync(dataset->h_query, 0); + + computeGraphParameters(); + +@@ -186,7 +181,7 @@ struct GGNNGPUInstance { + } + + GGNNGPUInstance(const GGNNGPUInstance& other) +- : dataset{nullptr}, ggnn_query{0, D, KQuery}, ++ : + gpu_id{0}, N_shard{0}, num_parts{0} { + // this exists to allow using vector::emplace_back + // when it triggers a reallocation, this code will be called. +@@ -305,6 +300,7 @@ struct GGNNGPUInstance { + + // io + ++ /* + void waitForDiskIO(const int shard_id) { + auto& cpu_buffer = ggnn_cpu_buffers[shard_id%ggnn_cpu_buffers.size()]; + if (cpu_buffer.disk_io_thread.joinable()) +@@ -468,11 +464,12 @@ struct GGNNGPUInstance { + CHECK_CUDA(cudaDeviceSynchronize()); + CHECK_CUDA(cudaPeekAtLastError()); + } ++ */ + + // graph operations + + template +- void queryLayer(const int shard_id = 0) const { ++ void queryLayer(const BaseT* d_query, int batch_size, KeyT* d_query_result_ids, ValueT* d_query_result_dists, const int shard_id = 0) const { + CHECK_CUDA(cudaSetDevice(gpu_id)); + const auto& shard = ggnn_shards.at(shard_id%ggnn_shards.size()); + +@@ -482,21 +479,21 @@ struct GGNNGPUInstance { + + int* m_dist_statistics = nullptr; + if (DIST_STATS) +- cudaMallocManaged(&m_dist_statistics, dataset->N_query * sizeof(int)); ++ cudaMallocManaged(&m_dist_statistics, batch_size * sizeof(int)); + + QueryKernel query_kernel; + query_kernel.d_base = shard.d_base; +- query_kernel.d_query = ggnn_query.d_query; ++ query_kernel.d_query = d_query; + + query_kernel.d_graph = shard.d_graph; +- query_kernel.d_query_results = ggnn_query.d_query_result_ids; +- query_kernel.d_query_results_dists = ggnn_query.d_query_result_dists; ++ query_kernel.d_query_results = d_query_result_ids; ++ query_kernel.d_query_results_dists = d_query_result_dists; + + query_kernel.d_translation = shard.d_translation; + + query_kernel.d_nn1_stats = shard.d_nn1_stats; + +- query_kernel.N = dataset->N_query; ++ query_kernel.N = batch_size; + query_kernel.N_offset = 0; + + query_kernel.d_dist_stats = m_dist_statistics; +@@ -771,6 +768,16 @@ struct GGNNGPUInstance { + sym(layer, shard_id); + } + } ++ ++ void set_stream(cudaStream_t stream) { ++ assert(ggnn_shards.size() == 1); ++ ggnn_shards.at(0).stream = stream; ++ } ++ ++ void set_base_data(const BaseT* dataset) { ++ assert(ggnn_shards.size() == 1); ++ ggnn_shards.at(0).d_base = dataset; ++ } + }; + + #endif // INCLUDE_GGNN_CUDA_KNN_GGNN_GPU_INSTANCE_CUH_ +diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh +index c94a8f1..781226d 100644 +--- a/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh ++++ b/include/ggnn/graph/cuda_knn_ggnn_graph_device.cuh +@@ -50,7 +50,7 @@ struct GGNNGraphDevice { + ValueT* d_nn1_stats; + + /// base data pointer for the shard. +- BaseT* d_base; ++ const BaseT* d_base; + + /// combined memory pool + char* d_memory; +@@ -69,7 +69,9 @@ struct GGNNGraphDevice { + const size_t selection_translation_size = align8(ST_all * sizeof(KeyT)); + const size_t nn1_stats_size = align8(2 * sizeof(ValueT)); + total_graph_size = graph_size + 2 * selection_translation_size + nn1_stats_size; +- base_size = align8(static_cast(N) * D * sizeof(BaseT)); ++ // base_size = align8(static_cast(N) * D * sizeof(BaseT)); ++ (void) N; ++ (void) D; + + const size_t total_size = base_size+total_graph_size; + +@@ -86,8 +88,7 @@ struct GGNNGraphDevice { + CHECK_CUDA(cudaMalloc(&d_memory, total_size)); + + size_t pos = 0; +- d_base = reinterpret_cast(d_memory+pos); +- pos += base_size; ++ d_base = nullptr; + d_graph = reinterpret_cast(d_memory+pos); + pos += graph_size; + d_translation = reinterpret_cast(d_memory+pos); +@@ -99,14 +100,14 @@ struct GGNNGraphDevice { + + CHECK_EQ(pos, total_size); + +- CHECK_CUDA(cudaStreamCreate(&stream)); ++ // CHECK_CUDA(cudaStreamCreate(&stream)); + + CHECK_CUDA(cudaPeekAtLastError()); + CHECK_CUDA(cudaDeviceSynchronize()); + CHECK_CUDA(cudaPeekAtLastError()); + } + +- GGNNGraphDevice(const GGNNGraphDevice& other) { ++ GGNNGraphDevice(const GGNNGraphDevice&) { + // this exists to allow using vector::emplace_back + // when it triggers a reallocation, this code will be called. + // always make sure that enough memory is reserved ahead of time. +@@ -116,7 +117,7 @@ struct GGNNGraphDevice { + ~GGNNGraphDevice() { + cudaFree(d_memory); + +- CHECK_CUDA(cudaStreamDestroy(stream)); ++ // CHECK_CUDA(cudaStreamDestroy(stream)); + } + }; + +diff --git a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh +index 2055f9e..ef5843a 100644 +--- a/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh ++++ b/include/ggnn/graph/cuda_knn_ggnn_graph_host.cuh +@@ -92,7 +92,7 @@ struct GGNNGraphHost { + CHECK_CUDA(cudaPeekAtLastError()); + } + +- GGNNGraphHost(const GGNNGraphHost& other) { ++ GGNNGraphHost(const GGNNGraphHost&) { + // this exists to allow using vector::emplace_back + // when it triggers a reallocation, this code will be called. + // always make sure that enough memory is reserved ahead of time. +diff --git a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh +index 49d76a1..eef69e6 100644 +--- a/include/ggnn/select/cuda_knn_wrs_select_layer.cuh ++++ b/include/ggnn/select/cuda_knn_wrs_select_layer.cuh +@@ -22,7 +22,6 @@ limitations under the License. + #include + #include + +-#include + #include + + #include "ggnn/utils/cuda_knn_constants.cuh" +-- +2.43.0 diff --git a/cpp/cmake/patches/ggnn_override.json b/cpp/cmake/patches/ggnn_override.json new file mode 100644 index 0000000000..f1bd8c8a4b --- /dev/null +++ b/cpp/cmake/patches/ggnn_override.json @@ -0,0 +1,16 @@ +{ + "packages" : { + "ggnn" : { + "version": "0.5", + "git_url": "https://github.com/cgtuebingen/ggnn.git", + "git_tag": "release_${version}", + "patches" : [ + { + "file" : "${current_json_dir}/ggnn.diff", + "issue" : "Correct compilation issues", + "fixed_in" : "" + } + ] + } + } + } diff --git a/cpp/cmake/patches/hnswlib.diff b/cpp/cmake/patches/hnswlib.diff new file mode 100644 index 0000000000..52ed5ab377 --- /dev/null +++ b/cpp/cmake/patches/hnswlib.diff @@ -0,0 +1,357 @@ +diff --git a/hnswlib/hnswalg.h b/hnswlib/hnswalg.h +index bef0017..0ee7931 100644 +--- a/hnswlib/hnswalg.h ++++ b/hnswlib/hnswalg.h +@@ -16,6 +16,9 @@ typedef unsigned int linklistsizeint; + template + class HierarchicalNSW : public AlgorithmInterface { + public: ++ bool base_layer_only = false; ++ int num_seeds = 32; ++ bool base_layer_init = true; + static const tableint MAX_LABEL_OPERATION_LOCKS = 65536; + static const unsigned char DELETE_MARK = 0x01; + +@@ -1098,7 +1101,7 @@ class HierarchicalNSW : public AlgorithmInterface { + + std::unique_lock lock_el(link_list_locks_[cur_c]); + int curlevel = getRandomLevel(mult_); +- if (level > 0) ++ if (level > -1) + curlevel = level; + + element_levels_[cur_c] = curlevel; +@@ -1116,6 +1119,9 @@ class HierarchicalNSW : public AlgorithmInterface { + memcpy(getExternalLabeLp(cur_c), &label, sizeof(labeltype)); + memcpy(getDataByInternalId(cur_c), data_point, data_size_); + ++ if (!base_layer_init && curlevel == 0) ++ return cur_c; ++ + if (curlevel) { + linkLists_[cur_c] = (char *) malloc(size_links_per_element_ * curlevel + 1); + if (linkLists_[cur_c] == nullptr) +@@ -1138,7 +1144,7 @@ class HierarchicalNSW : public AlgorithmInterface { + tableint *datal = (tableint *) (data + 1); + for (int i = 0; i < size; i++) { + tableint cand = datal[i]; +- if (cand < 0 || cand > max_elements_) ++ if (static_cast(cand) < 0 || cand > max_elements_) + throw std::runtime_error("cand error"); + dist_t d = fstdistfunc_(data_point, getDataByInternalId(cand), dist_func_param_); + if (d < curdist) { +@@ -1188,28 +1194,41 @@ class HierarchicalNSW : public AlgorithmInterface { + tableint currObj = enterpoint_node_; + dist_t curdist = fstdistfunc_(query_data, getDataByInternalId(enterpoint_node_), dist_func_param_); + +- for (int level = maxlevel_; level > 0; level--) { +- bool changed = true; +- while (changed) { +- changed = false; +- unsigned int *data; ++ if (base_layer_only) { ++ // You can increase the number of seeds when testing large-scale dataset, num_seeds = 48 for 100M-scale ++ for (int i = 0; i < num_seeds; i++) { ++ tableint obj = i * (max_elements_ / num_seeds); ++ dist_t dist = fstdistfunc_(query_data, getDataByInternalId(obj), dist_func_param_); ++ if (dist < curdist) { ++ curdist = dist; ++ currObj = obj; ++ } ++ } ++ } ++ else { ++ for (int level = maxlevel_; level > 0; level--) { ++ bool changed = true; ++ while (changed) { ++ changed = false; ++ unsigned int *data; + +- data = (unsigned int *) get_linklist(currObj, level); +- int size = getListCount(data); +- metric_hops++; +- metric_distance_computations+=size; ++ data = (unsigned int *) get_linklist(currObj, level); ++ int size = getListCount(data); ++ metric_hops++; ++ metric_distance_computations+=size; ++ ++ tableint *datal = (tableint *) (data + 1); ++ for (int i = 0; i < size; i++) { ++ tableint cand = datal[i]; ++ if (static_cast(cand) < 0 || cand > max_elements_) ++ throw std::runtime_error("cand error"); ++ dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); + +- tableint *datal = (tableint *) (data + 1); +- for (int i = 0; i < size; i++) { +- tableint cand = datal[i]; +- if (cand < 0 || cand > max_elements_) +- throw std::runtime_error("cand error"); +- dist_t d = fstdistfunc_(query_data, getDataByInternalId(cand), dist_func_param_); +- +- if (d < curdist) { +- curdist = d; +- currObj = cand; +- changed = true; ++ if (d < curdist) { ++ curdist = d; ++ currObj = cand; ++ changed = true; ++ } + } + } + } +diff --git a/hnswlib/space_ip.h b/hnswlib/space_ip.h +index 2b1c359..e311f9d 100644 +--- a/hnswlib/space_ip.h ++++ b/hnswlib/space_ip.h +@@ -3,19 +3,22 @@ + + namespace hnswlib { + +-static float ++template ++static DistanceType + InnerProduct(const void *pVect1, const void *pVect2, const void *qty_ptr) { + size_t qty = *((size_t *) qty_ptr); +- float res = 0; ++ DistanceType res = 0; + for (unsigned i = 0; i < qty; i++) { +- res += ((float *) pVect1)[i] * ((float *) pVect2)[i]; ++ const DistanceType t = ((DataType *) pVect1)[i] * ((DataType *) pVect2)[i]; ++ res += t; + } + return res; + } + +-static float ++template ++static DistanceType + InnerProductDistance(const void *pVect1, const void *pVect2, const void *qty_ptr) { +- return 1.0f - InnerProduct(pVect1, pVect2, qty_ptr); ++ return DistanceType{1} - InnerProduct(pVect1, pVect2, qty_ptr); + } + + #if defined(USE_AVX) +@@ -294,7 +297,7 @@ InnerProductDistanceSIMD16ExtResiduals(const void *pVect1v, const void *pVect2v, + float *pVect2 = (float *) pVect2v + qty16; + + size_t qty_left = qty - qty16; +- float res_tail = InnerProduct(pVect1, pVect2, &qty_left); ++ float res_tail = InnerProduct(pVect1, pVect2, &qty_left); + return 1.0f - (res + res_tail); + } + +@@ -308,20 +311,21 @@ InnerProductDistanceSIMD4ExtResiduals(const void *pVect1v, const void *pVect2v, + + float *pVect1 = (float *) pVect1v + qty4; + float *pVect2 = (float *) pVect2v + qty4; +- float res_tail = InnerProduct(pVect1, pVect2, &qty_left); ++ float res_tail = InnerProduct(pVect1, pVect2, &qty_left); + + return 1.0f - (res + res_tail); + } + #endif + +-class InnerProductSpace : public SpaceInterface { +- DISTFUNC fstdistfunc_; ++template ++class InnerProductSpace : public SpaceInterface { ++ DISTFUNC fstdistfunc_; + size_t data_size_; + size_t dim_; + + public: + InnerProductSpace(size_t dim) { +- fstdistfunc_ = InnerProductDistance; ++ fstdistfunc_ = InnerProductDistance; + #if defined(USE_AVX) || defined(USE_SSE) || defined(USE_AVX512) + #if defined(USE_AVX512) + if (AVX512Capable()) { +@@ -344,24 +348,26 @@ class InnerProductSpace : public SpaceInterface { + } + #endif + +- if (dim % 16 == 0) +- fstdistfunc_ = InnerProductDistanceSIMD16Ext; +- else if (dim % 4 == 0) +- fstdistfunc_ = InnerProductDistanceSIMD4Ext; +- else if (dim > 16) +- fstdistfunc_ = InnerProductDistanceSIMD16ExtResiduals; +- else if (dim > 4) +- fstdistfunc_ = InnerProductDistanceSIMD4ExtResiduals; ++ if constexpr (std::is_same_v) { ++ if (dim % 16 == 0) ++ fstdistfunc_ = InnerProductDistanceSIMD16Ext; ++ else if (dim % 4 == 0) ++ fstdistfunc_ = InnerProductDistanceSIMD4Ext; ++ else if (dim > 16) ++ fstdistfunc_ = InnerProductDistanceSIMD16ExtResiduals; ++ else if (dim > 4) ++ fstdistfunc_ = InnerProductDistanceSIMD4ExtResiduals; ++ } + #endif + dim_ = dim; +- data_size_ = dim * sizeof(float); ++ data_size_ = dim * sizeof(DataType); + } + + size_t get_data_size() { + return data_size_; + } + +- DISTFUNC get_dist_func() { ++ DISTFUNC get_dist_func() { + return fstdistfunc_; + } + +diff --git a/hnswlib/space_l2.h b/hnswlib/space_l2.h +index 834d19f..c57c87a 100644 +--- a/hnswlib/space_l2.h ++++ b/hnswlib/space_l2.h +@@ -3,15 +3,16 @@ + + namespace hnswlib { + +-static float ++template ++static DistanceType + L2Sqr(const void *pVect1v, const void *pVect2v, const void *qty_ptr) { +- float *pVect1 = (float *) pVect1v; +- float *pVect2 = (float *) pVect2v; ++ DataType *pVect1 = (DataType *) pVect1v; ++ DataType *pVect2 = (DataType *) pVect2v; + size_t qty = *((size_t *) qty_ptr); + +- float res = 0; ++ DistanceType res = 0; + for (size_t i = 0; i < qty; i++) { +- float t = *pVect1 - *pVect2; ++ DistanceType t = *pVect1 - *pVect2; + pVect1++; + pVect2++; + res += t * t; +@@ -155,7 +156,7 @@ L2SqrSIMD16ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qt + float *pVect2 = (float *) pVect2v + qty16; + + size_t qty_left = qty - qty16; +- float res_tail = L2Sqr(pVect1, pVect2, &qty_left); ++ float res_tail = L2Sqr(pVect1, pVect2, &qty_left); + return (res + res_tail); + } + #endif +@@ -199,20 +200,21 @@ L2SqrSIMD4ExtResiduals(const void *pVect1v, const void *pVect2v, const void *qty + + float *pVect1 = (float *) pVect1v + qty4; + float *pVect2 = (float *) pVect2v + qty4; +- float res_tail = L2Sqr(pVect1, pVect2, &qty_left); ++ float res_tail = L2Sqr(pVect1, pVect2, &qty_left); + + return (res + res_tail); + } + #endif + +-class L2Space : public SpaceInterface { +- DISTFUNC fstdistfunc_; ++template ++class L2Space : public SpaceInterface { ++ DISTFUNC fstdistfunc_; + size_t data_size_; + size_t dim_; + + public: + L2Space(size_t dim) { +- fstdistfunc_ = L2Sqr; ++ fstdistfunc_ = L2Sqr; + #if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512) + #if defined(USE_AVX512) + if (AVX512Capable()) +@@ -224,24 +226,26 @@ class L2Space : public SpaceInterface { + L2SqrSIMD16Ext = L2SqrSIMD16ExtAVX; + #endif + +- if (dim % 16 == 0) +- fstdistfunc_ = L2SqrSIMD16Ext; +- else if (dim % 4 == 0) +- fstdistfunc_ = L2SqrSIMD4Ext; +- else if (dim > 16) +- fstdistfunc_ = L2SqrSIMD16ExtResiduals; +- else if (dim > 4) +- fstdistfunc_ = L2SqrSIMD4ExtResiduals; ++ if constexpr (std::is_same_v) { ++ if (dim % 16 == 0) ++ fstdistfunc_ = L2SqrSIMD16Ext; ++ else if (dim % 4 == 0) ++ fstdistfunc_ = L2SqrSIMD4Ext; ++ else if (dim > 16) ++ fstdistfunc_ = L2SqrSIMD16ExtResiduals; ++ else if (dim > 4) ++ fstdistfunc_ = L2SqrSIMD4ExtResiduals; ++ } + #endif + dim_ = dim; +- data_size_ = dim * sizeof(float); ++ data_size_ = dim * sizeof(DataType); + } + + size_t get_data_size() { + return data_size_; + } + +- DISTFUNC get_dist_func() { ++ DISTFUNC get_dist_func() { + return fstdistfunc_; + } + +@@ -252,12 +256,13 @@ class L2Space : public SpaceInterface { + ~L2Space() {} + }; + ++template + static int + L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const void *__restrict qty_ptr) { + size_t qty = *((size_t *) qty_ptr); + int res = 0; +- unsigned char *a = (unsigned char *) pVect1; +- unsigned char *b = (unsigned char *) pVect2; ++ T *a = (T *) pVect1; ++ T *b = (T *) pVect2; + + qty = qty >> 2; + for (size_t i = 0; i < qty; i++) { +@@ -277,11 +282,12 @@ L2SqrI4x(const void *__restrict pVect1, const void *__restrict pVect2, const voi + return (res); + } + ++template + static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, const void* __restrict qty_ptr) { + size_t qty = *((size_t*)qty_ptr); + int res = 0; +- unsigned char* a = (unsigned char*)pVect1; +- unsigned char* b = (unsigned char*)pVect2; ++ T* a = (T*)pVect1; ++ T* b = (T*)pVect2; + + for (size_t i = 0; i < qty; i++) { + res += ((*a) - (*b)) * ((*a) - (*b)); +@@ -291,6 +297,7 @@ static int L2SqrI(const void* __restrict pVect1, const void* __restrict pVect2, + return (res); + } + ++template + class L2SpaceI : public SpaceInterface { + DISTFUNC fstdistfunc_; + size_t data_size_; +@@ -299,9 +306,9 @@ class L2SpaceI : public SpaceInterface { + public: + L2SpaceI(size_t dim) { + if (dim % 4 == 0) { +- fstdistfunc_ = L2SqrI4x; ++ fstdistfunc_ = L2SqrI4x; + } else { +- fstdistfunc_ = L2SqrI; ++ fstdistfunc_ = L2SqrI; + } + dim_ = dim; + data_size_ = dim * sizeof(unsigned char); diff --git a/cpp/cmake/patches/hnswlib_override.json b/cpp/cmake/patches/hnswlib_override.json new file mode 100644 index 0000000000..812af74bc1 --- /dev/null +++ b/cpp/cmake/patches/hnswlib_override.json @@ -0,0 +1,16 @@ +{ + "packages": { + "hnswlib": { + "version": "0.7.0", + "git_url": "https://github.com/nmslib/hnswlib.git", + "git_tag": "v${version}", + "patches": [ + { + "file": "${current_json_dir}/hnswlib.diff", + "issue": "Correct compilation issues", + "fixed_in": "" + } + ] + } + } +} diff --git a/cpp/cmake/thirdparty/get_cutlass.cmake b/cpp/cmake/thirdparty/get_cutlass.cmake index 61065318ba..04eaca8940 100644 --- a/cpp/cmake/thirdparty/get_cutlass.cmake +++ b/cpp/cmake/thirdparty/get_cutlass.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2021-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,10 +13,11 @@ # ============================================================================= function(find_and_configure_cutlass) - set(oneValueArgs VERSION REPOSITORY PINNED_TAG) + set(options) + set(oneValueArgs) + set(multiValueArgs) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - # if(RAFT_ENABLE_DIST_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) set(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library" @@ -34,13 +35,22 @@ function(find_and_configure_cutlass) set(CUDART_LIBRARY "${CUDA_cudart_static_LIBRARY}" CACHE FILEPATH "fixing cutlass cmake code" FORCE) endif() + include("${rapids-cmake-dir}/cpm/package_override.cmake") + rapids_cpm_package_override("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches/cutlass_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(cutlass version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(cutlass ${version} patch_command build_patch_only) + rapids_cpm_find( - NvidiaCutlass ${PKG_VERSION} + NvidiaCutlass ${version} ${build_patch_only} GLOBAL_TARGETS nvidia::cutlass::cutlass CPM_ARGS - GIT_REPOSITORY ${PKG_REPOSITORY} - GIT_TAG ${PKG_PINNED_TAG} - GIT_SHALLOW TRUE + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} OPTIONS "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" ) @@ -48,45 +58,6 @@ function(find_and_configure_cutlass) add_library(nvidia::cutlass::cutlass ALIAS CUTLASS) endif() - if(NvidiaCutlass_ADDED) - rapids_export( - BUILD NvidiaCutlass - EXPORT_SET NvidiaCutlass - GLOBAL_TARGETS nvidia::cutlass::cutlass - NAMESPACE nvidia::cutlass:: - ) - endif() - # endif() - - # We generate the cutlass-config files when we built cutlass locally, so always do - # `find_dependency` - rapids_export_package( - BUILD NvidiaCutlass cuvs-exports GLOBAL_TARGETS nvidia::cutlass::cutlass - ) - rapids_export_package( - INSTALL NvidiaCutlass cuvs-exports GLOBAL_TARGETS nvidia::cutlass::cutlass - ) - - # Tell cmake where it can find the generated NvidiaCutlass-config.cmake we wrote. - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root( - INSTALL NvidiaCutlass [=[${CMAKE_CURRENT_LIST_DIR}/../]=] - EXPORT_SET cuvs-exports - ) - rapids_export_find_package_root( - BUILD NvidiaCutlass [=[${CMAKE_CURRENT_LIST_DIR}]=] - EXPORT_SET cuvs-exports - ) endfunction() -if(NOT RAFT_CUTLASS_GIT_TAG) - set(RAFT_CUTLASS_GIT_TAG v2.10.0) -endif() - -if(NOT RAFT_CUTLASS_GIT_REPOSITORY) - set(RAFT_CUTLASS_GIT_REPOSITORY https://github.com/NVIDIA/cutlass.git) -endif() - -find_and_configure_cutlass( - VERSION 2.10.0 REPOSITORY ${RAFT_CUTLASS_GIT_REPOSITORY} PINNED_TAG ${RAFT_CUTLASS_GIT_TAG} -) +find_and_configure_cutlass() diff --git a/cpp/cmake/thirdparty/get_cuvs.cmake b/cpp/cmake/thirdparty/get_cuvs.cmake new file mode 100644 index 0000000000..6ef4791474 --- /dev/null +++ b/cpp/cmake/thirdparty/get_cuvs.cmake @@ -0,0 +1,64 @@ +# ============================================================================= +# Copyright (c) 2023-2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. + +# Use RAPIDS_VERSION_MAJOR_MINOR from cmake/rapids_config.cmake +set(CUVS_VERSION "${RAPIDS_VERSION_MAJOR_MINOR}") +set(CUVS_FORK "rapidsai") +set(CUVS_PINNED_TAG "${rapids-cmake-checkout-tag}") + +function(find_and_configure_cuvs) + set(oneValueArgs VERSION FORK PINNED_TAG ENABLE_NVTX CLONE_ON_PIN BUILD_CPU_ONLY BUILD_SHARED_LIBS) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "${rapids-cmake-checkout-tag}") + message(STATUS "cuVS: pinned tag found: ${PKG_PINNED_TAG}. Cloning cuVS locally.") + set(CPM_DOWNLOAD_cuvs ON) + endif() + + #----------------------------------------------------- + # Invoke CPM find_package() + #----------------------------------------------------- + rapids_cpm_find(cuvs ${PKG_VERSION} + GLOBAL_TARGETS cuvs::cuvs + BUILD_EXPORT_SET cuvs-bench-exports + INSTALL_EXPORT_SET cuvs-bench-exports + COMPONENTS cuvs + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/cuvs.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_SHARED_LIBS ${PKG_BUILD_SHARED_LIBS}" + "BUILD_CPU_ONLY ${PKG_BUILD_CPU_ONLY}" + "BUILD_TESTS OFF" + "BUILD_CAGRA_HNSWLIB OFF" + "CUVS_CLONE_ON_PIN ${PKG_CLONE_ON_PIN}" + ) +endfunction() + + +# Change pinned tag here to test a commit in CI +# To use a different cuVS locally, set the CMake variable +# CPM_cuvs_SOURCE=/path/to/local/cuvs +find_and_configure_cuvs(VERSION ${CUVS_VERSION}.00 + FORK ${CUVS_FORK} + PINNED_TAG ${CUVS_PINNED_TAG} + ENABLE_NVTX OFF + # When PINNED_TAG above doesn't match the default rapids branch, + # force local cuvs clone in build directory + # even if it's already installed. + CLONE_ON_PIN ${CUVS_CLONE_ON_PIN} + BUILD_CPU_ONLY ${BUILD_CPU_ONLY} + BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS} +) diff --git a/cpp/cmake/thirdparty/get_diskann.cmake b/cpp/cmake/thirdparty/get_diskann.cmake new file mode 100644 index 0000000000..6a37bc6639 --- /dev/null +++ b/cpp/cmake/thirdparty/get_diskann.cmake @@ -0,0 +1,49 @@ +#============================================================================= +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_diskann) + set(oneValueArgs VERSION REPOSITORY PINNED_TAG) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + include(${rapids-cmake-dir}/cpm/package_override.cmake) + set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") + rapids_cpm_package_override("${patch_dir}/diskann_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(diskann version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(diskann ${version} patch_command build_patch_only) + + rapids_cpm_find(diskann ${version} ${build_patch_only} + GLOBAL_TARGETS diskann + CPM_ARGS + OPTIONS + "PYBIND OFF" + "UNIT_TEST OFF" + "RESTAPI OFF" + "PORTABLE OFF") + + include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") + rapids_cpm_display_patch_status(diskann) + + if(NOT TARGET diskann::diskann) + target_include_directories(diskann INTERFACE "$") + add_library(diskann::diskann ALIAS diskann) + endif() +endfunction() +find_and_configure_diskann() diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake new file mode 100644 index 0000000000..8075abe050 --- /dev/null +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -0,0 +1,120 @@ +#============================================================================= +# Copyright (c) 2024-2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_faiss) + set(oneValueArgs VERSION REPOSITORY PINNED_TAG BUILD_STATIC_LIBS EXCLUDE_FROM_ALL ENABLE_GPU) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_find_generate_module(faiss + HEADER_NAMES faiss/IndexFlat.h + LIBRARY_NAMES faiss + ) + + set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") + rapids_cpm_package_override("${patch_dir}/faiss_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(faiss version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(faiss ${version} patch_command build_patch_only) + + set(BUILD_SHARED_LIBS ON) + if (PKG_BUILD_STATIC_LIBS) + set(BUILD_SHARED_LIBS OFF) + set(CPM_DOWNLOAD_faiss ON) + endif() + + include(cmake/modules/FindAVX) + # Link against AVX CPU lib if it exists + set(CUVS_FAISS_OPT_LEVEL "generic") + if(CXX_AVX2_FOUND) + set(CUVS_FAISS_OPT_LEVEL "avx2") + endif() + + + rapids_cpm_find(faiss ${version} ${build_patch_only} + GLOBAL_TARGETS faiss faiss_avx2 faiss_gpu_objs faiss::faiss faiss::faiss_avx2 + CPM_ARGS + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} + EXCLUDE_FROM_ALL ${exclude} + OPTIONS + "FAISS_ENABLE_GPU ${PKG_ENABLE_GPU}" + "FAISS_ENABLE_CUVS ${PKG_ENABLE_GPU}" + "FAISS_ENABLE_PYTHON OFF" + "FAISS_OPT_LEVEL ${CUVS_FAISS_OPT_LEVEL}" + "FAISS_USE_CUDA_TOOLKIT_STATIC ${CUDA_STATIC_RUNTIME}" + "BUILD_TESTING OFF" + "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" + ) + + + include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") + rapids_cpm_display_patch_status(faiss) + + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) + # We need to ensure that faiss has all the conda information. So we use this approach so that + # faiss will have the conda includes/link dirs + target_link_libraries(faiss PRIVATE $) + endif() + if(TARGET faiss_avx2 AND NOT TARGET faiss::faiss_avx2) + add_library(faiss::faiss_avx2 ALIAS faiss_avx2) + # We need to ensure that faiss has all the conda information. So we use this approach so that + # faiss will have the conda includes/link dirs + target_link_libraries(faiss_avx2 PRIVATE $) + endif() + if(TARGET faiss_gpu_objs AND NOT TARGET faiss::faiss_gpu_objs) + add_library(faiss::faiss_gpu_objs ALIAS faiss_gpu_objs) + # We need to ensure that faiss has all the conda information. So we use this approach so that + # faiss will have the conda includes/link dirs + target_link_libraries(faiss_gpu_objs PRIVATE $) + endif() + + if(faiss_ADDED) + rapids_export(BUILD faiss + EXPORT_SET faiss-targets + GLOBAL_TARGETS ${CUVS_FAISS_EXPORT_GLOBAL_TARGETS} + NAMESPACE faiss::) + endif() + + # Need to tell CMake to rescan the link group of faiss::faiss_gpu_objs and faiss + # so that we get proper link order when they are static + # + # We don't look at the existence of `faiss_avx2` as it will always exist + # even when CXX_AVX2_FOUND is false. In addition for arm builds the + # faiss_avx2 is marked as `EXCLUDE_FROM_ALL` so we don't want to add + # a dependency to it. Adding a dependency will cause it to compile, + # and fail due to invalid compiler flags. + if(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS AND CXX_AVX2_FOUND) + set(CUVS_FAISS_TARGETS "$,faiss::faiss_avx2>" PARENT_SCOPE) + elseif(PKG_ENABLE_GPU AND PKG_BUILD_STATIC_LIBS) + set(CUVS_FAISS_TARGETS "$,faiss::faiss>" PARENT_SCOPE) + elseif(CXX_AVX2_FOUND) + set(CUVS_FAISS_TARGETS faiss::faiss_avx2 PARENT_SCOPE) + else() + set(CUVS_FAISS_TARGETS faiss::faiss PARENT_SCOPE) + endif() + +endfunction() + +find_and_configure_faiss( + BUILD_STATIC_LIBS ${CUVS_USE_FAISS_STATIC} + ENABLE_GPU ${CUVS_FAISS_ENABLE_GPU} +) diff --git a/cpp/cmake/thirdparty/get_ggnn.cmake b/cpp/cmake/thirdparty/get_ggnn.cmake new file mode 100644 index 0000000000..5bd77215c8 --- /dev/null +++ b/cpp/cmake/thirdparty/get_ggnn.cmake @@ -0,0 +1,49 @@ +#============================================================================= +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_ggnn) + + include(${rapids-cmake-dir}/cpm/package_override.cmake) + set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") + rapids_cpm_package_override("${patch_dir}/ggnn_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(ggnn version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(ggnn ${version} patch_command build_patch_only) + + rapids_cpm_find( + ggnn ${version} ${build_patch_only} + GLOBAL_TARGETS ggnn::ggnn + CPM_ARGS + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} + DOWNLOAD_ONLY ON + ) + + include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") + rapids_cpm_display_patch_status(ggnn) + + if(NOT TARGET ggnn::ggnn) + add_library(ggnn INTERFACE) + target_include_directories(ggnn INTERFACE "$") + add_library(ggnn::ggnn ALIAS ggnn) + endif() + +endfunction() +find_and_configure_ggnn() diff --git a/cpp/cmake/thirdparty/get_glog.cmake b/cpp/cmake/thirdparty/get_glog.cmake index 56ed452746..5839ee3d5a 100644 --- a/cpp/cmake/thirdparty/get_glog.cmake +++ b/cpp/cmake/thirdparty/get_glog.cmake @@ -1,5 +1,5 @@ #============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,6 @@ function(find_and_configure_glog) rapids_cpm_find(glog ${PKG_VERSION} GLOBAL_TARGETS glog::glog - BUILD_EXPORT_SET cuvs-exports - INSTALL_EXPORT_SET cuvs-exports CPM_ARGS GIT_REPOSITORY https://github.com/${PKG_FORK}/glog.git GIT_TAG ${PKG_PINNED_TAG} @@ -30,17 +28,13 @@ function(find_and_configure_glog) ) if(glog_ADDED) - message(VERBOSE "RAFT: Using glog located in ${glog_SOURCE_DIR}") + message(VERBOSE "cuVS: Using glog located in ${glog_SOURCE_DIR}") else() - message(VERBOSE "RAFT: Using glog located in ${glog_DIR}") + message(VERBOSE "cuVS: Using glog located in ${glog_DIR}") endif() - endfunction() -# Change pinned tag here to test a commit in CI -# To use a different RAFT locally, set the CMake variable -# CPM_glog_SOURCE=/path/to/local/glog find_and_configure_glog(VERSION 0.6.0 FORK google PINNED_TAG v0.6.0 diff --git a/cpp/cmake/thirdparty/get_hnswlib.cmake b/cpp/cmake/thirdparty/get_hnswlib.cmake new file mode 100644 index 0000000000..a99e3b032f --- /dev/null +++ b/cpp/cmake/thirdparty/get_hnswlib.cmake @@ -0,0 +1,89 @@ +#============================================================================= +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_hnswlib) + message(STATUS "Finding or building hnswlib") + set(oneValueArgs) + + include(${rapids-cmake-dir}/cpm/package_override.cmake) + set(patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../patches") + rapids_cpm_package_override("${patch_dir}/hnswlib_override.json") + + include("${rapids-cmake-dir}/cpm/detail/package_details.cmake") + rapids_cpm_package_details(hnswlib version repository tag shallow exclude) + + include("${rapids-cmake-dir}/cpm/detail/generate_patch_command.cmake") + rapids_cpm_generate_patch_command(hnswlib ${version} patch_command build_patch_only) + + rapids_cpm_find( + hnswlib ${version} ${build_patch_only} + GLOBAL_TARGETS hnswlib hnswlib::hnswlib + CPM_ARGS + GIT_REPOSITORY ${repository} + GIT_TAG ${tag} + GIT_SHALLOW ${shallow} ${patch_command} + EXCLUDE_FROM_ALL ${exclude} + DOWNLOAD_ONLY ON + ) + + include("${rapids-cmake-dir}/cpm/detail/display_patch_status.cmake") + rapids_cpm_display_patch_status(hnswlib) + + if(NOT TARGET hnswlib::hnswlib) + add_library(hnswlib INTERFACE ) + add_library(hnswlib::hnswlib ALIAS hnswlib) + target_include_directories(hnswlib INTERFACE + "$" + "$") + endif() + + if(hnswlib_ADDED) + # write build export rules + install(TARGETS hnswlib EXPORT hnswlib-exports) + if(NOT exclude) + install(DIRECTORY "${hnswlib_SOURCE_DIR}/hnswlib/" DESTINATION include/hnswlib) + + # write install export rules + rapids_export( + INSTALL hnswlib + VERSION ${version} + EXPORT_SET hnswlib-exports + GLOBAL_TARGETS hnswlib + NAMESPACE hnswlib::) + endif() + + rapids_export( + BUILD hnswlib + VERSION ${version} + EXPORT_SET hnswlib-exports + GLOBAL_TARGETS hnswlib + NAMESPACE hnswlib::) + + include("${rapids-cmake-dir}/export/package.cmake") + rapids_export_package(INSTALL hnswlib cuvs-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) + rapids_export_package(BUILD hnswlib cuvs-exports VERSION ${version} GLOBAL_TARGETS hnswlib hnswlib::hnswlib) + + + # When using cuVS from the build dir, ensure hnswlib is also found in cuVS' build dir. This + # line adds `set(hnswlib_ROOT "${CMAKE_CURRENT_LIST_DIR}")` to build/cuvs-dependencies.cmake + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root( + BUILD hnswlib [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET cuvs-exports + ) + endif() +endfunction() + +find_and_configure_hnswlib() diff --git a/cpp/cmake/thirdparty/get_nlohmann_json.cmake b/cpp/cmake/thirdparty/get_nlohmann_json.cmake new file mode 100644 index 0000000000..8d0b671e47 --- /dev/null +++ b/cpp/cmake/thirdparty/get_nlohmann_json.cmake @@ -0,0 +1,42 @@ +#============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +function(find_and_configure_nlohmann_json) + set(oneValueArgs VERSION FORK PINNED_TAG EXCLUDE_FROM_ALL) + cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" + "${multiValueArgs}" ${ARGN} ) + + rapids_cpm_find(nlohmann_json ${PKG_VERSION} + GLOBAL_TARGETS nlohmann_json::nlohmann_json + CPM_ARGS + GIT_REPOSITORY https://github.com/${PKG_FORK}/json.git + GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL ${PKG_EXCLUDE_FROM_ALL} + ) + + if(glog_ADDED) + message(VERBOSE "cuVS: Using glog located in ${glog_SOURCE_DIR}") + else() + message(VERBOSE "cuVS: Using glog located in ${glog_DIR}") + endif() + +endfunction() + +find_and_configure_nlohmann_json(VERSION 3.11.2 + FORK nlohmann + PINNED_TAG v3.11.2 + EXCLUDE_FROM_ALL ON + ) diff --git a/cpp/cmake/thirdparty/get_raft.cmake b/cpp/cmake/thirdparty/get_raft.cmake index d57d27312f..6a08edb2ab 100644 --- a/cpp/cmake/thirdparty/get_raft.cmake +++ b/cpp/cmake/thirdparty/get_raft.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -11,21 +11,27 @@ # or implied. See the License for the specific language governing permissions and limitations under # the License. -# Use RAPIDS_VERSION from cmake/thirdparty/fetch_rapids.cmake -set(RAFT_VERSION "${RAPIDS_VERSION}") +# Use RAPIDS_VERSION_MAJOR_MINOR from rapids_config.cmake +set(RAFT_VERSION "${RAPIDS_VERSION_MAJOR_MINOR}") set(RAFT_FORK "rapidsai") -set(RAFT_PINNED_TAG "branch-${RAPIDS_VERSION}") +set(RAFT_PINNED_TAG "${rapids-cmake-checkout-tag}") function(find_and_configure_raft) - set(oneValueArgs VERSION FORK PINNED_TAG COMPILE_LIBRARY ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES) + set(oneValueArgs VERSION FORK PINNED_TAG BUILD_STATIC_DEPS ENABLE_NVTX ENABLE_MNMG_DEPENDENCIES CLONE_ON_PIN) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - set(RAFT_COMPONENTS "") - if(PKG_COMPILE_LIBRARY) - string(APPEND RAFT_COMPONENTS " compiled") + if(PKG_CLONE_ON_PIN AND NOT PKG_PINNED_TAG STREQUAL "${rapids-cmake-checkout-tag}") + message(STATUS "cuVS: RAFT pinned tag found: ${PKG_PINNED_TAG}. Cloning raft locally.") + set(CPM_DOWNLOAD_raft ON) + elseif(PKG_BUILD_STATIC_DEPS AND (NOT CPM_raft_SOURCE)) + message(STATUS "cuVS: Cloning raft locally to build static libraries.") + set(CPM_DOWNLOAD_raft ON) + set(BUILD_SHARED_LIBS OFF) endif() + set(RAFT_COMPONENTS "") + if(PKG_ENABLE_MNMG_DEPENDENCIES) string(APPEND RAFT_COMPONENTS " distributed") endif() @@ -39,25 +45,31 @@ function(find_and_configure_raft) INSTALL_EXPORT_SET cuvs-exports COMPONENTS ${RAFT_COMPONENTS} CPM_ARGS - GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git - GIT_TAG ${PKG_PINNED_TAG} - SOURCE_SUBDIR cpp - OPTIONS - "BUILD_TESTS OFF" - "BUILD_PRIMS_BENCH OFF" - "BUILD_ANN_BENCH OFF" - "RAFT_NVTX ${PKG_ENABLE_NVTX}" - "RAFT_COMPILE_LIBRARY ${PKG_COMPILE_LIBRARY}" + EXCLUDE_FROM_ALL TRUE + GIT_REPOSITORY https://github.com/${PKG_FORK}/raft.git + GIT_TAG ${PKG_PINNED_TAG} + SOURCE_SUBDIR cpp + OPTIONS + "BUILD_TESTS OFF" + "BUILD_PRIMS_BENCH OFF" + "RAFT_NVTX ${PKG_ENABLE_NVTX}" + "RAFT_COMPILE_LIBRARY OFF" ) endfunction() + # Change pinned tag here to test a commit in CI # To use a different RAFT locally, set the CMake variable # CPM_raft_SOURCE=/path/to/local/raft find_and_configure_raft(VERSION ${RAFT_VERSION}.00 FORK ${RAFT_FORK} PINNED_TAG ${RAFT_PINNED_TAG} - COMPILE_LIBRARY ON ENABLE_MNMG_DEPENDENCIES OFF ENABLE_NVTX OFF + BUILD_STATIC_DEPS ${CUVS_STATIC_RAPIDS_LIBRARIES} + # When PINNED_TAG above doesn't match the default rapids branch, + # force local raft clone in build directory + # even if it's already installed. + CLONE_ON_PIN ${CUVS_RAFT_CLONE_ON_PIN} + ) diff --git a/cpp/cmake/thirdparty/get_spdlog.cmake b/cpp/cmake/thirdparty/get_spdlog.cmake deleted file mode 100644 index 7be7804c7e..0000000000 --- a/cpp/cmake/thirdparty/get_spdlog.cmake +++ /dev/null @@ -1,33 +0,0 @@ -# ============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone speedlog -function(find_and_configure_spdlog) - - include(${rapids-cmake-dir}/cpm/spdlog.cmake) - rapids_cpm_spdlog(FMT_OPTION "EXTERNAL_FMT_HO" INSTALL_EXPORT_SET rmm-exports) - rapids_export_package(BUILD spdlog rmm-exports) - - if(spdlog_ADDED) - rapids_export( - BUILD spdlog - EXPORT_SET spdlog - GLOBAL_TARGETS spdlog spdlog_header_only - NAMESPACE spdlog::) - include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD spdlog [=[${CMAKE_CURRENT_LIST_DIR}]=] EXPORT_SET rmm-exports) - endif() -endfunction() - -find_and_configure_spdlog() \ No newline at end of file diff --git a/cpp/doxygen/Doxyfile b/cpp/doxygen/Doxyfile index 0a2c7f8f61..a1c20422e7 100644 --- a/cpp/doxygen/Doxyfile +++ b/cpp/doxygen/Doxyfile @@ -1,7 +1,7 @@ -# Doxyfile 1.8.20 +# Doxyfile 1.13.2 # This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. +# Doxygen (www.doxygen.org) for a project. # # All text after a double hash (##) is considered a comment and is placed in # front of the TAG it is preceding. @@ -12,6 +12,16 @@ # For lists, items can also be appended using: # TAG += value [value, ...] # Values that contain spaces should be placed between quotes (\" \"). +# +# Note: +# +# Use Doxygen to compare the used configuration file with the template +# configuration file: +# doxygen -x [configFile] +# Use Doxygen to compare the used configuration file with the template +# configuration file without replacing the environment variables or CMake type +# replacement variables: +# doxygen -x_noenv [configFile] #--------------------------------------------------------------------------- # Project related configuration options @@ -38,10 +48,10 @@ PROJECT_NAME = "cuVS C++ API" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "24.02" +PROJECT_NUMBER = $(RAPIDS_VERSION_MAJOR_MINOR) # Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a +# for a project that appears at the top of each page and should give viewers a # quick idea about the purpose of the project. Keep the description short. PROJECT_BRIEF = @@ -53,24 +63,42 @@ PROJECT_BRIEF = PROJECT_LOGO = +# With the PROJECT_ICON tag one can specify an icon that is included in the tabs +# when the HTML document is shown. Doxygen will copy the logo to the output +# directory. + +PROJECT_ICON = + # The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path # into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If +# entered, it will be relative to the location where Doxygen was started. If # left blank the current directory will be used. OUTPUT_DIRECTORY = -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. +# If the CREATE_SUBDIRS tag is set to YES then Doxygen will create up to 4096 +# sub-directories (in 2 levels) under the output directory of each output format +# and will distribute the generated files over these directories. Enabling this +# option can be useful when feeding Doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise cause +# performance problems for the file system. Adapt CREATE_SUBDIRS_LEVEL to +# control the number of sub-directories. # The default value is: NO. CREATE_SUBDIRS = NO -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# Controls the number of sub-directories that will be created when +# CREATE_SUBDIRS tag is set to YES. Level 0 represents 16 directories, and every +# level increment doubles the number of directories, resulting in 4096 +# directories at level 8 which is the default and also the maximum value. The +# sub-directories are organized in 2 levels, the first level always has a fixed +# number of 16 directories. +# Minimum value: 0, maximum value: 8, default value: 8. +# This tag requires that the tag CREATE_SUBDIRS is set to YES. + +CREATE_SUBDIRS_LEVEL = 8 + +# If the ALLOW_UNICODE_NAMES tag is set to YES, Doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII # characters will be escaped, for example _xE3_x81_x84 will be used for Unicode # U+3044. @@ -79,36 +107,28 @@ CREATE_SUBDIRS = NO ALLOW_UNICODE_NAMES = NO # The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this +# documentation generated by Doxygen is written. Doxygen will use this # information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Bulgarian, +# Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, English +# (United States), Esperanto, Farsi (Persian), Finnish, French, German, Greek, +# Hindi, Hungarian, Indonesian, Italian, Japanese, Japanese-en (Japanese with +# English messages), Korean, Korean-en (Korean with English messages), Latvian, +# Lithuanian, Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, +# Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, +# Swedish, Turkish, Ukrainian and Vietnamese. # The default value is: English. OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# If the BRIEF_MEMBER_DESC tag is set to YES, Doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. # The default value is: YES. BRIEF_MEMBER_DESC = YES -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# If the REPEAT_BRIEF tag is set to YES, Doxygen will prepend the brief # description of a member or function before the detailed description # # Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the @@ -129,13 +149,13 @@ REPEAT_BRIEF = YES ABBREVIATE_BRIEF = # If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief +# Doxygen will generate a detailed section even if there is only a brief # description. # The default value is: NO. ALWAYS_DETAILED_SEC = NO -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# If the INLINE_INHERITED_MEMB tag is set to YES, Doxygen will show all # inherited members of a class in the documentation of that class as if those # members were ordinary class members. Constructors, destructors and assignment # operators of the base classes will not be shown. @@ -143,7 +163,7 @@ ALWAYS_DETAILED_SEC = NO INLINE_INHERITED_MEMB = NO -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# If the FULL_PATH_NAMES tag is set to YES, Doxygen will prepend the full path # before files name in the file list and in the header files. If set to NO the # shortest path that makes the file name unique will be used # The default value is: YES. @@ -153,11 +173,11 @@ FULL_PATH_NAMES = YES # The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. # Stripping is only done if one of the specified strings matches the left-hand # part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to +# If left blank the directory from which Doxygen is run is used as the path to # strip. # # Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. +# will be relative from the directory where Doxygen is started. # This tag requires that the tag FULL_PATH_NAMES is set to YES. STRIP_FROM_PATH = @@ -171,41 +191,42 @@ STRIP_FROM_PATH = STRIP_FROM_INC_PATH = -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't +# If the SHORT_NAMES tag is set to YES, Doxygen will generate much shorter (but +# less readable) file names. This can be useful if your file system doesn't # support long names like on DOS, Mac, or CD-ROM. # The default value is: NO. SHORT_NAMES = NO -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen will interpret the +# first line (until the first dot, question mark or exclamation mark) of a +# Javadoc-style comment as the brief description. If set to NO, the Javadoc- +# style will behave just like regular Qt-style comments (thus requiring an +# explicit @brief command for a brief description.) # The default value is: NO. JAVADOC_AUTOBRIEF = NO -# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# If the JAVADOC_BANNER tag is set to YES then Doxygen will interpret a line # such as # /*************** # as being the beginning of a Javadoc-style comment "banner". If set to NO, the # Javadoc-style will behave just like regular comments and it will not be -# interpreted by doxygen. +# interpreted by Doxygen. # The default value is: NO. JAVADOC_BANNER = NO -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will interpret the first +# line (until the first dot, question mark or exclamation mark) of a Qt-style +# comment as the brief description. If set to NO, the Qt-style will behave just +# like regular Qt-style comments (thus requiring an explicit \brief command for +# a brief description.) # The default value is: NO. QT_AUTOBRIEF = NO -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen treat a # multi-line C++ special comment block (i.e. a block of //! or /// comments) as # a brief description. This used to be the default behavior. The new default is # to treat a multi-line C++ comment block as a detailed description. Set this @@ -217,10 +238,10 @@ QT_AUTOBRIEF = NO MULTILINE_CPP_IS_BRIEF = NO -# By default Python docstrings are displayed as preformatted text and doxygen's +# By default Python docstrings are displayed as preformatted text and Doxygen's # special commands cannot be used. By setting PYTHON_DOCSTRING to NO the -# doxygen's special commands can be used and the contents of the docstring -# documentation blocks is shown as doxygen documentation. +# Doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as Doxygen documentation. # The default value is: YES. PYTHON_DOCSTRING = YES @@ -231,7 +252,7 @@ PYTHON_DOCSTRING = YES INHERIT_DOCS = YES -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# If the SEPARATE_MEMBER_PAGES tag is set to YES then Doxygen will produce a new # page for each member. If set to NO, the documentation of a member will be part # of the file/class/namespace that contains it. # The default value is: NO. @@ -248,16 +269,16 @@ TAB_SIZE = 4 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) ALIASES = @@ -301,28 +322,31 @@ OPTIMIZE_OUTPUT_SLICE = NO # parses. With this tag you can assign which parser to use for a given # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# language is one of the parsers supported by Doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files). For instance to make doxygen treat .inc files +# default for Fortran type files). For instance to make Doxygen treat .inc files # as Fortran files (default is PHP), and .f files as C (default is Fortran), # use: inc=Fortran f=C. # # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by Doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = cu=C++ \ cuh=C++ -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# If the MARKDOWN_SUPPORT tag is enabled then Doxygen pre-processes all comments # according to the Markdown format, which allows for more readable # documentation. See https://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# The output of markdown processing is further processed by Doxygen, so you can +# mix Doxygen, HTML, and XML commands with Markdown formatting. Disable only in # case of backward compatibilities issues. # The default value is: YES. @@ -332,25 +356,45 @@ MARKDOWN_SUPPORT = YES # to that level are automatically included in the table of contents, even if # they do not have an id attribute. # Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 5. +# Minimum value: 0, maximum value: 99, default value: 6. # This tag requires that the tag MARKDOWN_SUPPORT is set to YES. TOC_INCLUDE_HEADINGS = 5 -# When enabled doxygen tries to link words that correspond to documented +# The MARKDOWN_ID_STYLE tag can be used to specify the algorithm used to +# generate identifiers for the Markdown headings. Note: Every identifier is +# unique. +# Possible values are: DOXYGEN use a fixed 'autotoc_md' string followed by a +# sequence number starting at 0 and GITHUB use the lower case version of title +# with any whitespace replaced by '-' and punctuation characters removed. +# The default value is: DOXYGEN. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +MARKDOWN_ID_STYLE = DOXYGEN + +# When enabled Doxygen tries to link words that correspond to documented # classes, or namespaces to their corresponding documentation. Such a link can # be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. +# globally by setting AUTOLINK_SUPPORT to NO. Words listed in the +# AUTOLINK_IGNORE_WORDS tag are excluded from automatic linking. # The default value is: YES. AUTOLINK_SUPPORT = YES +# This tag specifies a list of words that, when matching the start of a word in +# the documentation, will suppress auto links generation, if it is enabled via +# AUTOLINK_SUPPORT. This list does not affect affect links explicitly created +# using \# or the \link or commands. +# This tag requires that the tag AUTOLINK_SUPPORT is set to YES. + +AUTOLINK_IGNORE_WORDS = + # If you use STL classes (i.e. std::string, std::vector, etc.) but do not want # to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and +# tag to YES in order to let Doxygen match functions declarations and # definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. +# versus func(std::string) {}). This also makes the inheritance and +# collaboration diagrams that involve STL classes more complete and accurate. # The default value is: NO. BUILTIN_STL_SUPPORT = NO @@ -362,16 +406,16 @@ BUILTIN_STL_SUPPORT = NO CPP_CLI_SUPPORT = NO # Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. +# https://www.riverbankcomputing.com/software) sources only. Doxygen will parse +# them like normal C++ but will assume all classes use public instead of private +# inheritance when no explicit protection keyword is present. # The default value is: NO. SIP_SUPPORT = NO # For Microsoft's IDL there are propget and propput attributes to indicate # getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. +# Doxygen to replace the get and set methods by a property in the documentation. # This will only work if the methods are indeed getting or setting a simple # type. If this is not the case, or you want to show the methods anyway, you # should set this option to NO. @@ -380,7 +424,7 @@ SIP_SUPPORT = NO IDL_PROPERTY_SUPPORT = YES # If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first +# tag is set to YES then Doxygen will reuse the documentation of the first # member in the group (if any) for the other members of the group. By default # all members of a group must be documented explicitly. # The default value is: NO. @@ -438,34 +482,42 @@ TYPEDEF_HIDES_STRUCT = NO # The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This # cache is used to resolve symbols given their name and scope. Since this can be # an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The +# code, Doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# Doxygen will become slower. If the cache is too large, memory is wasted. The # cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range # is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest +# symbols. At the end of a run Doxygen will report the cache usage and suggest # the optimal cache size from a speed point of view. # Minimum value: 0, maximum value: 9, default value: 0. LOOKUP_CACHE_SIZE = 0 -# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use -# during processing. When set to 0 doxygen will based this on the number of +# The NUM_PROC_THREADS specifies the number of threads Doxygen is allowed to use +# during processing. When set to 0 Doxygen will based this on the number of # cores available in the system. You can set it explicitly to a value larger # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. NUM_PROC_THREADS = 0 +# If the TIMESTAMP tag is set different from NO then each generated page will +# contain the date or date and time when the page was generated. Setting this to +# NO can help when comparing the output of multiple runs. +# Possible values are: YES, NO, DATETIME and DATE. +# The default value is: NO. + +TIMESTAMP = NO + #--------------------------------------------------------------------------- # Build related configuration options #--------------------------------------------------------------------------- -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# If the EXTRACT_ALL tag is set to YES, Doxygen will assume all entities in # documentation are documented, even if no documentation was available. Private # class members and static file members will be hidden unless the # EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. @@ -524,7 +576,14 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation # section is generated. This option has no effect if EXTRACT_ALL is enabled. @@ -532,22 +591,31 @@ EXTRACT_ANON_NSPACES = NO HIDE_UNDOC_MEMBERS = NO -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all # undocumented classes that are normally visible in the class hierarchy. If set # to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. +# will also hide undocumented C++ concepts if enabled. This option has no effect +# if EXTRACT_ALL is enabled. # The default value is: NO. HIDE_UNDOC_CLASSES = NO -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# If the HIDE_UNDOC_NAMESPACES tag is set to YES, Doxygen will hide all +# undocumented namespaces that are normally visible in the namespace hierarchy. +# If set to NO, these namespaces will be included in the various overviews. This +# option has no effect if EXTRACT_ALL is enabled. +# The default value is: YES. + +HIDE_UNDOC_NAMESPACES = YES + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all friend # declarations. If set to NO, these declarations will be included in the # documentation. # The default value is: NO. HIDE_FRIEND_COMPOUNDS = NO -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any # documentation blocks found inside the body of a function. If set to NO, these # blocks will be appended to the function's detailed documentation block. # The default value is: NO. @@ -561,30 +629,44 @@ HIDE_IN_BODY_DOCS = NO INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# (including Cygwin) and Mac users are advised to set this option to NO. -# The default value is: system dependent. +# With the correct setting of option CASE_SENSE_NAMES Doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and macOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# Possible values are: SYSTEM, NO and YES. +# The default value is: SYSTEM. CASE_SENSE_NAMES = YES -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# If the HIDE_SCOPE_NAMES tag is set to NO then Doxygen will show members with # their full class and namespace scopes in the documentation. If set to YES, the # scope will be hidden. # The default value is: NO. HIDE_SCOPE_NAMES = NO -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then Doxygen will # append additional text to a page's title, such as Class Reference. If set to # YES the compound reference will be hidden. # The default value is: NO. HIDE_COMPOUND_REFERENCE= NO -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + +# If the SHOW_INCLUDE_FILES tag is set to YES then Doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -597,7 +679,7 @@ SHOW_INCLUDE_FILES = YES SHOW_GROUPED_MEMB_INC = NO -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen will list include # files with double quotes in the documentation rather than with sharp brackets. # The default value is: NO. @@ -609,14 +691,14 @@ FORCE_LOCAL_INCLUDES = NO INLINE_INFO = YES -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# If the SORT_MEMBER_DOCS tag is set to YES then Doxygen will sort the # (detailed) documentation of file and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. # The default value is: YES. SORT_MEMBER_DOCS = YES -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# If the SORT_BRIEF_DOCS tag is set to YES then Doxygen will sort the brief # descriptions of file, namespace and class members alphabetically by member # name. If set to NO, the members will appear in declaration order. Note that # this will also influence the order of the classes in the class list. @@ -624,7 +706,7 @@ SORT_MEMBER_DOCS = YES SORT_BRIEF_DOCS = NO -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then Doxygen will sort the # (brief and detailed) documentation of class members so that constructors and # destructors are listed first. If set to NO the constructors will appear in the # respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. @@ -636,7 +718,7 @@ SORT_BRIEF_DOCS = NO SORT_MEMBERS_CTORS_1ST = NO -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# If the SORT_GROUP_NAMES tag is set to YES then Doxygen will sort the hierarchy # of group names into alphabetical order. If set to NO the group names will # appear in their defined order. # The default value is: NO. @@ -653,11 +735,11 @@ SORT_GROUP_NAMES = NO SORT_BY_SCOPE_NAME = NO -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# If the STRICT_PROTO_MATCHING option is enabled and Doxygen fails to do proper # type resolution of all parameters of a function it will reject a match between # the prototype and the implementation of a member function even if there is # only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# simple string match. By disabling STRICT_PROTO_MATCHING Doxygen will still # accept a match between prototype and implementation in such cases. # The default value is: NO. @@ -727,24 +809,25 @@ SHOW_FILES = YES SHOW_NAMESPACES = YES # The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from +# Doxygen should invoke to get the current version for each file (typically from # the version control system). Doxygen will invoke the program by executing (via # popen()) the command command input-file, where command is the value of the # FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file +# by Doxygen. Whatever the program writes to standard output is used as the file # version. For an example see the documentation. FILE_VERSION_FILTER = # The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated +# by Doxygen. The layout file controls the global structure of the generated # output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can +# that represents Doxygen's defaults, run Doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# Note that if you run Doxygen from a directory containing a file called +# DoxygenLayout.xml, Doxygen will parse it automatically even if the LAYOUT_FILE # tag is left empty. LAYOUT_FILE = @@ -759,19 +842,35 @@ LAYOUT_FILE = CITE_BIB_FILES = +# The EXTERNAL_TOOL_PATH tag can be used to extend the search path (PATH +# environment variable) so that external tools such as latex and gs can be +# found. +# Note: Directories specified with EXTERNAL_TOOL_PATH are added in front of the +# path already specified by the PATH variable, and are added in the order +# specified. +# Note: This option is particularly useful for macOS version 14 (Sonoma) and +# higher, when running Doxygen from Doxywizard, because in this case any user- +# defined changes to the PATH are ignored. A typical example on macOS is to set +# EXTERNAL_TOOL_PATH = /Library/TeX/texbin /usr/local/bin +# together with the standard path, the full search path used by doxygen when +# launching external tools will then become +# PATH=/Library/TeX/texbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin + +EXTERNAL_TOOL_PATH = + #--------------------------------------------------------------------------- # Configuration options related to warning and progress messages #--------------------------------------------------------------------------- # The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the +# standard output by Doxygen. If QUIET is set to YES this implies that the # messages are off. # The default value is: NO. QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# generated to standard error (stderr) by Doxygen. If WARNINGS is set to YES # this implies that the warnings are on. # # Tip: Turn warnings on while writing the documentation. @@ -779,49 +878,97 @@ QUIET = NO WARNINGS = YES -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# If the WARN_IF_UNDOCUMENTED tag is set to YES then Doxygen will generate # warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag # will automatically be disabled. # The default value is: YES. WARN_IF_UNDOCUMENTED = YES -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# If the WARN_IF_DOC_ERROR tag is set to YES, Doxygen will generate warnings for +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, Doxygen will warn about incomplete +# function parameter documentation. If set to NO, Doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, Doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = YES -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# If WARN_IF_UNDOC_ENUM_VAL option is set to YES, Doxygen will warn about +# undocumented enumeration values. If set to NO, Doxygen will accept +# undocumented enumeration values. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: NO. + +WARN_IF_UNDOC_ENUM_VAL = NO + +# If WARN_LAYOUT_FILE option is set to YES, Doxygen will warn about issues found +# while parsing the user defined layout file, such as missing or wrong elements. +# See also LAYOUT_FILE for details. If set to NO, problems with the layout file +# will be suppressed. +# The default value is: YES. + +WARN_LAYOUT_FILE = YES + +# If the WARN_AS_ERROR tag is set to YES then Doxygen will immediately stop when +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then Doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the Doxygen process Doxygen will return with a non-zero status. +# If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS_PRINT then Doxygen behaves +# like FAIL_ON_WARNINGS but in case no WARN_LOGFILE is defined Doxygen will not +# write the warning messages in between other messages but write them at the end +# of a run, in case a WARN_LOGFILE is defined the warning messages will be +# besides being in the defined file also be shown at the end of a run, unless +# the WARN_LOGFILE is defined as - i.e. standard output (stdout) in that case +# the behavior will remain as with the setting FAIL_ON_WARNINGS. +# Possible values are: NO, YES, FAIL_ON_WARNINGS and FAIL_ON_WARNINGS_PRINT. # The default value is: NO. WARN_AS_ERROR = YES -# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# The WARN_FORMAT tag determines the format of the warning messages that Doxygen # can produce. The string should contain the $file, $line, and $text tags, which # will be replaced by the file and line number from which the warning originated # and the warning text. Optionally the format may contain $version, which will # be replaced by the version of the file (if it could be obtained via # FILE_VERSION_FILTER) +# See also: WARN_LINE_FORMAT # The default value is: $file:$line: $text. WARN_FORMAT = "$file:$line: $text" +# In the $text part of the WARN_FORMAT command it is possible that a reference +# to a more specific place is given. To make it easier to jump to this place +# (outside of Doxygen) the user can define a custom "cut" / "paste" string. +# Example: +# WARN_LINE_FORMAT = "'vi $file +$line'" +# See also: WARN_FORMAT +# The default value is: at line $line of file $file. + +WARN_LINE_FORMAT = "at line $line of file $file" + # The WARN_LOGFILE tag can be used to specify a file to which warning and error # messages should be written. If left blank the output is written to standard -# error (stderr). +# error (stderr). In case the file specified cannot be opened for writing the +# warning and error messages are written to standard error. When as file - is +# specified the warning and error messages are written to standard output +# (stdout). WARN_LOGFILE = @@ -836,34 +983,49 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = main_page.md \ - ../include + ../include \ + ../../c/include # This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# that Doxygen parses. Internally Doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# See also: INPUT_FILE_ENCODING # The default value is: UTF-8. INPUT_ENCODING = UTF-8 +# This tag can be used to specify the character encoding of the source files +# that Doxygen parses. The INPUT_FILE_ENCODING tag can be used to specify +# character encoding on a per file pattern basis. Doxygen will compare the file +# name with each pattern and apply the encoding instead of the default +# INPUT_ENCODING if there is a match. The character encodings are a list of the +# form: pattern=encoding (like *.php=ISO-8859-1). +# See also: INPUT_ENCODING for further information on supported encodings. + +INPUT_FILE_ENCODING = + # If the value of the INPUT tag contains directories, you can use the # FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and # *.h) to filter out the source-files in the directories. # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. +# read by Doxygen. # -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen -# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, -# *.vhdl, *.ucf, *.qsf and *.ice. +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cxxm, +# *.cpp, *.cppm, *.ccm, *.c++, *.c++m, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, +# *.idl, *.ddl, *.odl, *.h, *.hh, *.hxx, *.hpp, *.h++, *.ixx, *.l, *.cs, *.d, +# *.php, *.php4, *.php5, *.phtml, *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to +# be provided as Doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.hpp \ + *.h \ *.cuh # The RECURSIVE tag can be used to specify whether or not subdirectories should @@ -876,10 +1038,13 @@ RECURSIVE = YES # excluded from the INPUT source files. This way you can easily exclude a # subdirectory from a directory tree whose root is specified with the INPUT tag. # -# Note that relative paths are relative to the directory from which doxygen is +# Note that relative paths are relative to the directory from which Doxygen is # run. -EXCLUDE = ../include/cuvs/sparse/selection +EXCLUDE = ../include/cuvs/sparse/selection \ + ../build \ + ../src + # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -895,7 +1060,6 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -# TODO: remove specializations from exclude patterns when headers have been removed. EXCLUDE_PATTERNS = */detail/* \ */specializations/* \ */thirdparty/* @@ -904,10 +1068,7 @@ EXCLUDE_PATTERNS = */detail/* \ # (namespaces, classes, functions, etc.) that should be excluded from the # output. The symbol name can be a fully qualified name, a word, or if the # wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* +# ANamespace::AClass, ANamespace::*Test EXCLUDE_SYMBOLS = detail @@ -937,7 +1098,7 @@ EXAMPLE_RECURSIVE = NO IMAGE_PATH = -# The INPUT_FILTER tag can be used to specify a program that doxygen should +# The INPUT_FILTER tag can be used to specify a program that Doxygen should # invoke to filter for each input file. Doxygen will invoke the filter program # by executing (via popen()) the command: # @@ -952,9 +1113,14 @@ IMAGE_PATH = # code is scanned, but not when the output code is generated. If lines are added # or removed, the anchors will not be placed correctly. # +# Note that Doxygen will use the data processed and written to standard output +# for further processing, therefore nothing else, like debug statements or used +# commands (so in case of a Windows batch file always use @echo OFF), should be +# written to standard output. +# # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. +# properly processed by Doxygen. INPUT_FILTER = @@ -967,7 +1133,7 @@ INPUT_FILTER = # # Note that for custom extensions or not directly supported extensions you also # need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. +# properly processed by Doxygen. FILTER_PATTERNS = @@ -989,10 +1155,28 @@ FILTER_SOURCE_PATTERNS = # If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that # is part of the input, its contents will be placed on the main page # (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. +# and want to reuse the introduction page also for the Doxygen output. USE_MDFILE_AS_MAINPAGE = main_page.md +# If the IMPLICIT_DIR_DOCS tag is set to YES, any README.md file found in sub- +# directories of the project's root, is used as the documentation for that sub- +# directory, except when the README.md starts with a \dir, \page or \mainpage +# command. If set to NO, the README.md file needs to start with an explicit \dir +# command in order to be used as directory documentation. +# The default value is: YES. + +IMPLICIT_DIR_DOCS = YES + +# The Fortran standard specifies that for fixed formatted Fortran code all +# characters from position 72 are to be considered as comment. A common +# extension is to allow longer lines before the automatic comment starts. The +# setting FORTRAN_COMMENT_AFTER will also make it possible that longer lines can +# be processed before the automatic comment starts. +# Minimum value: 7, maximum value: 10000, default value: 72. + +FORTRAN_COMMENT_AFTER = 72 + #--------------------------------------------------------------------------- # Configuration options related to source browsing #--------------------------------------------------------------------------- @@ -1007,12 +1191,13 @@ USE_MDFILE_AS_MAINPAGE = main_page.md SOURCE_BROWSER = NO # Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. +# multi-line macros, enums or list initialized variables directly into the +# documentation. # The default value is: NO. INLINE_SOURCES = NO -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct Doxygen to hide any # special comment blocks from generated source code fragments. Normal C, C++ and # Fortran comments will always remain visible. # The default value is: YES. @@ -1050,7 +1235,7 @@ REFERENCES_LINK_SOURCE = YES SOURCE_TOOLTIPS = YES # If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# point to the HTML generated by the htags(1) tool instead of Doxygen built-in # source browser. The htags tool is part of GNU's global source tagging system # (see https://www.gnu.org/software/global/global.html). You will need version # 4.8.6 or higher. @@ -1064,14 +1249,14 @@ SOURCE_TOOLTIPS = YES # Doxygen will invoke htags (and that will in turn invoke gtags), so these # tools must be available from the command line (i.e. in the search path). # -# The result: instead of the source browser generated by doxygen, the links to +# The result: instead of the source browser generated by Doxygen, the links to # source code will now point to the output of htags. # The default value is: NO. # This tag requires that the tag SOURCE_BROWSER is set to YES. USE_HTAGS = NO -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# If the VERBATIM_HEADERS tag is set the YES then Doxygen will generate a # verbatim copy of the header file for each class for which an include is # specified. Set to NO to disable this. # See also: Section \class. @@ -1090,17 +1275,11 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. +# The IGNORE_PREFIX tag can be used to specify a prefix (or a list of prefixes) +# that should be ignored while generating the index headers. The IGNORE_PREFIX +# tag works for classes, function and member names. The entity will be placed in +# the alphabetical list under the first letter of the entity name that remains +# after removing the prefix. # This tag requires that the tag ALPHABETICAL_INDEX is set to YES. IGNORE_PREFIX = @@ -1109,7 +1288,7 @@ IGNORE_PREFIX = # Configuration options related to the HTML output #--------------------------------------------------------------------------- -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# If the GENERATE_HTML tag is set to YES, Doxygen will generate HTML output # The default value is: YES. GENERATE_HTML = NO @@ -1130,40 +1309,40 @@ HTML_OUTPUT = html HTML_FILE_EXTENSION = .html # The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a +# each generated HTML page. If the tag is left blank Doxygen will generate a # standard header. # # To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. +# that Doxygen needs, which is dependent on the configuration options used (e.g. # the setting GENERATE_TREEVIEW). It is highly recommended to start with a # default header using # doxygen -w html new_header.html new_footer.html new_stylesheet.css # YourConfigFile # and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally +# for information on how to generate the default header that Doxygen normally # uses. # Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description +# default header when upgrading to a newer version of Doxygen. For a description # of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_HEADER = header.html # The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard +# generated HTML page. If the tag is left blank Doxygen will generate a standard # footer. See HTML_HEADER for more information on how to generate a default # footer and what special commands can be used inside the footer. See also # section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. +# that Doxygen normally uses. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_FOOTER = # The HTML_STYLESHEET tag can be used to specify a user-defined cascading style # sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. +# the HTML output. If left blank Doxygen will generate a default style sheet. # See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. +# sheet that Doxygen normally uses. # Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as # it is more robust and this tag (HTML_STYLESHEET) will in the future become # obsolete. @@ -1173,13 +1352,18 @@ HTML_STYLESHEET = # The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined # cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. +# created by Doxygen. Using this option one can overrule certain style aspects. # This is preferred over using HTML_STYLESHEET since it does not replace the # standard style sheet and is therefore more robust against future updates. # Doxygen will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. +# list). +# Note: Since the styling of scrollbars can currently not be overruled in +# Webkit/Chromium, the styling will be left out of the default doxygen.css if +# one or more extra stylesheets have been specified. So if scrollbar +# customization is desired it has to be added explicitly. For an example see the +# documentation. # This tag requires that the tag GENERATE_HTML is set to YES. HTML_EXTRA_STYLESHEET = @@ -1194,9 +1378,22 @@ HTML_EXTRA_STYLESHEET = HTML_EXTRA_FILES = +# The HTML_COLORSTYLE tag can be used to specify if the generated HTML output +# should be rendered with a dark or light theme. +# Possible values are: LIGHT always generates light mode output, DARK always +# generates dark mode output, AUTO_LIGHT automatically sets the mode according +# to the user preference, uses light mode if no preference is set (the default), +# AUTO_DARK automatically sets the mode according to the user preference, uses +# dark mode if no preference is set and TOGGLE allows a user to switch between +# light and dark mode via a button. +# The default value is: AUTO_LIGHT. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE = AUTO_LIGHT + # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1206,7 +1403,7 @@ HTML_EXTRA_FILES = HTML_COLORSTYLE_HUE = 266 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1224,15 +1421,6 @@ HTML_COLORSTYLE_SAT = 255 HTML_COLORSTYLE_GAMMA = 52 -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = NO - # If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML # documentation will contain a main index with vertical navigation menus that # are dynamically created via JavaScript. If disabled, the navigation index will @@ -1252,6 +1440,33 @@ HTML_DYNAMIC_MENUS = YES HTML_DYNAMIC_SECTIONS = NO +# If the HTML_CODE_FOLDING tag is set to YES then classes and functions can be +# dynamically folded and expanded in the generated HTML source code. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_CODE_FOLDING = YES + +# If the HTML_COPY_CLIPBOARD tag is set to YES then Doxygen will show an icon in +# the top right corner of code and text fragments that allows the user to copy +# its content to the clipboard. Note this only works if supported by the browser +# and the web page is served via a secure context (see: +# https://www.w3.org/TR/secure-contexts/), i.e. using the https: or file: +# protocol. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COPY_CLIPBOARD = YES + +# Doxygen stores a couple of settings persistently in the browser (via e.g. +# cookies). By default these settings apply to all HTML pages generated by +# Doxygen across all projects. The HTML_PROJECT_COOKIE tag can be used to store +# the settings under a project specific key, such that the user preferences will +# be stored separately. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_PROJECT_COOKIE = + # With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries # shown in the various tree structured indices initially; the user can expand # and collapse entries dynamically later on. Doxygen will expand the tree to @@ -1267,10 +1482,11 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, Doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. @@ -1287,6 +1503,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1309,14 +1532,18 @@ DOCSET_PUBLISHER_ID = org.doxygen.Publisher DOCSET_PUBLISHER_NAME = Publisher -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# If the GENERATE_HTMLHELP tag is set to YES then Doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline (the HTML help workshop was already many +# years in maintenance mode). You can download the HTML help workshop from the +# web archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# generated by Doxygen into a single compiled HTML file (.chm). Compiled HTML # files are now used as the Windows 98 help format, and will replace the old # Windows help format (.hlp) on all Windows platforms in the future. Compressed # HTML files also contain an index, a table of contents, and you can search for @@ -1336,7 +1563,7 @@ CHM_FILE = # The HHC_LOCATION tag can be used to specify the location (absolute path # including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. +# Doxygen will try to run the HTML help compiler on the generated index.hhp. # The file has to be specified with full path. # This tag requires that the tag GENERATE_HTMLHELP is set to YES. @@ -1370,6 +1597,16 @@ BINARY_TOC = NO TOC_EXPAND = NO +# The SITEMAP_URL tag is used to specify the full URL of the place where the +# generated documentation will be placed on the server by the user during the +# deployment of the documentation. The generated sitemap is called sitemap.xml +# and placed on the directory specified by HTML_OUTPUT. In case no SITEMAP_URL +# is specified no sitemap is generated. For information about the sitemap +# protocol see https://www.sitemaps.org +# This tag requires that the tag GENERATE_HTML is set to YES. + +SITEMAP_URL = + # If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and # QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that # can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help @@ -1388,7 +1625,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1396,8 +1634,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1405,16 +1643,16 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = @@ -1426,9 +1664,9 @@ QHP_CUST_FILTER_ATTRS = QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty Doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1459,7 +1697,7 @@ ECLIPSE_DOC_ID = org.doxygen.Project # of each HTML page. A value of NO enables the index and the value YES disables # it. Since the tabs in the index contain the same information as the navigation # tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. +# The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. DISABLE_INDEX = NO @@ -1471,18 +1709,30 @@ DISABLE_INDEX = NO # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by Doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. +# The default value is: YES. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = NO +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. +# Doxygen will group on one line in the generated HTML documentation. # # Note that a value of 0 will completely suppress the enum values from appearing # in the overview section. @@ -1491,6 +1741,12 @@ GENERATE_TREEVIEW = NO ENUM_VALUES_PER_LINE = 4 +# When the SHOW_ENUM_VALUES tag is set doxygen will show the specified +# enumeration values besides the enumeration mnemonics. +# The default value is: NO. + +SHOW_ENUM_VALUES = NO + # If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used # to set the initial width (in pixels) of the frame in which the tree is shown. # Minimum value: 0, maximum value: 1500, default value: 250. @@ -1498,14 +1754,21 @@ ENUM_VALUES_PER_LINE = 4 TREEVIEW_WIDTH = 250 -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# If the EXT_LINKS_IN_WINDOW option is set to YES, Doxygen will open links to # external symbols imported via tag files in a separate window. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. EXT_LINKS_IN_WINDOW = NO -# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# If the OBFUSCATE_EMAILS tag is set to YES, Doxygen will obfuscate email +# addresses. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +OBFUSCATE_EMAILS = YES + +# If the HTML_FORMULA_FORMAT option is set to svg, Doxygen will use the pdf2svg # tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see # https://inkscape.org) to generate formulas as SVG images instead of PNGs for # the HTML output. These images will generally look nicer at scaled resolutions. @@ -1518,24 +1781,13 @@ HTML_FORMULA_FORMAT = png # Use this tag to change the font size of LaTeX formulas included as images in # the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML +# Doxygen run you need to manually remove any form_*.png images from the HTML # output directory to force them to be regenerated. # Minimum value: 8, maximum value: 50, default value: 10. # This tag requires that the tag GENERATE_HTML is set to YES. FORMULA_FONTSIZE = 10 -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - # The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands # to create new LaTeX commands to be used in formulas as building blocks. See # the section "Including formulas" for details. @@ -1553,11 +1805,29 @@ FORMULA_MACROFILE = USE_MATHJAX = YES +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for MathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1570,33 +1840,40 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see +# https://docs.mathjax.org/en/v2.7-latest/tex.html#tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# The MATHJAX_CODEFILE tag can be used to specify a file with JavaScript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_CODEFILE = -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and +# When the SEARCHENGINE tag is enabled Doxygen will generate a search box for +# the HTML output. The underlying search engine uses JavaScript and DHTML and # should work on any modern browser. Note that when using HTML help # (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) # there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then +# For large projects the JavaScript based search engine can be slow, then # enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to # search using the keyboard; to jump to the search box use + S # (what the is depends on the OS and browser, but it is typically @@ -1615,7 +1892,7 @@ SEARCHENGINE = YES # When the SERVER_BASED_SEARCH tag is enabled the search engine will be # implemented using a web server instead of a web client using JavaScript. There # are two flavors of web server based searching depending on the EXTERNAL_SEARCH -# setting. When disabled, doxygen will generate a PHP script for searching and +# setting. When disabled, Doxygen will generate a PHP script for searching and # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing # and searching needs to be provided by external tools. See the section # "External Indexing and Searching" for details. @@ -1624,7 +1901,7 @@ SEARCHENGINE = YES SERVER_BASED_SEARCH = NO -# When EXTERNAL_SEARCH tag is enabled doxygen will no longer generate the PHP +# When EXTERNAL_SEARCH tag is enabled Doxygen will no longer generate the PHP # script for searching. Instead the search results are written to an XML file # which needs to be processed by an external indexer. Doxygen will invoke an # external search engine pointed to by the SEARCHENGINE_URL option to obtain the @@ -1632,7 +1909,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1645,8 +1923,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1667,7 +1946,7 @@ SEARCHDATA_FILE = searchdata.xml EXTERNAL_SEARCH_ID = -# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through doxygen +# The EXTRA_SEARCH_MAPPINGS tag can be used to enable searching through Doxygen # projects other than the one defined by this configuration file, but that are # all added to the same external search index. Each project needs to have a # unique id set via EXTERNAL_SEARCH_ID. The search mapping then maps the id of @@ -1681,7 +1960,7 @@ EXTRA_SEARCH_MAPPINGS = # Configuration options related to the LaTeX output #--------------------------------------------------------------------------- -# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# If the GENERATE_LATEX tag is set to YES, Doxygen will generate LaTeX output. # The default value is: YES. GENERATE_LATEX = NO @@ -1726,7 +2005,7 @@ MAKEINDEX_CMD_NAME = makeindex LATEX_MAKEINDEX_CMD = makeindex -# If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX +# If the COMPACT_LATEX tag is set to YES, Doxygen generates more compact LaTeX # documents. This may be useful for small projects and may help to save some # trees in general. # The default value is: NO. @@ -1755,36 +2034,38 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank Doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that Doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of Doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank Doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that Doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = # The LATEX_EXTRA_STYLESHEET tag can be used to specify additional user-defined # LaTeX style sheets that are included after the standard style sheets created -# by doxygen. Using this option one can overrule certain style aspects. Doxygen +# by Doxygen. Using this option one can overrule certain style aspects. Doxygen # will copy the style sheet files to the output directory. # Note: The order of the extra style sheet files is of importance (e.g. the last # style sheet in the list overrules the setting of the previous ones in the @@ -1810,7 +2091,7 @@ LATEX_EXTRA_FILES = PDF_HYPERLINKS = YES -# If the USE_PDFLATEX tag is set to YES, doxygen will use the engine as +# If the USE_PDFLATEX tag is set to YES, Doxygen will use the engine as # specified with LATEX_CMD_NAME to generate the PDF file directly from the LaTeX # files. Set this option to YES, to get a higher quality PDF documentation. # @@ -1820,48 +2101,36 @@ PDF_HYPERLINKS = YES USE_PDFLATEX = YES -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode -# command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# The LATEX_BATCHMODE tag signals the behavior of LaTeX in case of an error. +# Possible values are: NO same as ERROR_STOP, YES same as BATCH, BATCH In batch +# mode nothing is printed on the terminal, errors are scrolled as if is +# hit at every error; missing files that TeX tries to input or request from +# keyboard input (\read on a not open input stream) cause the job to abort, +# NON_STOP In nonstop mode the diagnostic message will appear on the terminal, +# but there is no possibility of user interaction just like in batch mode, +# SCROLL In scroll mode, TeX will stop only for missing files to input or if +# keyboard input is necessary and ERROR_STOP In errorstop mode, TeX will stop at +# each error, asking for user intervention. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_BATCHMODE = NO -# If the LATEX_HIDE_INDICES tag is set to YES then doxygen will not include the +# If the LATEX_HIDE_INDICES tag is set to YES then Doxygen will not include the # index chapters (such as File Index, Compound Index, etc.) in the output. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. -# The default value is: plain. +# The default value is: plainnat. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_BIB_STYLE = plain -# If the LATEX_TIMESTAMP tag is set to YES then the footer of each generated -# page will contain the date and time when the page was generated. Setting this -# to NO can help when comparing the output of multiple runs. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_TIMESTAMP = NO - # The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute) # path from which the emoji images will be read. If a relative path is entered, # it will be relative to the LATEX_OUTPUT directory. If left blank the @@ -1874,7 +2143,7 @@ LATEX_EMOJI_DIRECTORY = # Configuration options related to the RTF output #--------------------------------------------------------------------------- -# If the GENERATE_RTF tag is set to YES, doxygen will generate RTF output. The +# If the GENERATE_RTF tag is set to YES, Doxygen will generate RTF output. The # RTF output is optimized for Word 97 and may not look too pretty with other RTF # readers/editors. # The default value is: NO. @@ -1889,7 +2158,7 @@ GENERATE_RTF = NO RTF_OUTPUT = rtf -# If the COMPACT_RTF tag is set to YES, doxygen generates more compact RTF +# If the COMPACT_RTF tag is set to YES, Doxygen generates more compact RTF # documents. This may be useful for small projects and may help to save some # trees in general. # The default value is: NO. @@ -1909,38 +2178,36 @@ COMPACT_RTF = NO RTF_HYPERLINKS = NO -# Load stylesheet definitions from file. Syntax is similar to doxygen's +# Load stylesheet definitions from file. Syntax is similar to Doxygen's # configuration file, i.e. a series of assignments. You only have to provide # replacements, missing definitions are set to their default value. # # See also section "Doxygen usage" for information on how to generate the -# default style sheet that doxygen normally uses. +# default style sheet that Doxygen normally uses. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_STYLESHEET_FILE = # Set optional variables used in the generation of an RTF document. Syntax is -# similar to doxygen's configuration file. A template extensions file can be +# similar to Doxygen's configuration file. A template extensions file can be # generated using doxygen -e rtf extensionFile. # This tag requires that the tag GENERATE_RTF is set to YES. RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. +# The RTF_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the RTF_OUTPUT output directory. +# Note that the files will be copied as-is; there are no commands or markers +# available. # This tag requires that the tag GENERATE_RTF is set to YES. -RTF_SOURCE_CODE = NO +RTF_EXTRA_FILES = #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- -# If the GENERATE_MAN tag is set to YES, doxygen will generate man pages for +# If the GENERATE_MAN tag is set to YES, Doxygen will generate man pages for # classes and files. # The default value is: NO. @@ -1971,7 +2238,7 @@ MAN_EXTENSION = .3 MAN_SUBDIR = -# If the MAN_LINKS tag is set to YES and doxygen generates man output, then it +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, then it # will generate one additional man file for each entity documented in the real # man page(s). These additional files only source the real man page, but without # them the man command would be unable to find the correct page. @@ -1984,7 +2251,7 @@ MAN_LINKS = NO # Configuration options related to the XML output #--------------------------------------------------------------------------- -# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that +# If the GENERATE_XML tag is set to YES, Doxygen will generate an XML file that # captures the structure of the code including all documentation. # The default value is: NO. @@ -1998,7 +2265,7 @@ GENERATE_XML = YES XML_OUTPUT = _xml -# If the XML_PROGRAMLISTING tag is set to YES, doxygen will dump the program +# If the XML_PROGRAMLISTING tag is set to YES, Doxygen will dump the program # listings (including syntax highlighting and cross-referencing information) to # the XML output. Note that enabling this will significantly increase the size # of the XML output. @@ -2007,7 +2274,7 @@ XML_OUTPUT = _xml XML_PROGRAMLISTING = YES -# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include +# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, Doxygen will include # namespace members in file scope as well, matching the HTML output. # The default value is: NO. # This tag requires that the tag GENERATE_XML is set to YES. @@ -2018,7 +2285,7 @@ XML_NS_MEMB_FILE_SCOPE = NO # Configuration options related to the DOCBOOK output #--------------------------------------------------------------------------- -# If the GENERATE_DOCBOOK tag is set to YES, doxygen will generate Docbook files +# If the GENERATE_DOCBOOK tag is set to YES, Doxygen will generate Docbook files # that can be used to generate PDF. # The default value is: NO. @@ -2032,32 +2299,49 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- -# If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an -# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures +# If the GENERATE_AUTOGEN_DEF tag is set to YES, Doxygen will generate an +# AutoGen Definitions (see https://autogen.sourceforge.net/) file that captures # the structure of the code including all documentation. Note that this feature # is still experimental and incomplete at the moment. # The default value is: NO. GENERATE_AUTOGEN_DEF = NO +#--------------------------------------------------------------------------- +# Configuration options related to Sqlite3 output +#--------------------------------------------------------------------------- + +# If the GENERATE_SQLITE3 tag is set to YES Doxygen will generate a Sqlite3 +# database with symbols found by Doxygen stored in tables. +# The default value is: NO. + +GENERATE_SQLITE3 = NO + +# The SQLITE3_OUTPUT tag is used to specify where the Sqlite3 database will be +# put. If a relative path is entered the value of OUTPUT_DIRECTORY will be put +# in front of it. +# The default directory is: sqlite3. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_OUTPUT = sqlite3 + +# The SQLITE3_RECREATE_DB tag is set to YES, the existing doxygen_sqlite3.db +# database file will be recreated with each Doxygen run. If set to NO, Doxygen +# will warn if a database file is already found and not modify it. +# The default value is: YES. +# This tag requires that the tag GENERATE_SQLITE3 is set to YES. + +SQLITE3_RECREATE_DB = YES + #--------------------------------------------------------------------------- # Configuration options related to the Perl module output #--------------------------------------------------------------------------- -# If the GENERATE_PERLMOD tag is set to YES, doxygen will generate a Perl module +# If the GENERATE_PERLMOD tag is set to YES, Doxygen will generate a Perl module # file that captures the structure of the code including all documentation. # # Note that this feature is still experimental and incomplete at the moment. @@ -2065,7 +2349,7 @@ GENERATE_AUTOGEN_DEF = NO GENERATE_PERLMOD = NO -# If the PERLMOD_LATEX tag is set to YES, doxygen will generate the necessary +# If the PERLMOD_LATEX tag is set to YES, Doxygen will generate the necessary # Makefile rules, Perl scripts and LaTeX code to be able to generate PDF and DVI # output from the Perl module output. # The default value is: NO. @@ -2095,13 +2379,13 @@ PERLMOD_MAKEVAR_PREFIX = # Configuration options related to the preprocessor #--------------------------------------------------------------------------- -# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all +# If the ENABLE_PREPROCESSING tag is set to YES, Doxygen will evaluate all # C-preprocessor directives found in the sources and include files. # The default value is: YES. ENABLE_PREPROCESSING = YES -# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# If the MACRO_EXPANSION tag is set to YES, Doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be # performed. Macro expansion can be done in a controlled way by setting # EXPAND_ONLY_PREDEF to YES. @@ -2127,7 +2411,8 @@ SEARCH_INCLUDES = YES # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the -# preprocessor. +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @@ -2148,7 +2433,8 @@ INCLUDE_FILE_PATTERNS = # recursively expanded use the := operator instead of the = operator. # This tag requires that the tag ENABLE_PREPROCESSING is set to YES. -PREDEFINED = +PREDEFINED = "CUVS_BUILD_MG_ALGOS=1" \ + "CUVS_BUILD_CAGRA_HNSWLIB=1" # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this # tag can be used to specify a list of macro names that should be expanded. The @@ -2159,7 +2445,7 @@ PREDEFINED = EXPAND_AS_DEFINED = -# If the SKIP_FUNCTION_MACROS tag is set to YES then doxygen's preprocessor will +# If the SKIP_FUNCTION_MACROS tag is set to YES then Doxygen's preprocessor will # remove all references to function-like macros that are alone on a line, have # an all uppercase name, and do not end with a semicolon. Such function macros # are typically used for boiler-plate code, and will confuse the parser if not @@ -2183,26 +2469,26 @@ SKIP_FUNCTION_MACROS = YES # section "Linking to external documentation" for more information about the use # of tag files. # Note: Each tag file must have a unique name (where the name does NOT include -# the path). If a tag file is not located in the directory in which doxygen is +# the path). If a tag file is not located in the directory in which Doxygen is # run, you must also specify the path to the tagfile here. TAGFILES = -# When a file name is specified after GENERATE_TAGFILE, doxygen will create a +# When a file name is specified after GENERATE_TAGFILE, Doxygen will create a # tag file that is based on the input files it reads. See section "Linking to # external documentation" for more information about the usage of tag files. GENERATE_TAGFILE = -# If the ALLEXTERNALS tag is set to YES, all external class will be listed in -# the class index. If set to NO, only the inherited external classes will be -# listed. +# If the ALLEXTERNALS tag is set to YES, all external classes and namespaces +# will be listed in the class and namespace index. If set to NO, only the +# inherited external classes will be listed. # The default value is: NO. ALLEXTERNALS = NO # If the EXTERNAL_GROUPS tag is set to YES, all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will be +# in the topic index. If set to NO, only the current project's groups will be # listed. # The default value is: YES. @@ -2216,42 +2502,26 @@ EXTERNAL_GROUPS = YES EXTERNAL_PAGES = YES #--------------------------------------------------------------------------- -# Configuration options related to the dot tool +# Configuration options related to diagram generator tools #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = YES - -# You can include diagrams made with dia in doxygen documentation. Doxygen will -# then run dia to produce the diagram and insert it in the documentation. The -# DIA_PATH tag allows you to specify the directory where the dia binary resides. -# If left empty dia is assumed to be found in the default search path. - -DIA_PATH = - # If set to YES the inheritance and collaboration graphs will hide inheritance # and usage relations if the target is undocumented or is not a class. # The default value is: YES. HIDE_UNDOC_RELATIONS = YES -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# If you set the HAVE_DOT tag to YES then Doxygen will assume the dot tool is # available from the path. This tool is part of Graphviz (see: -# http://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent +# https://www.graphviz.org/), a graph visualization toolkit from AT&T and Lucent # Bell Labs. The other options in this section have no effect if this option is # set to NO # The default value is: NO. -HAVE_DOT = +HAVE_DOT = NO -# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is allowed -# to run in parallel. When set to 0 doxygen will base this on the number of +# The DOT_NUM_THREADS specifies the number of dot invocations Doxygen is allowed +# to run in parallel. When set to 0 Doxygen will base this on the number of # processors available in the system. You can set it explicitly to a value # larger than 0 to get control over the balance between CPU load and processing # speed. @@ -2260,55 +2530,83 @@ HAVE_DOT = DOT_NUM_THREADS = 0 -# When you want a differently looking font in the dot files that doxygen -# generates you can specify the font name using DOT_FONTNAME. You need to make -# sure dot is able to find the font, which can be done by putting it in a -# standard location or by setting the DOTFONTPATH environment variable or by -# setting DOT_FONTPATH to the directory containing the font. -# The default value is: Helvetica. +# DOT_COMMON_ATTR is common attributes for nodes, edges and labels of +# subgraphs. When you want a differently looking font in the dot files that +# Doxygen generates you can specify fontname, fontcolor and fontsize attributes. +# For details please see Node, +# Edge and Graph Attributes specification You need to make sure dot is able +# to find the font, which can be done by putting it in a standard location or by +# setting the DOTFONTPATH environment variable or by setting DOT_FONTPATH to the +# directory containing the font. Default graphviz fontsize is 14. +# The default value is: fontname=Helvetica,fontsize=10. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_COMMON_ATTR = "fontname=Helvetica,fontsize=10" + +# DOT_EDGE_ATTR is concatenated with DOT_COMMON_ATTR. For elegant style you can +# add 'arrowhead=open, arrowtail=open, arrowsize=0.5'. Complete documentation about +# arrows shapes. +# The default value is: labelfontname=Helvetica,labelfontsize=10. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTNAME = Helvetica +DOT_EDGE_ATTR = "labelfontname=Helvetica,labelfontsize=10" -# The DOT_FONTSIZE tag can be used to set the size (in points) of the font of -# dot graphs. -# Minimum value: 4, maximum value: 24, default value: 10. +# DOT_NODE_ATTR is concatenated with DOT_COMMON_ATTR. For view without boxes +# around nodes set 'shape=plain' or 'shape=plaintext' Shapes specification +# The default value is: shape=box,height=0.2,width=0.4. # This tag requires that the tag HAVE_DOT is set to YES. -DOT_FONTSIZE = 10 +DOT_NODE_ATTR = "shape=box,height=0.2,width=0.4" -# By default doxygen will tell dot to use the default font as specified with -# DOT_FONTNAME. If you specify a different font using DOT_FONTNAME you can set -# the path where dot can find it using this tag. +# You can set the path where dot can find font specified with fontname in +# DOT_COMMON_ATTR and others dot attributes. # This tag requires that the tag HAVE_DOT is set to YES. DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES or GRAPH or BUILTIN then Doxygen will +# generate a graph for each documented class showing the direct and indirect +# inheritance relations. In case the CLASS_GRAPH tag is set to YES or GRAPH and +# HAVE_DOT is enabled as well, then dot will be used to draw the graph. In case +# the CLASS_GRAPH tag is set to YES and HAVE_DOT is disabled or if the +# CLASS_GRAPH tag is set to BUILTIN, then the built-in generator will be used. +# If the CLASS_GRAPH tag is set to TEXT the direct and indirect inheritance +# relations will be shown as texts / links. Explicit enabling an inheritance +# graph or choosing a different representation for an inheritance graph of a +# specific class, can be accomplished by means of the command \inheritancegraph. +# Disabling an inheritance graph can be accomplished by means of the command +# \hideinheritancegraph. +# Possible values are: NO, YES, TEXT, GRAPH and BUILTIN. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES -# If the COLLABORATION_GRAPH tag is set to YES then doxygen will generate a +# If the COLLABORATION_GRAPH tag is set to YES then Doxygen will generate a # graph for each documented class showing the direct and indirect implementation # dependencies (inheritance, containment, and class references variables) of the -# class with other documented classes. +# class with other documented classes. Explicit enabling a collaboration graph, +# when COLLABORATION_GRAPH is set to NO, can be accomplished by means of the +# command \collaborationgraph. Disabling a collaboration graph can be +# accomplished by means of the command \hidecollaborationgraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. COLLABORATION_GRAPH = YES -# If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for -# groups, showing the direct groups dependencies. +# If the GROUP_GRAPHS tag is set to YES then Doxygen will generate a graph for +# groups, showing the direct groups dependencies. Explicit enabling a group +# dependency graph, when GROUP_GRAPHS is set to NO, can be accomplished by means +# of the command \groupgraph. Disabling a directory graph can be accomplished by +# means of the command \hidegroupgraph. See also the chapter Grouping in the +# manual. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GROUP_GRAPHS = YES -# If the UML_LOOK tag is set to YES, doxygen will generate inheritance and +# If the UML_LOOK tag is set to YES, Doxygen will generate inheritance and # collaboration diagrams in a style similar to the OMG's Unified Modeling # Language. # The default value is: NO. @@ -2325,10 +2623,32 @@ UML_LOOK = YES # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, Doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, Doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, Doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will be wrapped across multiple lines. Some heuristics are +# applied to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2338,24 +2658,29 @@ UML_LIMIT_NUM_FIELDS = 10 TEMPLATE_RELATIONS = NO # If the INCLUDE_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are set to -# YES then doxygen will generate a graph for each documented file showing the +# YES then Doxygen will generate a graph for each documented file showing the # direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an include graph, when INCLUDE_GRAPH is is set to NO, +# can be accomplished by means of the command \includegraph. Disabling an +# include graph can be accomplished by means of the command \hideincludegraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. INCLUDE_GRAPH = YES # If the INCLUDED_BY_GRAPH, ENABLE_PREPROCESSING and SEARCH_INCLUDES tags are -# set to YES then doxygen will generate a graph for each documented file showing +# set to YES then Doxygen will generate a graph for each documented file showing # the direct and indirect include dependencies of the file with other documented -# files. +# files. Explicit enabling an included by graph, when INCLUDED_BY_GRAPH is set +# to NO, can be accomplished by means of the command \includedbygraph. Disabling +# an included by graph can be accomplished by means of the command +# \hideincludedbygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. INCLUDED_BY_GRAPH = YES -# If the CALL_GRAPH tag is set to YES then doxygen will generate a call +# If the CALL_GRAPH tag is set to YES then Doxygen will generate a call # dependency graph for every global function or class method. # # Note that enabling this option will significantly increase the time of a run. @@ -2367,7 +2692,7 @@ INCLUDED_BY_GRAPH = YES CALL_GRAPH = NO -# If the CALLER_GRAPH tag is set to YES then doxygen will generate a caller +# If the CALLER_GRAPH tag is set to YES then Doxygen will generate a caller # dependency graph for every global function or class method. # # Note that enabling this option will significantly increase the time of a run. @@ -2379,44 +2704,59 @@ CALL_GRAPH = NO CALLER_GRAPH = NO -# If the GRAPHICAL_HIERARCHY tag is set to YES then doxygen will graphical +# If the GRAPHICAL_HIERARCHY tag is set to YES then Doxygen will graphical # hierarchy of all classes instead of a textual one. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GRAPHICAL_HIERARCHY = YES -# If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the +# If the DIRECTORY_GRAPH tag is set to YES then Doxygen will show the # dependencies a directory has on other directories in a graphical way. The # dependency relations are determined by the #include relations between the -# files in the directories. +# files in the directories. Explicit enabling a directory graph, when +# DIRECTORY_GRAPH is set to NO, can be accomplished by means of the command +# \directorygraph. Disabling a directory graph can be accomplished by means of +# the command \hidedirectorygraph. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: -# http://www.graphviz.org/)). -# Note: If you choose svg you need to set HTML_FILE_EXTENSION to xhtml in order -# to make the SVG files visible in IE 9+ (other browsers do not have this -# requirement). +# https://www.graphviz.org/)). +# +# Note the formats svg:cairo and svg:cairo:cairo cannot be used in combination +# with INTERACTIVE_SVG (the INTERACTIVE_SVG will be set to NO). # Possible values are: png, jpg, gif, svg, png:gd, png:gd:gd, png:cairo, -# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus and -# png:gdiplus:gdiplus. +# png:cairo:gd, png:cairo:cairo, png:cairo:gdiplus, png:gdiplus, +# png:gdiplus:gdiplus, svg:cairo, svg:cairo:cairo, svg:svg, svg:svg:core, +# gif:cairo, gif:cairo:gd, gif:cairo:gdiplus, gif:gdiplus, gif:gdiplus:gdiplus, +# gif:gd, gif:gd:gd, jpg:cairo, jpg:cairo:gd, jpg:cairo:gdiplus, jpg:gd, +# jpg:gd:gd, jpg:gdiplus and jpg:gdiplus:gdiplus. # The default value is: png. # This tag requires that the tag HAVE_DOT is set to YES. DOT_IMAGE_FORMAT = png -# If DOT_IMAGE_FORMAT is set to svg, then this option can be set to YES to -# enable generation of interactive SVG images that allow zooming and panning. +# If DOT_IMAGE_FORMAT is set to svg or svg:svg or svg:svg:core, then this option +# can be set to YES to enable generation of interactive SVG images that allow +# zooming and panning. # # Note that this requires a modern browser other than Internet Explorer. Tested # and working are Firefox, Chrome, Safari, and Opera. -# Note: For IE 9+ you need to set HTML_FILE_EXTENSION to xhtml in order to make -# the SVG files visible. Older versions of IE do not have SVG support. +# +# Note This option will be automatically disabled when DOT_IMAGE_FORMAT is set +# to svg:cairo or svg:cairo:cairo. # The default value is: NO. # This tag requires that the tag HAVE_DOT is set to YES. @@ -2435,11 +2775,12 @@ DOT_PATH = DOTFILE_DIRS = -# The MSCFILE_DIRS tag can be used to specify one or more directories that -# contain msc files that are included in the documentation (see the \mscfile -# command). +# You can include diagrams made with dia in Doxygen documentation. Doxygen will +# then run dia to produce the diagram and insert it in the documentation. The +# DIA_PATH tag allows you to specify the directory where the dia binary resides. +# If left empty dia is assumed to be found in the default search path. -MSCFILE_DIRS = +DIA_PATH = # The DIAFILE_DIRS tag can be used to specify one or more directories that # contain dia files that are included in the documentation (see the \diafile @@ -2447,28 +2788,34 @@ MSCFILE_DIRS = DIAFILE_DIRS = -# When using plantuml, the PLANTUML_JAR_PATH tag should be used to specify the -# path where java can find the plantuml.jar file. If left blank, it is assumed -# PlantUML is not used or called during a preprocessing step. Doxygen will -# generate a warning when it encounters a \startuml command in this case and -# will not generate output for the diagram. +# When using PlantUML, the PLANTUML_JAR_PATH tag should be used to specify the +# path where java can find the plantuml.jar file or to the filename of jar file +# to be used. If left blank, it is assumed PlantUML is not used or called during +# a preprocessing step. Doxygen will generate a warning when it encounters a +# \startuml command in this case and will not generate output for the diagram. PLANTUML_JAR_PATH = -# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a -# configuration file for plantuml. +# When using PlantUML, the PLANTUML_CFG_FILE tag can be used to specify a +# configuration file for PlantUML. PLANTUML_CFG_FILE = -# When using plantuml, the specified paths are searched for files specified by -# the !include statement in a plantuml block. +# When using PlantUML, the specified paths are searched for files specified by +# the !include statement in a PlantUML block. PLANTUML_INCLUDE_PATH = +# The PLANTUMLFILE_DIRS tag can be used to specify one or more directories that +# contain PlantUml files that are included in the documentation (see the +# \plantumlfile command). + +PLANTUMLFILE_DIRS = + # The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of nodes # that will be shown in the graph. If the number of nodes in a graph becomes -# larger than this value, doxygen will truncate the graph, which is visualized -# by representing a node as a red box. Note that doxygen if the number of direct +# larger than this value, Doxygen will truncate the graph, which is visualized +# by representing a node as a red box. Note that if the number of direct # children of the root node in a graph is already larger than # DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note that # the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. @@ -2489,18 +2836,6 @@ DOT_GRAPH_MAX_NODES = 100 MAX_DOT_GRAPH_DEPTH = 0 -# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent -# background. This is disabled by default, because dot on Windows does not seem -# to support this out of the box. -# -# Warning: Depending on the platform used, enabling this option may lead to -# badly anti-aliased labels on the edges of a graph (i.e. they become hard to -# read). -# The default value is: NO. -# This tag requires that the tag HAVE_DOT is set to YES. - -DOT_TRANSPARENT = NO - # Set the DOT_MULTI_TARGETS tag to YES to allow dot to generate multiple output # files in one run (i.e. multiple -o and -T options on the command line). This # makes dot run faster, but since only newer versions of dot (>1.8.10) support @@ -2510,17 +2845,37 @@ DOT_TRANSPARENT = NO DOT_MULTI_TARGETS = NO -# If the GENERATE_LEGEND tag is set to YES doxygen will generate a legend page +# If the GENERATE_LEGEND tag is set to YES Doxygen will generate a legend page # explaining the meaning of the various boxes and arrows in the dot generated # graphs. +# Note: This tag requires that UML_LOOK isn't set, i.e. the Doxygen internal +# graphical representation for inheritance and collaboration diagrams is used. # The default value is: YES. # This tag requires that the tag HAVE_DOT is set to YES. GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, Doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES + +# You can define message sequence charts within Doxygen comments using the \msc +# command. If the MSCGEN_TOOL tag is left empty (the default), then Doxygen will +# use a built-in version of mscgen tool to produce the charts. Alternatively, +# the MSCGEN_TOOL tag can also specify the name an external tool. For instance, +# specifying prog as the value, Doxygen will call the tool as prog -T +# -o . The external tool should support +# output file formats "png", "eps", "svg", and "ismap". + +MSCGEN_TOOL = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the \mscfile +# command). + +MSCFILE_DIRS = diff --git a/cpp/doxygen/main_page.md b/cpp/doxygen/main_page.md index d002df0017..81757f5dbd 100644 --- a/cpp/doxygen/main_page.md +++ b/cpp/doxygen/main_page.md @@ -1,3 +1,3 @@ # libcuvs -cuVS is a library for vector search on the GPU \ No newline at end of file +cuVS is a library for vector search on the GPU diff --git a/cpp/include/cuvs/cluster/agglomerative.hpp b/cpp/include/cuvs/cluster/agglomerative.hpp new file mode 100644 index 0000000000..a4b333a9af --- /dev/null +++ b/cpp/include/cuvs/cluster/agglomerative.hpp @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2021-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace cuvs::cluster::agglomerative { + +// constant to indirectly control the number of neighbors. k = sqrt(n) + c. default to 15 +constexpr int DEFAULT_CONST_C = 15; + +/** + * @defgroup agglomerative_params agglomerative clustering hyperparameters + * @{ + */ + +/** + * Determines the method for computing the minimum spanning tree (MST) + */ +enum Linkage { + + /** + * Use a pairwise distance matrix as input to the mst. This + * is very fast and the best option for fairly small datasets (~50k data points) + */ + PAIRWISE = 0, + + /** + * Construct a KNN graph as input to the mst and provide additional + * edges if the mst does not converge. This is slower but scales + * to very large datasets. + */ + KNN_GRAPH = 1 +}; + +/** + * @} + */ + +/** + * Simple container object for consolidating linkage results. This closely + * mirrors the trained instance variables populated in + * Scikit-learn's AgglomerativeClustering estimator. + * @tparam idx_t + */ +template +class single_linkage_output { + public: + idx_t m; + idx_t n_clusters; + + idx_t n_leaves; + idx_t n_connected_components; + + // TODO: These will be made private in a future release + idx_t* labels; // size: m + idx_t* children; // size: (m-1, 2) + + raft::device_vector_view get_labels() + { + return raft::make_device_vector_view(labels, m); + } + + raft::device_matrix_view get_children() + { + return raft::make_device_matrix_view(children, m - 1, 2); + } +}; + +/** + * @defgroup single_linkage single-linkage clustering APIs + * @{ + */ +/** + * Single-linkage clustering, capable of constructing a KNN graph to + * scale the algorithm beyond the n^2 memory consumption of implementations + * that use the fully-connected graph of pairwise distances by connecting + * a knn graph when k is not large enough to connect it. + + * @param[in] handle raft handle + * @param[in] X dense input matrix in row-major layout + * @param[out] dendrogram output dendrogram (size [n_rows - 1] * 2) + * @param[out] labels output labels vector (size n_rows) + * @param[in] metric distance metrix to use when constructing connectivities graph + * @param[in] n_clusters number of clusters to assign data samples + * @param[in] linkage strategy for constructing the linkage. PAIRWISE uses more memory but can be + faster for + * smaller datasets. KNN_GRAPH allows the memory usage to be controlled (using + parameter c) + * at the expense of potentially additional minimum spanning tree iterations. + * @param[in] c a constant used when constructing linkage from knn graph. Allows the indirect + control of k. The algorithm will set `k = log(n) + c` + */ +void single_linkage( + raft::resources const& handle, + raft::device_matrix_view X, + raft::device_matrix_view dendrogram, + raft::device_vector_view labels, + cuvs::distance::DistanceType metric, + size_t n_clusters, + cuvs::cluster::agglomerative::Linkage linkage = cuvs::cluster::agglomerative::Linkage::KNN_GRAPH, + std::optional c = std::make_optional(DEFAULT_CONST_C)); + +namespace helpers { + +namespace linkage_graph_params { +/** Specialized parameters to build the KNN graph with regular distances */ +struct distance_params { + /** a constant used when constructing linkage from knn graph. Allows the indirect control of k. + * The algorithm will set `k = log(n) + c` */ + int c = DEFAULT_CONST_C; + + /** strategy for constructing the linkage. PAIRWISE uses more memory but can be faster for smaller + * datasets. KNN_GRAPH allows the memory usage to be controlled (using parameter c) */ + cuvs::cluster::agglomerative::Linkage dist_type = + cuvs::cluster::agglomerative::Linkage::KNN_GRAPH; +}; + +/** Specialized parameters to build the Mutual Reachability graph */ +struct mutual_reachability_params { + /** this neighborhood will be selected for core distances. */ + int min_samples; + + /** weight applied when internal distance is chosen for mutual reachability (value of 1.0 disables + * the weighting) */ + float alpha = 1.0; + + /** + * Parameters for building the mutual reachability graph using an underlying KNN algorithm. + * + * The all-neighbors graph construction algorithm enables building the mutual reachability + * graph on datasets larger than device memory by: + * 1. Partitioning the dataset into overlapping clusters, + * 2. Computing local KNN graphs within each cluster, and + * 3. Merging the local graphs into a single global graph. + * + * Key fields: + * - graph_build_params: Selects the KNN construction method (Brute Force or NN Descent) + * and controls algorithm-specific parameters. + * + * - n_clusters: Number of partitions (batches) to split the data into. Larger `n_clusters` + * reduces memory usage but may reduce accuracy if `overlap_factor` is too low. + * Recommended starting value: `n_clusters = 4`. Increase progressively (4 → 8 → 16 ...) + * to reduce memory usage at the cost of some accuracy. This is independent of `overlap_factor` + * as long as `overlap_factor < n_clusters`. + * + * - overlap_factor: Number of nearest clusters each data point is assigned to. Higher + * `overlap_factor` improves accuracy at the cost of memory and performance. + * Recommended starting value: `overlap_factor = 2`. Increase gradually (2 → 3 → 4 ...) + * for better accuracy with higher device memory usage. + * + * - metric: Distance metric to use when computing nearest neighbors. + */ + cuvs::neighbors::all_neighbors::all_neighbors_params all_neighbors_params{ + cuvs::neighbors::graph_build_params::brute_force_params{}}; +}; +} // namespace linkage_graph_params + +/** + * Given a dataset, builds the KNN graph, connects graph components and builds a linkage + * (dendrogram). Returns the Minimum Spanning Tree edges sorted by weight and the dendrogram. + * @param[in] handle raft handle for resource reuse + * @param[in] X data points on device memory (size n_rows * d) + * @param[in] linkage_graph_params + * Parameters controlling how the KNN graph is built. This can be either: + * - distance_params: standard distance-based KNN graph construction for traditional + * agglomerative clustering. + * - mutual_reachability_params: parameters to compute a mutual reachability graph for + * density-aware hierarchical clustering (e.g. HDBSCAN). + * @param[in] metric distance metric to use + * @param[out] out_mst output MST sorted by edge weights (size n_rows - 1) + * @param[out] dendrogram output dendrogram (size [n_rows - 1] * 2) + * @param[out] out_distances distances for output + * @param[out] out_sizes cluster sizes of output + * @param[out] core_dists (optional) core distances (size m). Must be supplied in the Mutual + * Reachability space + */ +void build_linkage( + raft::resources const& handle, + raft::device_matrix_view X, + std::variant linkage_graph_params, + cuvs::distance::DistanceType metric, + raft::device_coo_matrix_view out_mst, + raft::device_matrix_view dendrogram, + raft::device_vector_view out_distances, + raft::device_vector_view out_sizes, + std::optional> core_dists); + +/** + * Given a dataset, builds the KNN graph, connects graph components and builds a linkage + * (dendrogram). Returns the Minimum Spanning Tree edges sorted by weight and the dendrogram. + * @param[in] handle raft handle for resource reuse + * @param[in] X data points on host memory (size n_rows * d) + * @param[in] linkage_graph_params + * Parameters controlling how the KNN graph is built. This can be either: + * - distance_params: standard distance-based KNN graph construction for traditional + * agglomerative clustering. + * - mutual_reachability_params: parameters to compute a mutual reachability graph for + * density-aware hierarchical clustering (e.g. HDBSCAN). + * @param[in] metric distance metric to use + * @param[out] out_mst output MST sorted by edge weights (size n_rows - 1) + * @param[out] dendrogram output dendrogram (size [n_rows - 1] * 2) + * @param[out] out_distances distances for output + * @param[out] out_sizes cluster sizes of output + * @param[out] core_dists (optional) core distances (size m). Must be supplied in the Mutual + * Reachability space + */ +void build_linkage( + raft::resources const& handle, + raft::host_matrix_view X, + std::variant linkage_graph_params, + cuvs::distance::DistanceType metric, + raft::device_coo_matrix_view out_mst, + raft::device_matrix_view dendrogram, + raft::device_vector_view out_distances, + raft::device_vector_view out_sizes, + std::optional> core_dists); +} // namespace helpers +/** + * @} + */ +}; // end namespace cuvs::cluster::agglomerative diff --git a/cpp/include/cuvs/cluster/kmeans.hpp b/cpp/include/cuvs/cluster/kmeans.hpp new file mode 100644 index 0000000000..43d2db5a5c --- /dev/null +++ b/cpp/include/cuvs/cluster/kmeans.hpp @@ -0,0 +1,1278 @@ +/* + * Copyright (c) 2022-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once +#include +#include +#include +#include +#include +#include +#include + +namespace cuvs::cluster::kmeans { + +/** Base structure for parameters that are common to all k-means algorithms */ +struct base_params { + /** + * Metric to use for distance computation. The supported metrics can vary per algorithm. + */ + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded; +}; + +/** + * @defgroup kmeans_params k-means hyperparameters + * @{ + */ + +/** + * Simple object to specify hyper-parameters to the kmeans algorithm. + */ +struct params : base_params { + enum InitMethod { + + /** + * Sample the centroids using the kmeans++ strategy + */ + KMeansPlusPlus, + + /** + * Sample the centroids uniformly at random + */ + Random, + + /** + * User provides the array of initial centroids + */ + Array + }; + + /** + * The number of clusters to form as well as the number of centroids to generate (default:8). + */ + int n_clusters = 8; + + /** + * Method for initialization, defaults to k-means++: + * - InitMethod::KMeansPlusPlus (k-means++): Use scalable k-means++ algorithm + * to select the initial cluster centers. + * - InitMethod::Random (random): Choose 'n_clusters' observations (rows) at + * random from the input data for the initial centroids. + * - InitMethod::Array (ndarray): Use 'centroids' as initial cluster centers. + */ + InitMethod init = KMeansPlusPlus; + + /** + * Maximum number of iterations of the k-means algorithm for a single run. + */ + int max_iter = 300; + + /** + * Relative tolerance with regards to inertia to declare convergence. + */ + double tol = 1e-4; + + /** + * verbosity level. + */ + rapids_logger::level_enum verbosity = rapids_logger::level_enum::info; + + /** + * Seed to the random number generator. + */ + raft::random::RngState rng_state{0}; + + /** + * Number of instance k-means algorithm will be run with different seeds. + */ + int n_init = 1; + + /** + * Oversampling factor for use in the k-means|| algorithm + */ + double oversampling_factor = 2.0; + + /** + * batch_samples and batch_centroids are used to tile 1NN computation which is + * useful to optimize/control the memory footprint + * Default tile is [batch_samples x n_clusters] i.e. when batch_centroids is 0 + * then don't tile the centroids + */ + int batch_samples = 1 << 15; + + /** + * if 0 then batch_centroids = n_clusters + */ + int batch_centroids = 0; // + + bool inertia_check = false; +}; + +/** + * Simple object to specify hyper-parameters to the balanced k-means algorithm. + * + * The following metrics are currently supported in k-means balanced: + * - CosineExpanded + * - InnerProduct + * - L2Expanded + * - L2SqrtExpanded + */ +struct balanced_params : base_params { + /** + * Number of training iterations + */ + uint32_t n_iters = 20; +}; + +/** + * @} + */ + +/** + * @defgroup kmeans k-means clustering APIs + * @{ + */ + +/** + * @brief Find clusters with k-means algorithm. + * Initial centroids are chosen with k-means++ algorithm. Empty + * clusters are reinitialized by choosing new centroids with + * k-means++ algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids, + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers. + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit(raft::resources const& handle, + const cuvs::cluster::kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Find clusters with k-means algorithm. + * Initial centroids are chosen with k-means++ algorithm. Empty + * clusters are reinitialized by choosing new centroids with + * k-means++ algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int64_t n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, + * n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids, + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers. + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit(raft::resources const& handle, + const cuvs::cluster::kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Find clusters with k-means algorithm. + * Initial centroids are chosen with k-means++ algorithm. Empty + * clusters are reinitialized by choosing new centroids with + * k-means++ algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids, + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers. + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit(raft::resources const& handle, + const cuvs::cluster::kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Find clusters with k-means algorithm. + * Initial centroids are chosen with k-means++ algorithm. Empty + * clusters are reinitialized by choosing new centroids with + * k-means++ algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int64_t n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, + * n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids, + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers. + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit(raft::resources const& handle, + const cuvs::cluster::kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Find clusters with k-means algorithm. + * Initial centroids are chosen with k-means++ algorithm. Empty + * clusters are reinitialized by choosing new centroids with + * k-means++ algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids, + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers. + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit(raft::resources const& handle, + const cuvs::cluster::kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Find balanced clusters with k-means algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15; + * int64_t n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[out] centroids [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids); + +/** + * @brief Find balanced clusters with k-means algorithm. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[inout] centroids [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + */ +void fit(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * false, + * labels.view(), + * raft::make_scalar_view(&inertia)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[in] normalize_weight True if the weights should be normalized + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to + * their closest cluster center. + */ +void predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + bool normalize_weight, + raft::host_scalar_view inertia); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * false, + * labels.view(), + * raft::make_scalar_view(&inertia)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[in] normalize_weight True if the weights should be normalized + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to + * their closest cluster center. + */ +void predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + bool normalize_weight, + raft::host_scalar_view inertia); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * false, + * labels.view(), + * raft::make_scalar_view(&inertia)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[in] normalize_weight True if the weights should be normalized + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to + * their closest cluster center. + */ +void predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + bool normalize_weight, + raft::host_scalar_view inertia); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * false, + * labels.view(), + * raft::make_scalar_view(&inertia)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[in] normalize_weight True if the weights should be normalized + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to + * their closest cluster center. + */ +void predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + raft::device_matrix_view centroids, + raft::device_vector_view labels, + bool normalize_weight, + raft::host_scalar_view inertia); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids.view()); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids.view()); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Predict the closest cluster each sample in X belongs to. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * + * kmeans::fit(handle, + * params, + * X, + * centroids.view()); + * ... + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X New data to predict. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X + * belongs to. + * [len = n_samples] + */ +void predict(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Compute k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * labels.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit_predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + std::optional> centroids, + raft::device_vector_view labels, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Compute k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int64_t n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, + * n_features); auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * labels.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit_predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + std::optional> centroids, + raft::device_vector_view labels, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Compute k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, n_features); + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * labels.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit_predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + std::optional> centroids, + raft::device_vector_view labels, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Compute k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::params params; + * int64_t n_features = 15, inertia, n_iter; + * auto centroids = raft::make_device_matrix(handle, params.n_clusters, + * n_features); auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * std::nullopt, + * centroids.view(), + * labels.view(), + * raft::make_scalar_view(&inertia), + * raft::make_scalar_view(&n_iter)); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[in] sample_weight Optional weights for each observation in X. + * [len = n_samples] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + * @param[out] inertia Sum of squared distances of samples to their + * closest cluster center. + * @param[out] n_iter Number of iterations run. + */ +void fit_predict(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + std::optional> sample_weight, + std::optional> centroids, + raft::device_vector_view labels, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter); + +/** + * @brief Compute balanced k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Compute balanced k-means clustering and predicts cluster index for each sample + * in the input. + * + * @code{.cpp} + * #include + * #include + * using namespace cuvs::cluster; + * ... + * raft::resources handle; + * cuvs::cluster::kmeans::balanced_params params; + * int64_t n_features = 15, n_clusters = 8; + * auto centroids = raft::make_device_matrix(handle, n_clusters, n_features); + * auto labels = raft::make_device_vector(handle, X.extent(0)); + * + * kmeans::fit_predict(handle, + * params, + * X, + * centroids.view(), + * labels.view()); + * @endcode + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must be + * in row-major format. + * [dim = n_samples x n_features] + * @param[inout] centroids Optional + * [in] When init is InitMethod::Array, use + * centroids as the initial cluster centers + * [out] The generated centroids from the + * kmeans algorithm are stored at the address + * pointed by 'centroids'. + * [dim = n_clusters x n_features] + * @param[out] labels Index of the cluster each sample in X belongs + * to. + * [len = n_samples] + */ +void fit_predict(const raft::resources& handle, + cuvs::cluster::kmeans::balanced_params const& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_vector_view labels); + +/** + * @brief Transform X to a cluster-distance space. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] X_new X transformed in the new space. + * [dim = n_samples x n_features] + */ +void transform(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_matrix_view X_new); + +/** + * @brief Transform X to a cluster-distance space. + * + * @param[in] handle The raft handle. + * @param[in] params Parameters for KMeans model. + * @param[in] X Training instances to cluster. The data must + * be in row-major format + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in row-major format. + * [dim = n_clusters x n_features] + * @param[out] X_new X transformed in the new space. + * [dim = n_samples x n_features] + */ +void transform(raft::resources const& handle, + const kmeans::params& params, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::device_matrix_view X_new); + +/** + * @brief Compute cluster cost + * + * @param[in] handle The raft handle + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[out] cost Resulting cluster cost + * + */ +void cluster_cost(const raft::resources& handle, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::host_scalar_view cost); + +/** + * @brief Compute cluster cost + * + * @param[in] handle The raft handle + * @param[in] X Training instances to cluster. The data must + * be in row-major format. + * [dim = n_samples x n_features] + * @param[in] centroids Cluster centroids. The data must be in + * row-major format. + * [dim = n_clusters x n_features] + * @param[out] cost Resulting cluster cost + * + */ +void cluster_cost(const raft::resources& handle, + raft::device_matrix_view X, + raft::device_matrix_view centroids, + raft::host_scalar_view cost); + +/** + * @} + */ + +/** + * @defgroup kmeans_helpers k-means API helpers + * @{ + */ + +namespace helpers { + +/** + * Automatically find the optimal value of k using a binary search. + * This method maximizes the Calinski-Harabasz Index while minimizing the per-cluster inertia. + * + * @code{.cpp} + * #include + * #include + * + * #include + * + * using namespace cuvs::cluster; + * + * raft::handle_t handle; + * int n_samples = 100, n_features = 15, n_clusters = 10; + * auto X = raft::make_device_matrix(handle, n_samples, n_features); + * auto labels = raft::make_device_vector(handle, n_samples); + * + * raft::random::make_blobs(handle, X, labels, n_clusters); + * + * auto best_k = raft::make_host_scalar(0); + * auto n_iter = raft::make_host_scalar(0); + * auto inertia = raft::make_host_scalar(0); + * + * kmeans::find_k(handle, X, best_k.view(), inertia.view(), n_iter.view(), n_clusters+1); + * + * @endcode + * + * @param handle raft handle + * @param X input observations (shape n_samples, n_dims) + * @param best_k best k found from binary search + * @param inertia inertia of best k found + * @param n_iter number of iterations used to find best k + * @param kmax maximum k to try in search + * @param kmin minimum k to try in search (should be >= 1) + * @param maxiter maximum number of iterations to run + * @param tol tolerance for early stopping convergence + */ +void find_k(raft::resources const& handle, + raft::device_matrix_view X, + raft::host_scalar_view best_k, + raft::host_scalar_view inertia, + raft::host_scalar_view n_iter, + int kmax, + int kmin = 1, + int maxiter = 100, + float tol = 1e-3); +} // namespace helpers + +/** + * @} + */ + +} // namespace cuvs::cluster::kmeans diff --git a/cpp/include/cuvs/core/bitmap.hpp b/cpp/include/cuvs/core/bitmap.hpp new file mode 100644 index 0000000000..80ae25cd28 --- /dev/null +++ b/cpp/include/cuvs/core/bitmap.hpp @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace cuvs::core { +/* To use bitmap functions containing CUDA code, include */ + +template +using bitmap_view = raft::core::bitmap_view; + +} // end namespace cuvs::core diff --git a/cpp/include/cuvs/core/bitset.hpp b/cpp/include/cuvs/core/bitset.hpp new file mode 100644 index 0000000000..8236bbf07d --- /dev/null +++ b/cpp/include/cuvs/core/bitset.hpp @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +extern template struct raft::core::bitset; +extern template struct raft::core::bitset; +extern template struct raft::core::bitset; +extern template struct raft::core::bitset; +extern template struct raft::core::bitset; + +namespace cuvs::core { +/* To use bitset functions containing CUDA code, include */ + +template +using bitset_view = raft::core::bitset_view; + +template +using bitset = raft::core::bitset; + +} // end namespace cuvs::core diff --git a/cpp/include/cuvs/core/c_api.h b/cpp/include/cuvs/core/c_api.h deleted file mode 100644 index b500329169..0000000000 --- a/cpp/include/cuvs/core/c_api.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -#include - -/** - * @defgroup c_api C API Core Types and Functions - * @{ - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief An opaque C handle for C++ type `raft::resources` - * - */ -typedef uintptr_t cuvsResources_t; - -/** - * @brief An enum denoting return values for function calls - * - */ -typedef enum { CUVS_ERROR, CUVS_SUCCESS } cuvsError_t; - -/** - * @brief Create an Initialized opaque C handle for C++ type `raft::resources` - * - * @param[in] res cuvsResources_t opaque C handle - * @return cuvsError_t - */ -cuvsError_t cuvsResourcesCreate(cuvsResources_t* res); - -/** - * @brief Destroy and de-allocate opaque C handle for C++ type `raft::resources` - * - * @param[in] res cuvsResources_t opaque C handle - * @return cuvsError_t - */ -cuvsError_t cuvsResourcesDestroy(cuvsResources_t res); - -/** - * @brief Set cudaStream_t on cuvsResources_t to queue CUDA kernels on APIs - * that accept a cuvsResources_t handle - * - * @param[in] res cuvsResources_t opaque C handle - * @param[in] stream cudaStream_t stream to queue CUDA kernels - * @return cuvsError_t - */ -cuvsError_t cuvsStreamSet(cuvsResources_t res, cudaStream_t stream); - -#ifdef __cplusplus -} -#endif - -/** @} */ diff --git a/cpp/include/cuvs/distance/distance.hpp b/cpp/include/cuvs/distance/distance.hpp new file mode 100644 index 0000000000..d1bdd839e5 --- /dev/null +++ b/cpp/include/cuvs/distance/distance.hpp @@ -0,0 +1,416 @@ +/* + * Copyright (c) 2021-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace cuvs::distance { + +using DistanceType = cuvsDistanceType; + +/** + * Whether minimal distance corresponds to similar elements (using the given metric). + */ +inline bool is_min_close(DistanceType metric) +{ + bool select_min; + switch (metric) { + case DistanceType::InnerProduct: + // Similarity metrics have the opposite meaning, i.e. nearest neighbors are those with larger + // similarity (See the same logic at cpp/include/raft/sparse/spatial/detail/knn.cuh:362 + // {perform_k_selection}) + select_min = false; + break; + default: select_min = true; + } + return select_min; +} + +namespace kernels { +enum KernelType { LINEAR, POLYNOMIAL, RBF, TANH }; + +/** + * Parameters for kernel matrices. + * The following kernels are implemented: + * - LINEAR \f[ K(x_1,x_2) = , \f] where \f$< , >\f$ is the dot product + * - POLYNOMIAL \f[ K(x_1, x_2) = (\gamma + \mathrm{coef0})^\mathrm{degree} \f] + * - RBF \f[ K(x_1, x_2) = \exp(- \gamma |x_1-x_2|^2) \f] + * - TANH \f[ K(x_1, x_2) = \tanh(\gamma + \mathrm{coef0}) \f] + */ +struct KernelParams { + // Kernel function parameters + KernelType kernel; //!< Type of the kernel function + int degree; //!< Degree of polynomial kernel (ignored by others) + double gamma; //!< multiplier in the + double coef0; //!< additive constant in poly and tanh kernels +}; +} // end namespace kernels + +/** + * @defgroup pairwise_distance Pairwise Distances API + * @{ + */ + +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); + +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + double metric_arg = 2.0f); +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); + +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + double metric_arg = 2.0f); +/** + * @brief Compute pairwise distances for two matrices + * + * Note: Only contiguous row- or column-major layouts supported currently. + * + * Usage example: + * @code{.cpp} + * #include + * #include + * #include + * + * raft::resources handle; + * int n_samples = 5000; + * int n_features = 50; + * + * auto input = raft::make_device_matrix(handle, n_samples, n_features); + * + * // ... fill input with data ... + * + * auto output = raft::make_device_matrix(handle, n_samples, n_samples); + * + * auto metric = cuvs::distance::DistanceType::L2SqrtExpanded; + * cuvs::distance::pairwise_distance(handle, + * raft::make_const(input.view()), + * raft::make_const(input.view()), + * output.view(), + * metric); + * @endcode + * + * @param[in] handle raft handle for managing expensive resources + * @param[in] x first set of points (size n*k) + * @param[in] y second set of points (size m*k) + * @param[out] dist output distance matrix (size n*m) + * @param[in] metric distance to evaluate + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance( + raft::resources const& handle, + raft::device_matrix_view const x, + raft::device_matrix_view const y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); + +/** + * @brief Compute sparse pairwise distances between x and y, using the provided + * input configuration and distance function. + * + * @code{.cpp} + * #include + * #include + * #include + * + * int x_n_rows = 100000; + * int y_n_rows = 50000; + * int n_cols = 10000; + * + * raft::device_resources handle; + * auto x = raft::make_device_csr_matrix(handle, x_n_rows, n_cols); + * auto y = raft::make_device_csr_matrix(handle, y_n_rows, n_cols); + * + * ... + * // populate data + * ... + * + * auto out = raft::make_device_matrix(handle, x_nrows, y_nrows); + * auto metric = cuvs::distance::DistanceType::L2Expanded; + * raft::sparse::distance::pairwise_distance(handle, x.view(), y.view(), out, metric); + * @endcode + * + * @param[in] handle raft::resources + * @param[in] x raft::device_csr_matrix_view + * @param[in] y raft::device_csr_matrix_view + * @param[out] dist raft::device_matrix_view dense matrix + * @param[in] metric distance metric to use + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance(raft::resources const& handle, + raft::device_csr_matrix_view x, + raft::device_csr_matrix_view y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); + +/** + * @brief Compute sparse pairwise distances between x and y, using the provided + * input configuration and distance function. + * + * @code{.cpp} + * #include + * #include + * #include + * + * int x_n_rows = 100000; + * int y_n_rows = 50000; + * int n_cols = 10000; + * + * raft::device_resources handle; + * auto x = raft::make_device_csr_matrix(handle, x_n_rows, n_cols); + * auto y = raft::make_device_csr_matrix(handle, y_n_rows, n_cols); + * + * ... + * // populate data + * ... + * + * auto out = raft::make_device_matrix(handle, x_nrows, y_nrows); + * auto metric = cuvs::distance::DistanceType::L2Expanded; + * raft::sparse::distance::pairwise_distance(handle, x.view(), y.view(), out, metric); + * @endcode + * + * @param[in] handle raft::resources + * @param[in] x raft::device_csr_matrix_view + * @param[in] y raft::device_csr_matrix_view + * @param[out] dist raft::device_matrix_view dense matrix + * @param[in] metric distance metric to use + * @param[in] metric_arg metric argument (used for Minkowski distance) + */ +void pairwise_distance(raft::resources const& handle, + raft::device_csr_matrix_view x, + raft::device_csr_matrix_view y, + raft::device_matrix_view dist, + cuvs::distance::DistanceType metric, + float metric_arg = 2.0f); + +/** @} */ // end group pairwise_distance_runtime + +}; // namespace cuvs::distance diff --git a/cpp/include/cuvs/distance/grammian.hpp b/cpp/include/cuvs/distance/grammian.hpp new file mode 100644 index 0000000000..4d0a90e6cd --- /dev/null +++ b/cpp/include/cuvs/distance/grammian.hpp @@ -0,0 +1,665 @@ +/* + * Copyright (c) 2022-2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace cuvs::distance::kernels { + +template +using dense_input_matrix_view_t = raft::device_matrix_view; +template +using dense_output_matrix_view_t = raft::device_matrix_view; +template +using csr_input_matrix_view_t = raft::device_csr_matrix_view; + +/** + * Base class for general Gram matrices + * A Gram matrix is the Hermitian matrix of inner probucts G_ik = + * Here, the inner product is evaluated for all elements from vectors sets X1, + * and X2. + * + * To be more precise, on exit the output buffer will store: + * - if is_row_major == true: out[j+k*n1] = , + * - if is_row_major == false: out[j*n2 + k] = , + * where x1_j is the j-th vector from the x1 set and x2_k is the k-th vector + * from the x2 set. + */ +template +class GramMatrixBase { + protected: + cublasHandle_t cublas_handle; + bool legacy_interface; + + public: + GramMatrixBase() : legacy_interface(false) {}; + [[deprecated]] GramMatrixBase(cublasHandle_t cublas_handle) + : cublas_handle(cublas_handle), legacy_interface(true) {}; + + virtual ~GramMatrixBase() {}; + + /** Convenience function to evaluate the Gram matrix for two vector sets. + * Vector sets are provided in Matrix format + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void operator()(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1 = nullptr, + math_t* norm_x2 = nullptr); + + /** Convenience function to evaluate the Gram matrix for two vector sets. + * Vector sets are provided in Matrix format + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void operator()(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1 = nullptr, + math_t* norm_x2 = nullptr); + + /** Convenience function to evaluate the Gram matrix for two vector sets. + * Vector sets are provided in Matrix format + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void operator()(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1 = nullptr, + math_t* norm_x2 = nullptr); + + // unfortunately, 'evaluate' cannot be templatized as it needs to be virtual + + /** Evaluate the Gram matrix for two vector sets using simple dot product. + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + virtual void evaluate(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix for two vector sets using simple dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + virtual void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix for two vector sets using simple dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + virtual void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix for two vector sets using simple dot product. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 (usually it is n1) + * @param ld2 leading dimension of x2 (usually it is n2) + * @param ld_out leading dimension of out (usually it is n1) + */ + [[deprecated]] virtual void evaluate(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1, + int ld2, + int ld_out); + + /** Convenience function to evaluate the Gram matrix for two vector sets. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 + * @param ld2 leading dimension of x2 + * @param ld_out leading dimension of out + */ + [[deprecated]] void operator()(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1 = 0, + int ld2 = 0, + int ld_out = 0); + + protected: + /** Calculates the Gram matrix using simple dot product between vector sets. + * + * out = x1 * x2 + * + * Can be used as a building block for more complex kernel functions. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 + * @param ld2 leading dimension of x2 + * @param ld_out leading dimension of out + */ + [[deprecated]] void linear(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1, + int ld2, + int ld_out); + + protected: + bool get_is_row_major(dense_output_matrix_view_t matrix); + bool get_is_row_major(dense_input_matrix_view_t matrix); + bool get_is_col_major(dense_output_matrix_view_t matrix); + bool get_is_col_major(dense_input_matrix_view_t matrix); + + /** Calculates the Gram matrix using simple dot product between vector sets. + * + * out = x1 * x2 + * + * Can be used as a building block for more complex kernel functions. + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + */ + void linear(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out); + + /** Calculates the Gram matrix using simple dot product between vector sets. + * + * out = x1 * x2 + * + * Can be used as a building block for more complex kernel functions. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + */ + void linear(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out); + + /** Calculates the Gram matrix using simple dot product between vector sets. + * + * out = x1 * x2 + * + * Can be used as a building block for more complex kernel functions. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + */ + void linear(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out); +}; + +template +class KernelFactory { + public: + static GramMatrixBase* create(KernelParams params); + [[deprecated]] static GramMatrixBase* create(KernelParams params, cublasHandle_t handle); +}; + +/** + * Create a kernel matrix using polynomial kernel function. + */ +template +class PolynomialKernel : public GramMatrixBase { + exp_t exponent; + math_t gain; + math_t offset; + + void applyKernel( + math_t* inout, int ld, int rows, int cols, bool is_row_major, cudaStream_t stream); + + public: + /** + * Constructs a polynomial kernel object. + * It evaluates the kernel matrix using the following formula: + * K_ij = (gain* + offset)^exponent + * + * @tparam math_t floating point type + * @tparam exp_t type of exponent + * @param exponent + * @param gain + * @param offset + */ + PolynomialKernel(exp_t exponent, math_t gain, math_t offset) + : GramMatrixBase(), exponent(exponent), gain(gain), offset(offset) {}; + + [[deprecated]] PolynomialKernel(exp_t exponent, math_t gain, math_t offset, cublasHandle_t handle) + : GramMatrixBase(handle), exponent(exponent), gain(gain), offset(offset) {}; + + /** Evaluate kernel matrix using polynomial kernel. + * + * output[i,k] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using polynomial kernel. + * + * output[i,k] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using polynomial kernel. + * + * output[i,k] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix using the legacy interface. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 (usually it is n1) + * @param ld2 leading dimension of x2 (usually it is n2) + * @param ld_out leading dimension of out (usually it is n1) + */ + [[deprecated]] void evaluate(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1, + int ld2, + int ld_out); +}; + +/** + * Create a kernel matrix using tanh kernel function. + */ +template +class TanhKernel : public GramMatrixBase { + math_t gain, offset; + + void applyKernel( + math_t* inout, int ld, int rows, int cols, bool is_row_major, cudaStream_t stream); + + public: + /** + * Constructs a tanh kernel object. + * It evaluates the kernel matrix using the following formula: + * K_ij = tanh(gain* + offset) + * + * @tparam math_t floating point type + * @param gain + * @param offset + */ + TanhKernel(math_t gain, math_t offset) : GramMatrixBase(), gain(gain), offset(offset) {} + + [[deprecated]] TanhKernel(math_t gain, math_t offset, cublasHandle_t handle) + : GramMatrixBase(handle), gain(gain), offset(offset) {}; + + /** Evaluate kernel matrix using tanh kernel. + * + * output_[i + k*n1] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using tanh kernel. + * + * output_[i + k*n1] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using tanh kernel. + * + * output_[i + k*n1] = (gain* + offset)^exponent, + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and < , > denotes dot product. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 unused. + * @param norm_x2 unused. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix using the legacy interface. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 (usually it is n1) + * @param ld2 leading dimension of x2 (usually it is n2) + * @param ld_out leading dimension of out (usually it is n1) + */ + [[deprecated]] void evaluate(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1, + int ld2, + int ld_out); +}; + +/** + * Create a kernel matrix using RBF kernel function. + */ +template +class RBFKernel : public GramMatrixBase { + math_t gain; + + void applyKernel(math_t* inout, + int ld, + int rows, + int cols, + math_t* norm_x1, + math_t* norm_x2, + bool is_row_major, + cudaStream_t stream); + + public: + /** + * Constructs a RBF kernel object. + * It evaluates the kernel matrix using the following formula: + * K_ij = exp(-gain*|x1_i- x2_k|^2) + * + * @tparam math_t floating point type + * @param gain + */ + RBFKernel(math_t gain) : GramMatrixBase(), gain(gain) {}; + + [[deprecated]] RBFKernel(math_t gain, cublasHandle_t handle) + : GramMatrixBase(handle), gain(gain) {}; + + void matrixRowNormL2(raft::resources const& handle, + dense_input_matrix_view_t matrix, + math_t* target); + + void matrixRowNormL2(raft::resources const& handle, + csr_input_matrix_view_t matrix, + math_t* target); + + /** Evaluate kernel matrix using RBF kernel. + * + * output_[i + k*n1] = exp(-gain*|x1_i - x2_k|^2), + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and | | euclidean distance. + * + * @param [in] handle raft handle + * @param [in] x1 dense device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void evaluate(raft::resources const& handle, + dense_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using RBF kernel. + * + * output_[i + k*n1] = exp(-gain*|x1_i - x2_k|^2), + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and | | euclidean distance. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 dense device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + dense_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate kernel matrix using RBF kernel. + * + * output_[i + k*n1] = exp(-gain*|x1_i - x2_k|^2), + * where x1_i is the i-th vector from the x1 set, and x2_k is k-th vector + * in the x2 set, and | | euclidean distance. + * + * @param [in] handle raft handle + * @param [in] x1 csr device matrix view, size [n1*n_cols] + * @param [in] x2 csr device matrix view, size [n2*n_cols] + * @param [out] out dense device matrix view for the Gram matrix, size [n1*n2] + * @param norm_x1 optional L2-norm of x1's rows for computation within RBF. + * @param norm_x2 optional L2-norm of x2's rows for computation within RBF. + */ + void evaluate(raft::resources const& handle, + csr_input_matrix_view_t x1, + csr_input_matrix_view_t x2, + dense_output_matrix_view_t out, + math_t* norm_x1, + math_t* norm_x2); + + /** Evaluate the Gram matrix using the legacy interface. + * + * @param [in] x1 device array of vectors, size [n1*n_cols] + * @param [in] n1 number vectors in x1 + * @param [in] n_cols number of columns (features) in x1 and x2 + * @param [in] x2 device array of vectors, size [n2*n_cols] + * @param [in] n2 number vectors in x2 + * @param [out] out device buffer to store the Gram matrix, size [n1*n2] + * @param [in] is_row_major whether the input and output matrices are in row + * major format + * @param [in] stream cuda stream + * @param ld1 leading dimension of x1 (usually it is n1) + * @param ld2 leading dimension of x2 (usually it is n2) + * @param ld_out leading dimension of out (usually it is n1) + */ + [[deprecated]] void evaluate(const math_t* x1, + int n1, + int n_cols, + const math_t* x2, + int n2, + math_t* out, + bool is_row_major, + cudaStream_t stream, + int ld1, + int ld2, + int ld_out); +}; +}; // end namespace cuvs::distance::kernels diff --git a/cpp/include/cuvs/embed/spectral.hpp b/cpp/include/cuvs/embed/spectral.hpp new file mode 100644 index 0000000000..1a8fed96a1 --- /dev/null +++ b/cpp/include/cuvs/embed/spectral.hpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace cuvs::embed::spectral { + +/** + * Given a COO formatted (symmetric) knn graph, this function computes the spectral embeddings + * (lowest n_components eigenvectors), using Lanczos min cut algorithm. Please note that this + * algorithm does not compute a full laplacian eigenmap, as the laplacian eigenmap would embed each + * connected component. Laplacian eigenmaps can be built from this algorithm by running it on the + * vectors for each connected component. + + * @param[in] handle + * @param[in] knn_graph KNN Graph + * @param[in] n_components the number of components to project into + * @param[out] out output array for embedding (size n*n_comonents) + * @param[in] seed + */ +void fit(const raft::resources& handle, + raft::device_coo_matrix_view knn_graph, + int n_components, + raft::device_matrix_view out, + unsigned long long seed = 0L); +}; // namespace cuvs::embed::spectral diff --git a/cpp/include/cuvs/neighbors/all_neighbors.hpp b/cpp/include/cuvs/neighbors/all_neighbors.hpp new file mode 100644 index 0000000000..b52a4ca294 --- /dev/null +++ b/cpp/include/cuvs/neighbors/all_neighbors.hpp @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +namespace cuvs::neighbors::all_neighbors { +// For re-exporting into all_neighbors namespace +namespace graph_build_params = cuvs::neighbors::graph_build_params; +/** + * @defgroup all_neighbors_cpp_params The all-neighbors algorithm parameters. + * @{ + */ + +using GraphBuildParams = std::variant; + +/** + * @brief Parameters used to build an all-neighbors graph (find nearest neighbors for all the + * training vectors). + * For scalability, the all-neighbors graph construction algorithm partitions a set of training + * vectors into overlapping clusters, computes a local knn graph on each cluster, and merges the + * local graphs into a single global graph. + * Device memory usage and accuracy can be configured by changing the `overlap_factor` and + * `n_clusters`. + * The algorithm used to build each local graph is also configurable. + * + */ +struct all_neighbors_params { + /** Parameters for knn graph building algorithm + * Approximate nearest neighbors methods or a brute force approach are supported to build the knn + * graph. Currently supported options are 'IVF-PQ', 'NN Descent', or 'Brute Force'. IVF-PQ is more + * accurate, but slower compared to NN Descent. Note that 'Brute Force' can also be approximate if + * n_clusters > 1. + * + * Set ivf_pq_params, nn_descent_params, or brute_force_params to select the graph build + * algorithm and control their parameters. + * + * @code{.cpp} + * all_neighbors::index_params params; + * // 1. Choose IVF-PQ algorithm + * params.graph_build_params = all_neighbors::graph_build_params::ivf_pq_params{}; + * + * // 2. Choose NN Descent algorithm for kNN graph construction + * params.graph_build_params = all_neighbors::graph_build_params::nn_descent_params{}; + * + * // 3. Choose Brute Force algorithm for kNN graph construction + * params.graph_build_params = all_neighbors::graph_build_params::brute_force_params{}; + * + * @endcode + */ + GraphBuildParams graph_build_params; + + /** + * Number of nearest clusters each data point will be assigned to in the batching algorithm. + * Start with `overlap_factor = 2` and gradually increase (2->3->4 ...) for better accuracy at the + * cost of device memory usage. + */ + size_t overlap_factor = 2; + + /** + * Number of total clusters (aka batches) to split the data into. If set to 1, algorithm creates + * an all-neighbors graph without batching. + * Start with `n_clusters = 4` and increase (4 → 8 → 16...) for less device memory usage at the + * cost of accuracy. This is independent from `overlap_factor` as long as `overlap_factor` < + * `n_clusters`. + * + * The ratio of `overlap_factor / n_clusters` determines device memory usage. + * Approximately `(overlap_factor / n_clusters) * num_rows_in_entire_data` number of rows will + * be put on device memory at once. + * E.g. between `(overlap_factor / n_clusters)` = 2/10 and 2/20, the latter will use less device + * memory. + * + * Larger `overlap_factor` results in better accuracy of the final all-neighbors knn + * graph. E.g. While using similar device memory, `(overlap_factor / n_clusters)` = 4/20 + * will have better accuracy than 2/10 at the cost of performance. + * + */ + size_t n_clusters = 1; // defaults to not batching + + /** Metric used. */ + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded; +}; + +/** @} */ + +/** + * @defgroup all_neighbors_cpp_build The all-neighbors knn graph build + * @{ + */ + +/** + * @brief Builds an approximate all-neighbors knn graph (find nearest neighbors for all the + * training vectors) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * all_neighbors::all_neighbors_params params; + * auto indices = raft::make_device_matrix(handle, n_row, k); + * auto distances = raft::make_device_matrix(handle, n_row, k); + * all_neighbors::build(res, params, dataset, indices.view(), distances.view()); + * @endcode + * + * @param[in] handle raft::resources is an object mangaging resources + * @param[in] params an instance of all_neighbors::all_neighbors_params that are parameters + * to build all-neighbors knn graph + * @param[in] dataset raft::host_matrix_view input dataset expected to be located + * in host memory + * @param[out] indices nearest neighbor indices of shape [n_row x k] + * @param[out] distances nearest neighbor distances [n_row x k] + * @param[out] core_distances array for core distances of size [n_row]. Requires distances matrix to + * compute core_distances. If core_distances is given, the resulting indices and distances will be + * mutual reachability space. + * @param[in] alpha distance scaling parameter as used in robust single linkage. + */ +void build( + const raft::resources& handle, + const all_neighbors_params& params, + raft::host_matrix_view dataset, + raft::device_matrix_view indices, + std::optional> distances = std::nullopt, + std::optional> core_distances = std::nullopt, + float alpha = 1.0); + +/** + * @brief Builds an approximate all-neighbors knn graph (find nearest neighbors for all the training + * vectors) params.n_clusters should be 1 for data on device. To use a larger params.n_clusters for + * efficient device memory usage, put data on host RAM. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * all_neighbors::all_neighbors_params params; + * auto indices = raft::make_device_matrix(handle, n_row, k); + * auto distances = raft::make_device_matrix(handle, n_row, k); + * all_neighbors::build(res, params, dataset, indices.view(), distances.view()); + * @endcode + * + * @param[in] handle raft::resources is an object mangaging resources + * @param[in] params an instance of all_neighbors::all_neighbors_params that are parameters + * to build all-neighbors knn graph + * @param[in] dataset raft::device_matrix_view input dataset expected to be located + * in device memory + * @param[out] indices nearest neighbor indices of shape [n_row x k] + * @param[out] distances nearest neighbor distances [n_row x k] + * @param[out] core_distances array for core distances of size [n_row]. Requires distances matrix to + * compute core_distances. If core_distances is given, the resulting indices and distances will be + * mutual reachability space. + * @param[in] alpha distance scaling parameter as used in robust single linkage. + */ +void build( + const raft::resources& handle, + const all_neighbors_params& params, + raft::device_matrix_view dataset, + raft::device_matrix_view indices, + std::optional> distances = std::nullopt, + std::optional> core_distances = std::nullopt, + float alpha = 1.0); + +/** @} */ +} // namespace cuvs::neighbors::all_neighbors diff --git a/cpp/include/cuvs/neighbors/ann_types.hpp b/cpp/include/cuvs/neighbors/ann_types.hpp deleted file mode 100644 index 36b545ebd0..0000000000 --- a/cpp/include/cuvs/neighbors/ann_types.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -namespace cuvs::neighbors::ann { - -/** - * @defgroup ann_types Approximate Nearest Neighbors Types - * @{ - */ - -/** The base for approximate KNN index structures. */ -struct index {}; - -/** The base for KNN index parameters. */ -struct index_params { - /** Distance type. */ - cuvs::distance::DistanceType metric = distance::DistanceType::L2Expanded; - /** The argument used by some distance metrics. */ - float metric_arg = 2.0f; - /** - * Whether to add the dataset content to the index, i.e.: - * - * - `true` means the index is filled with the dataset vectors and ready to search after calling - * `build`. - * - `false` means `build` only trains the underlying model (e.g. quantizer or clustering), but - * the index is left empty; you'd need to call `extend` on the index afterwards to populate it. - */ - bool add_data_on_build = true; -}; - -struct search_params {}; - -/** @} */ // end group ann_types - -}; // namespace cuvs::neighbors::ann diff --git a/cpp/include/cuvs/neighbors/ball_cover.hpp b/cpp/include/cuvs/neighbors/ball_cover.hpp new file mode 100644 index 0000000000..6f4e2f45bb --- /dev/null +++ b/cpp/include/cuvs/neighbors/ball_cover.hpp @@ -0,0 +1,352 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace cuvs::neighbors::ball_cover { + +/** + * @ingroup random_ball_cover + * @{ + */ + +/** + * Stores raw index data points, sampled landmarks, the 1-nns of index points + * to their closest landmarks, and the ball radii of each landmark. This + * class is intended to be constructed once and reused across subsequent + * queries. + * @tparam int64_t + * @tparam float + * @tparam int + */ +template +struct index : cuvs::neighbors::index { + public: + explicit index(raft::resources const& handle_, + raft::device_matrix_view X_, + cuvs::distance::DistanceType metric_) + : handle(handle_), + X(X_), + m(X_.extent(0)), + n(X_.extent(1)), + metric(metric_), + /** + * the sqrt() here makes the sqrt(m)^2 a linear-time lower bound + * + * Total memory footprint of index: (2 * sqrt(m)) + (n * sqrt(m)) + (2 * m) + */ + n_landmarks(raft::sqrt(X_.extent(0))), + R_indptr(raft::make_device_vector(handle, raft::sqrt(X_.extent(0)) + 1)), + R_1nn_cols(raft::make_device_vector(handle, X_.extent(0))), + R_1nn_dists(raft::make_device_vector(handle, X_.extent(0))), + R_closest_landmark_dists(raft::make_device_vector(handle, X_.extent(0))), + R(raft::make_device_matrix(handle, raft::sqrt(X_.extent(0)), X_.extent(1))), + X_reordered(raft::make_device_matrix(handle, X_.extent(0), X_.extent(1))), + R_radius(raft::make_device_vector(handle, raft::sqrt(X_.extent(0)))), + index_trained(false) + { + } + + auto get_R_indptr() const -> raft::device_vector_view + { + return R_indptr.view(); + } + auto get_R_1nn_cols() const -> raft::device_vector_view + { + return R_1nn_cols.view(); + } + auto get_R_1nn_dists() const -> raft::device_vector_view + { + return R_1nn_dists.view(); + } + auto get_R_radius() const -> raft::device_vector_view + { + return R_radius.view(); + } + auto get_R() const -> raft::device_matrix_view + { + return R.view(); + } + auto get_R_closest_landmark_dists() const -> raft::device_vector_view + { + return R_closest_landmark_dists.view(); + } + auto get_X_reordered() const -> raft::device_matrix_view + { + return X_reordered.view(); + } + + raft::device_vector_view get_R_indptr() { return R_indptr.view(); } + raft::device_vector_view get_R_1nn_cols() { return R_1nn_cols.view(); } + raft::device_vector_view get_R_1nn_dists() { return R_1nn_dists.view(); } + raft::device_vector_view get_R_radius() { return R_radius.view(); } + raft::device_matrix_view get_R() { return R.view(); } + raft::device_vector_view get_R_closest_landmark_dists() + { + return R_closest_landmark_dists.view(); + } + raft::device_matrix_view get_X_reordered() + { + return X_reordered.view(); + } + raft::device_matrix_view get_X() const { return X; } + + cuvs::distance::DistanceType get_metric() const { return metric; } + + int get_n_landmarks() const { return n_landmarks; } + bool is_index_trained() const { return index_trained; }; + + // This should only be set by internal functions + void set_index_trained() { index_trained = true; } + + raft::resources const& handle; + + int64_t m; + int64_t n; + int64_t n_landmarks; + + raft::device_matrix_view X; + + cuvs::distance::DistanceType metric; + + private: + // CSR storing the neighborhoods for each data point + raft::device_vector R_indptr; + raft::device_vector R_1nn_cols; + raft::device_vector R_1nn_dists; + raft::device_vector R_closest_landmark_dists; + + raft::device_vector R_radius; + + raft::device_matrix R; + raft::device_matrix X_reordered; + + protected: + bool index_trained; +}; + +/** @} */ + +/** + * @defgroup random_ball_cover Random Ball Cover algorithm + * @{ + */ + +/** + * Builds and populates a previously unbuilt cuvs::neighbors::ball_cover::index + * + * Usage example: + * @code{.cpp} + * + * #include + * #include + * #include + * using namespace cuvs::neighbors; + * + * raft::resources handle; + * ... + * auto metric = cuvs::distance::DistanceType::L2Expanded; + * ball_cover::index index(handle, X, metric); + * ball_cover::build_index(handle, index); + * @endcode + * + * @param[in] handle library resource management handle + * @param[inout] index an empty (and not previous built) instance of + * cuvs::neighbors::ball_cover::index + */ +void build(raft::resources const& handle, index& index); + +/** @} */ // end group random_ball_cover + +/** + * @ingroup random_ball_cover + * @{ + */ + +/** + * Performs a faster exact knn in metric spaces using the triangle + * inequality with a number of landmark points to reduce the + * number of distance computations from O(n^2) to O(sqrt(n)). This + * performs an all neighbors knn, which can reuse memory when + * the index and query are the same array. This function will + * build the index and assumes rbc_build_index() has not already + * been called. + * + * Usage example: + * @code{.cpp} + * + * #include + * #include + * #include + * using namespace cuvs::neighbors; + * + * raft::resources handle; + * ... + * auto metric = cuvs::distance::DistanceType::L2Expanded; + * + * // Construct a ball cover index + * ball_cover::index index(handle, X, metric); + * + * // Perform all neighbors knn query + * ball_cover::all_knn_query(handle, index, inds, dists, k); + * @endcode + * + * @param[in] handle raft handle for resource management + * @param[in] index ball cover index which has not yet been built + * @param[out] inds output knn indices + * @param[out] dists output knn distances + * @param[in] perform_post_filtering if this is false, only the closest k landmarks + * are considered (which will return approximate + * results). + * @param[in] weight a weight for overlap between the closest landmark and + * the radius of other landmarks when pruning distances. + * Setting this value below 1 can effectively turn off + * computing distances against many other balls, enabling + * approximate nearest neighbors. Recall can be adjusted + * based on how many relevant balls are ignored. Note that + * many datasets can still have great recall even by only + * looking in the closest landmark. + */ +void all_knn_query(raft::resources const& handle, + index& index, + raft::device_matrix_view inds, + raft::device_matrix_view dists, + bool perform_post_filtering = true, + float weight = 1.0); + +/** @} */ + +/** + * @brief Computes epsilon neighborhood for the L2 distance metric using rbc + * + * @param[in] handle raft handle for resource management + * @param[in] index ball cover index which has been built + * @param[out] adj adjacency matrix [row-major] [on device] [dim = m x n] + * @param[out] vd vertex degree array [on device] [len = m + 1] + * `vd + m` stores the total number of edges in the adjacency + * matrix. Pass a nullptr if you don't need this info. + * @param[in] query first matrix [row-major] [on device] [dim = m x k] + * @param[in] eps defines epsilon neighborhood radius + */ +void eps_nn(raft::resources const& handle, + const index& index, + raft::device_matrix_view adj, + raft::device_vector_view vd, + raft::device_matrix_view query, + float eps); +/** + * @brief Computes epsilon neighborhood for the L2 distance metric using rbc + * + * @param[in] handle raft handle for resource management + * @param[in] index ball cover index which has been built + * @param[out] adj_ia adjacency matrix CSR row offsets + * @param[out] adj_ja adjacency matrix CSR column indices, needs to be nullptr + * in first pass with max_k nullopt + * @param[out] vd vertex degree array [on device] [len = m + 1] + * `vd + m` stores the total number of edges in the adjacency + * matrix. Pass a nullptr if you don't need this info. + * @param[in] query first matrix [row-major] [on device] [dim = m x k] + * @param[in] eps defines epsilon neighborhood radius + * @param[inout] max_k if nullopt (default), the user needs to make 2 subsequent calls: + * The first call computes row offsets in adj_ia, where adj_ia[m] + * contains the minimum required size for adj_ja. + * The second call fills in adj_ja based on adj_ia. + * If max_k != nullopt the algorithm only fills up neighbors up to a + * maximum number of max_k for each row in a single pass. Note + * that it is not guarantueed to return the nearest neighbors. + * Upon return max_k is overwritten with the actual max_k found during + * computation. + */ +void eps_nn(raft::resources const& handle, + const index& index, + raft::device_vector_view adj_ia, + raft::device_vector_view adj_ja, + raft::device_vector_view vd, + raft::device_matrix_view query, + float eps, + std::optional> max_k = std::nullopt); + +/** + * @ingroup random_ball_cover + * @{ + */ + +/** + * Performs a faster exact knn in metric spaces using the triangle + * inequality with a number of landmark points to reduce the + * number of distance computations from O(n^2) to O(sqrt(n)). This + * function does not build the index and assumes rbc_build_index() has + * already been called. Use this function when the index and + * query arrays are different, otherwise use rbc_all_knn_query(). + * + * Usage example: + * @code{.cpp} + * + * #include + * #include + * #include + * using namespace cuvs::neighbors; + * + * raft::resources handle; + * ... + * auto metric = cuvs::distance::DistanceType::L2Expanded; + * + * // Build a ball cover index + * ball_cover::index index(handle, X, metric); + * ball_cover::build_index(handle, index); + * + * // Perform all neighbors knn query + * ball_cover::knn_query(handle, index, inds, dists, k); + * @endcode + * @param[in] handle raft handle for resource management + * @param[in] index ball cover index which has not yet been built + * @param[in] query device matrix containing query data points + * @param[out] inds output knn indices + * @param[out] dists output knn distances + * @param[in] perform_post_filtering if this is false, only the closest k landmarks + * are considered (which will return approximate + * results). + * @param[in] weight a weight for overlap between the closest landmark and + * the radius of other landmarks when pruning distances. + * Setting this value below 1 can effectively turn off + * computing distances against many other balls, enabling + * approximate nearest neighbors. Recall can be adjusted + * based on how many relevant balls are ignored. Note that + * many datasets can still have great recall even by only + * looking in the closest landmark. + */ +void knn_query(raft::resources const& handle, + const index& index, + raft::device_matrix_view query, + raft::device_matrix_view inds, + raft::device_matrix_view dists, + bool perform_post_filtering = true, + float weight = 1.0); + +/** @} */ + +} // namespace cuvs::neighbors::ball_cover diff --git a/cpp/include/cuvs/neighbors/brute_force.hpp b/cpp/include/cuvs/neighbors/brute_force.hpp new file mode 100644 index 0000000000..dade5d8c0e --- /dev/null +++ b/cpp/include/cuvs/neighbors/brute_force.hpp @@ -0,0 +1,939 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace cuvs::neighbors::brute_force { + +struct index_params : cuvs::neighbors::index_params {}; + +struct search_params : cuvs::neighbors::search_params {}; + +/** + * @defgroup bruteforce_cpp_index Bruteforce index + * @{ + */ +/** + * @brief Brute Force index. + * + * The index stores the dataset and norms for the dataset in device memory. + * + * @tparam T data element type + */ +template +struct index : cuvs::neighbors::index { + using index_params_type = brute_force::index_params; + using search_params_type = brute_force::search_params; + using index_type = int64_t; + using value_type = T; + + public: + index(const index&) = delete; + index(index&&) = default; + index& operator=(const index&) = delete; + index& operator=(index&&) = default; + ~index() = default; + + /** + * @brief Construct an empty index. + * + * Constructs an empty index. This index will either need to be trained with `build` + * or loaded from a saved copy with `deserialize` + */ + index(raft::resources const& handle); + + /** Construct a brute force index from dataset + * + * Constructs a brute force index from a dataset. This lets us precompute norms for + * the dataset, providing a speed benefit over doing this at query time. + * This index will copy the host dataset onto the device, and take ownership of any + * precaculated norms. + */ + index(raft::resources const& res, + raft::host_matrix_view dataset_view, + std::optional>&& norms, + cuvs::distance::DistanceType metric, + DistT metric_arg = 0.0); + + /** Construct a brute force index from dataset + * + * Constructs a brute force index from a dataset. This lets us precompute norms for + * the dataset, providing a speed benefit over doing this at query time. + * This index will store a non-owning reference to the dataset, but will move + * any norms supplied. + */ + index(raft::resources const& res, + raft::device_matrix_view dataset_view, + std::optional>&& norms, + cuvs::distance::DistanceType metric, + DistT metric_arg = 0.0); + + /** Construct a brute force index from dataset + * + * This class stores a non-owning reference to the dataset and norms. + * Having precomputed norms gives us a performance advantage at query time. + */ + index(raft::resources const& res, + raft::device_matrix_view dataset_view, + std::optional> norms_view, + cuvs::distance::DistanceType metric, + DistT metric_arg = 0.0); + + /** Construct a brute force index from dataset + * + * Constructs a brute force index from a dataset. This lets us precompute norms for + * the dataset, providing a speed benefit over doing this at query time. + * This index will store a non-owning reference to the dataset, but will move + * any norms supplied. + */ + index(raft::resources const& res, + raft::device_matrix_view dataset_view, + std::optional>&& norms, + cuvs::distance::DistanceType metric, + DistT metric_arg = 0.0); + + /** Construct a brute force index from dataset + * + * This class stores a non-owning reference to the dataset and norms, with + * the dataset being supplied on device in a col_major format + */ + index(raft::resources const& res, + raft::device_matrix_view dataset_view, + std::optional> norms_view, + cuvs::distance::DistanceType metric, + DistT metric_arg = 0.0); + + /** + * Replace the dataset with a new dataset. + */ + void update_dataset(raft::resources const& res, + raft::device_matrix_view dataset); + + /** + * Replace the dataset with a new dataset. + * + * We create a copy of the dataset on the device. The index manages the lifetime of this copy. + */ + void update_dataset(raft::resources const& res, + raft::host_matrix_view dataset); + + /** Distance metric used for retrieval */ + cuvs::distance::DistanceType metric() const noexcept { return metric_; } + + /** Metric argument */ + DistT metric_arg() const noexcept { return metric_arg_; } + + /** Total length of the index (number of vectors). */ + size_t size() const noexcept { return dataset_view_.extent(0); } + + /** Dimensionality of the data. */ + size_t dim() const noexcept { return dataset_view_.extent(1); } + + /** Dataset [size, dim] */ + raft::device_matrix_view dataset() const noexcept + { + return dataset_view_; + } + + /** Dataset norms */ + raft::device_vector_view norms() const + { + return norms_view_.value(); + } + + /** Whether ot not this index has dataset norms */ + inline bool has_norms() const noexcept { return norms_view_.has_value(); } + + private: + cuvs::distance::DistanceType metric_; + raft::device_matrix dataset_; + std::optional> norms_; + std::optional> norms_view_; + raft::device_matrix_view dataset_view_; + DistT metric_arg_; +}; +/** + * @} + */ + +/** + * @defgroup bruteforce_cpp_index_build Bruteforce index build + * @{ + */ +/** + * @brief Build the index from the dataset for efficient search. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * @endcode + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a device pointer to a row-major matrix [n_rows, dim] + * + * @return the constructed brute-force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a host pointer to a row-major matrix [n_rows, dim] + * + * @return the constructed brute-force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +[[deprecated]] auto build( + raft::resources const& handle, + raft::device_matrix_view dataset, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Unexpanded, + float metric_arg = 0) -> cuvs::neighbors::brute_force::index; +/** + * @brief Build the index from the dataset for efficient search. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * @endcode + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a device pointer to a row-major matrix [n_rows, dim] + * + * @return the constructed brute force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a host pointer to a row-major matrix [n_rows, dim] + * + * @return the constructed brute-force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +[[deprecated]] auto build( + raft::resources const& handle, + raft::device_matrix_view dataset, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Unexpanded, + float metric_arg = 0) -> cuvs::neighbors::brute_force::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * Usage example: + * @code{.cpp} + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * @endcode + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a device pointer to a col-major matrix [n_rows, dim] + * + * @return the constructed brute force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +[[deprecated]] auto build( + raft::resources const& handle, + raft::device_matrix_view dataset, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Unexpanded, + float metric_arg = 0) -> cuvs::neighbors::brute_force::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * Usage example: + * @code{.cpp} + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * @endcode + * + * @param[in] handle + * @param[in] index_params parameters such as the distance metric to use + * @param[in] dataset a device pointer to a col-major matrix [n_rows, dim] + * + * @return the constructed brute force index + */ +auto build(raft::resources const& handle, + const cuvs::neighbors::brute_force::index_params& index_params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::brute_force::index; + +[[deprecated]] auto build( + raft::resources const& handle, + raft::device_matrix_view dataset, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Unexpanded, + float metric_arg = 0) -> cuvs::neighbors::brute_force::index; +/** + * @} + */ + +/** + * @defgroup bruteforce_cpp_index_search Bruteforce index search + * @{ + */ +/** + * @brief Search ANN using the constructed index. + * + * See the [brute_force::build](#brute_force::build) documentation for a usage example. + * + * Note, this function requires a temporary buffer to store intermediate results between cuda kernel + * calls, which may lead to undesirable allocations and slowdown. To alleviate the problem, you can + * pass a pool memory resource or a large enough pre-allocated memory resource to reduce or + * eliminate entirely allocations happening within `search`. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * + * // use default index parameters + * brute_force::index_params index_params; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * // use default search parameters + * brute_force::search_params search_params; + * // create a bitset to filter the search + * auto removed_indices = raft::make_device_vector(res, n_removed_indices); + * raft::core::bitset removed_indices_bitset( + * res, removed_indices.view(), dataset.extent(0)); + * // search K nearest neighbours according to a bitset + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * auto filter = filtering::bitset_filter(removed_indices_bitset.view()); + * brute_force::search(res, search_params, index, queries, neighbors, distances, filter); + * @endcode + * + * @param[in] handle + * @param[in] params parameters configuring the search + * @param[in] index brute-force constructed index + * @param[in] queries a device pointer to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device pointer to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device pointer to the distances to the selected neighbors [n_queries, k] + * @param[in] sample_filter An optional device filter that restricts which dataset elements should + * be considered for each query. + * + * - Supports two types of filters: + * 1. **Bitset Filter**: A shared filter where each bit corresponds to a dataset element. + * All queries share the same filter, with a logical shape of `[1, index->size()]`. + * 2. **Bitmap Filter**: A per-query filter with a logical shape of `[n_queries, index->size()]`, + * where each bit indicates whether a specific dataset element should be considered for a + * particular query. (1 for inclusion, 0 for exclusion). + * + * - The default value is `none_sample_filter`, which applies no filtering. + */ +void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::search_params& params, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +[[deprecated]] void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [brute_force::build](#brute_force::build) documentation for a usage example. + * + * Note, this function requires a temporary buffer to store intermediate results between cuda kernel + * calls, which may lead to undesirable allocations and slowdown. To alleviate the problem, you can + * pass a pool memory resource or a large enough pre-allocated memory resource to reduce or + * eliminate entirely allocations happening within `search`. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * + * // use default index parameters + * brute_force::index_params index_params; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * // use default search parameters + * brute_force::search_params search_params; + * // create a bitset to filter the search + * auto removed_indices = raft::make_device_vector(res, n_removed_indices); + * raft::core::bitset removed_indices_bitset( + * res, removed_indices.view(), dataset.extent(0)); + * // search K nearest neighbours according to a bitset + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * auto filter = filtering::bitset_filter(removed_indices_bitset.view()); + * brute_force::search(res, search_params, index, queries, neighbors, distances, filter); + * @endcode + * + * @param[in] handle + * @param[in] params parameters configuring the search + * @param[in] index ivf-flat constructed index + * @param[in] queries a device pointer to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device pointer to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device pointer to the distances to the selected neighbors [n_queries, k] + * @param[in] sample_filter An optional device filter that restricts which dataset elements should + * be considered for each query. + * + * - Supports two types of filters: + * 1. **Bitset Filter**: A shared filter where each bit corresponds to a dataset element. + * All queries share the same filter, with a logical shape of `[1, index->size()]`. + * 2. **Bitmap Filter**: A per-query filter with a logical shape of `[n_queries, index->size()]`, + * where each bit indicates whether a specific dataset element should be considered for a + * particular query. (1 for inclusion, 0 for exclusion). + * + * - The default value is `none_sample_filter`, which applies no filtering. + */ +void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::search_params& params, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +[[deprecated]] void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); +/** + * @brief Search ANN using the constructed index. + * + * See the [brute_force::build](#brute_force::build) documentation for a usage example. + * + * Note, this function requires a temporary buffer to store intermediate results between cuda kernel + * calls, which may lead to undesirable allocations and slowdown. To alleviate the problem, you can + * pass a pool memory resource or a large enough pre-allocated memory resource to reduce or + * eliminate entirely allocations happening within `search`. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * + * // use default index parameters + * brute_force::index_params index_params; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * // use default search parameters + * brute_force::search_params search_params; + * // create a bitset to filter the search + * auto removed_indices = raft::make_device_vector(res, n_removed_indices); + * raft::core::bitset removed_indices_bitset( + * res, removed_indices.view(), dataset.extent(0)); + * // search K nearest neighbours according to a bitset + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * auto filter = filtering::bitset_filter(removed_indices_bitset.view()); + * brute_force::search(res, search_params, index, queries, neighbors, distances, filter); + * @endcode + * + * @param[in] handle + * @param[in] params parameters configuring the search + * @param[in] index bruteforce constructed index + * @param[in] queries a device pointer to a col-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device pointer to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device pointer to the distances to the selected neighbors [n_queries, k] + * @param[in] sample_filter An optional device filter that restricts which dataset elements should + * be considered for each query. + * + * - Supports two types of filters: + * 1. **Bitset Filter**: A shared filter where each bit corresponds to a dataset element. + * All queries share the same filter, with a logical shape of `[1, index->size()]`. + * 2. **Bitmap Filter**: A per-query filter with a logical shape of `[n_queries, index->size()]`, + * where each bit indicates whether a specific dataset element should be considered for a + * particular query. (1 for inclusion, 0 for exclusion). + * + * - The default value is `none_sample_filter`, which applies no filtering. + */ +void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::search_params& params, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +[[deprecated]] void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); +/** + * @brief Search ANN using the constructed index. + * + * See the [brute_force::build](#brute_force::build) documentation for a usage example. + * + * Note, this function requires a temporary buffer to store intermediate results between cuda kernel + * calls, which may lead to undesirable allocations and slowdown. To alleviate the problem, you can + * pass a pool memory resource or a large enough pre-allocated memory resource to reduce or + * eliminate entirely allocations happening within `search`. + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * + * // use default index parameters + * brute_force::index_params index_params; + * // create and fill the index from a [N, D] dataset + * brute_force::index_params index_params; + * auto index = brute_force::build(handle, index_params, dataset); + * // use default search parameters + * brute_force::search_params search_params; + * // create a bitset to filter the search + * auto removed_indices = raft::make_device_vector(res, n_removed_indices); + * raft::core::bitset removed_indices_bitset( + * res, removed_indices.view(), dataset.extent(0)); + * // search K nearest neighbours according to a bitset + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * auto filter = filtering::bitset_filter(removed_indices_bitset.view()); + * brute_force::search(res, search_params, index, queries, neighbors, distances, filter); + * @endcode + * + * @param[in] handle + * @param[in] params parameters configuring the search + * @param[in] index bruteforce constructed index + * @param[in] queries a device pointer to a col-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device pointer to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device pointer to the distances to the selected neighbors [n_queries, k] + * @param[in] sample_filter An optional device filter that restricts which dataset elements should + * be considered for each query. + * + * - Supports two types of filters: + * 1. **Bitset Filter**: A shared filter where each bit corresponds to a dataset element. + * All queries share the same filter, with a logical shape of `[1, index->size()]`. + * 2. **Bitmap Filter**: A per-query filter with a logical shape of `[n_queries, index->size()]`, + * where each bit indicates whether a specific dataset element should be considered for a + * particular query. (1 for inclusion, 0 for exclusion). + * + * - The default value is `none_sample_filter`, which applies no filtering. + */ +void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::search_params& params, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +[[deprecated]] void search(raft::resources const& handle, + const cuvs::neighbors::brute_force::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); +/** + * @} + */ + +/** + * @defgroup sparse_bruteforce_cpp_index Sparse Brute Force index + * @{ + */ +/** + * @brief Sparse Brute Force index. + * + * @tparam T Data element type + * @tparam IdxT Index element type + */ +template +struct sparse_index { + public: + sparse_index(const sparse_index&) = delete; + sparse_index(sparse_index&&) = default; + sparse_index& operator=(const sparse_index&) = delete; + sparse_index& operator=(sparse_index&&) = default; + ~sparse_index() = default; + + /** Construct a sparse brute force sparse_index from dataset */ + sparse_index(raft::resources const& res, + raft::device_csr_matrix_view dataset, + cuvs::distance::DistanceType metric, + T metric_arg); + + /** Distance metric used for retrieval */ + cuvs::distance::DistanceType metric() const noexcept { return metric_; } + + /** Metric argument */ + T metric_arg() const noexcept { return metric_arg_; } + + raft::device_csr_matrix_view dataset() const noexcept + { + return dataset_; + } + + private: + raft::device_csr_matrix_view dataset_; + cuvs::distance::DistanceType metric_; + T metric_arg_; +}; +/** + * @} + */ + +/** + * @defgroup sparse_bruteforce_cpp_index_build Sparse Brute Force index build + * @{ + */ + +/* + * @brief Build the Sparse index from the dataset + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // create and fill the index from a CSR dataset + * auto index = brute_force::build(handle, dataset, metric); + * @endcode + * + * @param[in] handle + * @param[in] dataset A sparse CSR matrix in device memory to search against + * @param[in] metric cuvs::distance::DistanceType + * @param[in] metric_arg metric argument + * + * @return the constructed Sparse brute-force index + */ +auto build(raft::resources const& handle, + raft::device_csr_matrix_view dataset, + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Unexpanded, + float metric_arg = 0) -> cuvs::neighbors::brute_force::sparse_index; +/** + * @} + */ + +/** + * @defgroup sparse_bruteforce_cpp_index_search Sparse Brute Force index search + * @{ + */ +struct sparse_search_params { + int batch_size_index = 2 << 14; + int batch_size_query = 2 << 14; +}; + +/* + * @brief Search the sparse bruteforce index for nearest neighbors + * + * @param[in] handle + * @param[in] params parameters configuring the search + * @param[in] index Sparse brute-force constructed index + * @param[in] queries a sparse CSR matrix on the device to query + * @param[out] neighbors a device pointer to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device pointer to the distances to the selected neighbors [n_queries, k] + */ +void search(raft::resources const& handle, + const sparse_search_params& params, + const sparse_index& index, + raft::device_csr_matrix_view dataset, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances); +/** + * @} + */ + +/** + * @defgroup bruteforce_cpp_index_serialize Bruteforce index serialize functions + * @{ + */ +/** + * Save the index to file. + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = brute_force::build(...);` + * cuvs::neighbors::brute_force::serialize(handle, filename, index); + * @endcode + * + * @tparam T data element type + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index brute force index + * @param[in] include_dataset whether to include the dataset in the serialized + * output + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::brute_force::index& index, + bool include_dataset = true); +/** + * Save the index to file. + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = brute_force::build(...);` + * cuvs::neighbors::brute_force::serialize(handle, filename, index); + * @endcode + * + * @tparam T data element type + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index brute force index + * @param[in] include_dataset whether to include the dataset in the serialized + * output + * + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::brute_force::index& index, + bool include_dataset = true); + +/** + * Write the index to an output stream + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::brute_force::build(...);` + * cuvs::neighbors::brute_force::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index brute force index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::brute_force::index& index, + bool include_dataset = true); + +/** + * Write the index to an output stream + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::brute_force::build(...);` + * cuvs::neighbors::brute_force::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index brute force index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::brute_force::index& index, + bool include_dataset = true); + +/** + * Load index from file. + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * using T = half; // data element type + * brute_force::index index(handle); + * cuvs::neighbors::brute_force::deserialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index brute force index + * + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::brute_force::index* index); +/** + * Load index from file. + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * using T = float; // data element type + * brute_force::index index(handle); + * cuvs::neighbors::brute_force::deserialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index brute force index + * + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::brute_force::index* index); +/** + * Load index from input stream + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * using T = half; // data element type + * brute_force::index index(handle); + * cuvs::neighbors::brute_force::deserialize(handle, is, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index brute force index + * + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::brute_force::index* index); +/** + * Load index from input stream + * The serialization format can be subject to changes, therefore loading + * an index saved with a previous version of cuvs is not guaranteed + * to work. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * using T = float; // data element type + * brute_force::index index(handle); + * cuvs::neighbors::brute_force::deserialize(handle, is, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index brute force index + * + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::brute_force::index* index); +/** + * @} + */ + +} // namespace cuvs::neighbors::brute_force diff --git a/cpp/include/cuvs/neighbors/cagra.hpp b/cpp/include/cuvs/neighbors/cagra.hpp index 3a0c60b782..c8d1393046 100644 --- a/cpp/include/cuvs/neighbors/cagra.hpp +++ b/cpp/include/cuvs/neighbors/cagra.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023-2024, NVIDIA CORPORATION. + * Copyright (c) 2023-2025, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,70 +16,129 @@ #pragma once -#include "ann_types.hpp" -#include +#include "common.hpp" +#include +#include +#include +#include +#include #include #include +#include #include +#include +#include #include -#include +#include #include +#include +#include + namespace cuvs::neighbors::cagra { +// For re-exporting into cagra namespace +namespace graph_build_params = cuvs::neighbors::graph_build_params; /** - * @addtogroup cagra + * @defgroup cagra_cpp_index_params CAGRA index build parameters * @{ */ -/** - * @brief ANN algorithm used by CAGRA to build knn graph - * - */ -enum class graph_build_algo { - /* Use IVF-PQ to build all-neighbors knn graph */ - IVF_PQ, - /* Experimental, use NN-Descent to build all-neighbors knn graph */ - NN_DESCENT -}; - -struct index_params : ann::index_params { +struct index_params : cuvs::neighbors::index_params { /** Degree of input graph for pruning. */ size_t intermediate_graph_degree = 128; /** Degree of output graph. */ size_t graph_degree = 64; - /** ANN algorithm to build knn graph. */ - graph_build_algo build_algo = graph_build_algo::IVF_PQ; - /** Number of Iterations to run if building with NN_DESCENT */ - size_t nn_descent_niter = 20; + /** + * Specify compression parameters if compression is desired. If set, overrides the + * attach_dataset_on_build (and the compressed dataset is always added to the index). + */ + std::optional compression = std::nullopt; - /** Build a raft CAGRA index params from an existing cuvs CAGRA index params. */ - operator raft::neighbors::cagra::index_params() const - { - return raft::neighbors::cagra::index_params{ - { - .metric = static_cast((int)this->metric), - .metric_arg = this->metric_arg, - .add_data_on_build = this->add_data_on_build, - }, - .intermediate_graph_degree = intermediate_graph_degree, - .graph_degree = graph_degree, - .build_algo = static_cast((int)build_algo), - .nn_descent_niter = nn_descent_niter}; - } + /** Parameters for graph building. + * + * Set ivf_pq_params, nn_descent_params, or iterative_search_params to select the graph build + * algorithm and control their parameters. The default (std::monostate) is to use a heuristic + * to decide the algorithm and its parameters. + * + * @code{.cpp} + * cagra::index_params params; + * // 1. Choose IVF-PQ algorithm + * params.graph_build_params = cagra::graph_build_params::ivf_pq_params(dataset.extent, + * params.metric); + * + * // 2. Choose NN Descent algorithm for kNN graph construction + * params.graph_build_params = + * cagra::graph_build_params::nn_descent_params(params.intermediate_graph_degree); + * + * // 3. Choose iterative graph building using CAGRA's search() and optimize() [Experimental] + * params.graph_build_params = + * cagra::graph_build_params::iterative_search_params(); + * @endcode + */ + std::variant + graph_build_params; + + /** + * Whether to use MST optimization to guarantee graph connectivity. + */ + bool guarantee_connectivity = false; + + /** + * Whether to add the dataset content to the index, i.e.: + * + * - `true` means the index is filled with the dataset vectors and ready to search after calling + * `build` provided there is enough memory available. + * - `false` means `build` only builds the graph and the user is expected to + * update the dataset using cuvs::neighbors::cagra::update_dataset. + * + * Regardless of the value of `attach_dataset_on_build`, the search graph is created using all + * the vectors in the dataset. Setting `attach_dataset_on_build = false` can be useful if + * the user needs to build only the search graph but does not intend to search it using CAGRA + * (e.g. search using another graph search algorithm), or if specific memory placement options + * need to be applied on the dataset before it is attached to the index using `update_dataset`. + * API. + * @code{.cpp} + * auto dataset = raft::make_device_matrix(res, n_rows, n_cols); + * // use default index_parameters + * cagra::index_params index_params; + * // update index_params to only build the CAGRA graph + * index_params.attach_dataset_on_build = false; + * auto index = cagra::build(res, index_params, dataset.view()); + * // assert that the dataset is not attached to the index + * ASSERT(index.dataset().extent(0) == 0); + * // update dataset + * index.update_dataset(res, dataset.view()); + * // The index is now ready for search + * cagra::search(res, search_params, index, queries, neighbors, distances); + * @endcode + */ + bool attach_dataset_on_build = true; }; +/** + * @} + */ + +/** + * @defgroup cagra_cpp_search_params CAGRA index search parameters + * @{ + */ + enum class search_algo { /** For large batch sizes. */ - SINGLE_CTA, + SINGLE_CTA = 0, /** For small batch sizes. */ - MULTI_CTA, - MULTI_KERNEL, - AUTO + MULTI_CTA = 1, + MULTI_KERNEL = 2, + AUTO = 100 }; -enum class hash_mode { HASH, SMALL, AUTO }; +enum class hash_mode { HASH = 0, SMALL = 1, AUTO = 100 }; -struct search_params : ann::search_params { +struct search_params : cuvs::neighbors::search_params { /** Maximum number of queries to search at the same time (batch size). Auto select when 0.*/ size_t max_queries = 0; @@ -122,48 +181,117 @@ struct search_params : ann::search_params { /** Bit mask used for initial random seed node selection. */ uint64_t rand_xor_mask = 0x128394; - /** Build a raft CAGRA search params from an existing cuvs CAGRA search params. */ - operator raft::neighbors::cagra::search_params() const - { - raft::neighbors::cagra::search_params result = { - {}, - max_queries, - itopk_size, - max_iterations, - static_cast((int)algo), - team_size, - search_width, - min_iterations, - thread_block_size, - static_cast((int)hashmap_mode), - hashmap_min_bitlen, - hashmap_max_fill_rate, - num_random_samplings, - rand_xor_mask}; - return result; - } + /** Whether to use the persistent version of the kernel (only SINGLE_CTA is supported a.t.m.) */ + bool persistent = false; + /** Persistent kernel: time in seconds before the kernel stops if no requests received. */ + float persistent_lifetime = 2; + /** + * Set the fraction of maximum grid size used by persistent kernel. + * Value 1.0 means the kernel grid size is maximum possible for the selected device. + * The value must be greater than 0.0 and not greater than 1.0. + * + * One may need to run other kernels alongside this persistent kernel. This parameter can + * be used to reduce the grid size of the persistent kernel to leave a few SMs idle. + * Note: running any other work on GPU alongside with the persistent kernel makes the setup + * fragile. + * - Running another kernel in another thread usually works, but no progress guaranteed + * - Any CUDA allocations block the context (this issue may be obscured by using pools) + * - Memory copies to not-pinned host memory may block the context + * + * Even when we know there are no other kernels working at the same time, setting + * kDeviceUsage to 1.0 surprisingly sometimes hurts performance. Proceed with care. + * If you suspect this is an issue, you can reduce this number to ~0.9 without a significant + * impact on the throughput. + */ + float persistent_device_usage = 1.0; + + /** + * A parameter indicating the rate of nodes to be filtered-out, when filtering is used. + * The value must be equal to or greater than 0.0 and less than 1.0. Default value is + * negative, in which case the filtering rate is automatically calculated. + */ + float filtering_rate = -1.0; +}; + +/** + * @} + */ + +/** + * @defgroup cagra_cpp_extend_params CAGRA index extend parameters + * @{ + */ + +struct extend_params { + /** The additional dataset is divided into chunks and added to the graph. This is the knob to + * adjust the tradeoff between the recall and operation throughput. Large chunk sizes can result + * in high throughput, but use more working memory (O(max_chunk_size*degree^2)). This can also + * degrade recall because no edges are added between the nodes in the same chunk. Auto select when + * 0. */ + uint32_t max_chunk_size = 0; +}; +/** + * @} + */ + +/** + * @defgroup cagra_cpp_merge_params CAGRA index merge parameters + * @{ + */ + +/** + * @brief Parameters for merging CAGRA indexes. + */ +struct merge_params : cuvs::neighbors::merge_params { + merge_params() = default; + + /** + * @brief Constructs merge parameters with given index parameters. + * @param params Parameters for creating the output index. + */ + explicit merge_params(const cagra::index_params& params) : output_index_params(params) {} + + /// Parameters for creating the output index. + cagra::index_params output_index_params; + + /// Strategy for merging. Defaults to `MergeStrategy::MERGE_STRATEGY_PHYSICAL`. + cuvs::neighbors::MergeStrategy merge_strategy = + cuvs::neighbors::MergeStrategy::MERGE_STRATEGY_PHYSICAL; + + /// Implementation of the polymorphic strategy() method + cuvs::neighbors::MergeStrategy strategy() const { return merge_strategy; } }; +/** + * @} + */ + static_assert(std::is_aggregate_v); static_assert(std::is_aggregate_v); +/** + * @defgroup cagra_cpp_index CAGRA index type + * @{ + */ + /** * @brief CAGRA index. * * The index stores the dataset and a kNN graph in device memory. * * @tparam T data element type - * @tparam IdxT type of the vector indices (represent dataset.extent(0)) + * @tparam IdxT the data type used to store the neighbor indices in the search graph. + * It must be large enough to represent values up to dataset.extent(0). * */ template -struct index : ann::index { - /** Build a cuvs CAGRA index from an existing RAFT CAGRA index. */ - index(raft::neighbors::cagra::index&& raft_idx) - : ann::index(), - raft_index_{std::make_unique>(std::move(raft_idx))} - { - } +struct index : cuvs::neighbors::index { + using index_params_type = cagra::index_params; + using search_params_type = cagra::search_params; + using index_type = IdxT; + using value_type = T; + using dataset_index_type = int64_t; + static_assert(!raft::is_narrowing_v, "IdxT must be able to represent all values of uint32_t"); @@ -171,52 +299,74 @@ struct index : ann::index { /** Distance metric used for clustering. */ [[nodiscard]] constexpr inline auto metric() const noexcept -> cuvs::distance::DistanceType { - return static_cast((int)raft_index_->metric()); + return metric_; } /** Total length of the index (number of vectors). */ - [[nodiscard]] constexpr inline auto size() const noexcept -> IdxT { return raft_index_->size(); } - - /** Dimensionality of the data. */ - [[nodiscard]] constexpr inline auto dim() const noexcept -> uint32_t + [[nodiscard]] constexpr inline auto size() const noexcept -> IdxT { - return raft_index_->dim(); + auto data_rows = dataset_->n_rows(); + return data_rows > 0 ? data_rows : graph_view_.extent(0); } + + /** Dimensionality of the data. */ + [[nodiscard]] constexpr inline auto dim() const noexcept -> uint32_t { return dataset_->dim(); } /** Graph degree */ [[nodiscard]] constexpr inline auto graph_degree() const noexcept -> uint32_t { - return raft_index_->graph_degree(); + return graph_view_.extent(1); } - /** Dataset [size, dim] */ [[nodiscard]] inline auto dataset() const noexcept -> raft::device_matrix_view { - return raft_index_->dataset(); + auto p = dynamic_cast*>(dataset_.get()); + if (p != nullptr) { return p->view(); } + auto d = dataset_->dim(); + return raft::make_device_strided_matrix_view(nullptr, 0, d, d); + } + + /** Dataset [size, dim] */ + [[nodiscard]] inline auto data() const noexcept -> const cuvs::neighbors::dataset& + { + return *dataset_; } /** neighborhood graph [size, graph-degree] */ [[nodiscard]] inline auto graph() const noexcept -> raft::device_matrix_view { - return raft_index_->graph(); + return graph_view_; + } + + /** Dataset norms for cosine distance [size] */ + [[nodiscard]] inline auto dataset_norms() const noexcept + -> std::optional> + { + if (dataset_norms_.has_value()) { return raft::make_const_mdspan(dataset_norms_->view()); } + return std::nullopt; } // Don't allow copying the index for performance reasons (try avoiding copying data) + /** \cond */ index(const index&) = delete; index(index&&) = default; auto operator=(const index&) -> index& = delete; auto operator=(index&&) -> index& = default; ~index() = default; + /** \endcond */ /** Construct an empty index. */ index(raft::resources const& res, cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded) - : ann::index(), - raft_index_(std::make_unique>( - res, static_cast((int)metric))) + : cuvs::neighbors::index(), + metric_(metric), + graph_(raft::make_device_matrix(res, 0, 0)), + dataset_(new cuvs::neighbors::empty_dataset(0)), + dataset_norms_(std::nullopt) { } + /** Construct an index from dataset and knn_graph arrays * * If the dataset and graph is already in GPU memory, then the index is just a thin wrapper around @@ -232,7 +382,7 @@ struct index : ann::index { * * - Cagra index is normally created by the cagra::build * @code{.cpp} - * using namespace cuvs::neighbors::experimental; + * using namespace raft::neighbors::experimental; * auto dataset = raft::make_host_matrix(n_rows, n_cols); * load_dataset(dataset.view()); * // use default index parameters @@ -244,15 +394,15 @@ struct index : ann::index { * // search K nearest neighbours * auto neighbors = raft::make_device_matrix(res, n_queries, k); * auto distances = raft::make_device_matrix(res, n_queries, k); - * cagra::search(res, search_params, index, queries, neighbors, distances); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); * @endcode * In the above example, we have passed a host dataset to build. The returned index will own a * device copy of the dataset and the knn_graph. In contrast, if we pass the dataset as a - * raft::device_mdspan to build, then it will only store a reference to it. + * device_mdspan to build, then it will only store a reference to it. * * - Constructing index using existing knn-graph * @code{.cpp} - * using namespace cuvs::neighbors::experimental; + * using namespace raft::neighbors::experimental; * * auto dataset = raft::make_device_matrix(res, n_rows, n_cols); * auto knn_graph = raft::make_device_matrix(res, n_rows, graph_degree); @@ -269,7 +419,6 @@ struct index : ann::index { * // the index only stores a reference to these. * cagra::search(res, search_params, index, queries, neighbors, distances); * @endcode - * */ template index(raft::resources const& res, @@ -277,14 +426,24 @@ struct index : ann::index { raft::mdspan, raft::row_major, data_accessor> dataset, raft::mdspan, raft::row_major, graph_accessor> knn_graph) - : ann::index(), - raft_index_(std::make_unique>( - res, static_cast((int)metric), dataset, knn_graph)) + : cuvs::neighbors::index(), + metric_(metric), + graph_(raft::make_device_matrix(res, 0, 0)), + dataset_(make_aligned_dataset(res, dataset, 16)), + dataset_norms_(std::nullopt) { RAFT_EXPECTS(dataset.extent(0) == knn_graph.extent(0), "Dataset and knn_graph must have equal number of rows"); - update_dataset(res, dataset); update_graph(res, knn_graph); + + if (metric_ == cuvs::distance::DistanceType::CosineExpanded) { + auto p = dynamic_cast*>(dataset_.get()); + if (p) { + auto dataset_view = p->view(); + if (dataset_view.extent(0) > 0) { compute_dataset_norms_(res); } + } + } + raft::resource::sync_stream(res); } @@ -293,22 +452,82 @@ struct index : ann::index { * * If the new dataset rows are aligned on 16 bytes, then only a reference is stored to the * dataset. It is the caller's responsibility to ensure that dataset stays alive as long as the - * index. + * index. It is expected that the same set of vectors are used for update_dataset and index build. + * + * Note: This will clear any precomputed dataset norms. */ void update_dataset(raft::resources const& res, raft::device_matrix_view dataset) { - raft_index_->update_dataset(res, dataset); + dataset_ = make_aligned_dataset(res, dataset, 16); + dataset_norms_.reset(); + + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + if (dataset.extent(0) > 0) { compute_dataset_norms_(res); } + } + } + + /** Set the dataset reference explicitly to a device matrix view with padding. */ + void update_dataset(raft::resources const& res, + raft::device_matrix_view dataset) + { + dataset_ = make_aligned_dataset(res, dataset, 16); + dataset_norms_.reset(); + + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + if (dataset.extent(0) > 0) { compute_dataset_norms_(res); } + } } + /** * Replace the dataset with a new dataset. * - * We create a copy of the dataset on the device. The index manages the lifetime of this copy. + * We create a copy of the dataset on the device. The index manages the lifetime of this copy. It + * is expected that the same set of vectors are used for update_dataset and index build. + * + * Note: This will clear any precomputed dataset norms. */ void update_dataset(raft::resources const& res, raft::host_matrix_view dataset) { - raft_index_->update_dataset(res, dataset); + dataset_ = make_aligned_dataset(res, dataset, 16); + dataset_norms_.reset(); + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + if (dataset.extent(0) > 0) { compute_dataset_norms_(res); } + } + } + + /** + * Replace the dataset with a new dataset. It is expected that the same set of vectors are used + * for update_dataset and index build. + * + * Note: This will clear any precomputed dataset norms. + */ + template + auto update_dataset(raft::resources const& res, DatasetT&& dataset) + -> std::enable_if_t, DatasetT>> + { + dataset_ = std::make_unique(std::move(dataset)); + dataset_norms_.reset(); + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + auto p = dynamic_cast*>(dataset_.get()); + if (p) { + auto dataset_view = p->view(); + if (dataset_view.extent(0) > 0) { compute_dataset_norms_(res); } + } + } + } + + template + auto update_dataset(raft::resources const& res, std::unique_ptr&& dataset) + -> std::enable_if_t, DatasetT>> + { + dataset_ = std::move(dataset); + dataset_norms_.reset(); + if (metric() == cuvs::distance::DistanceType::CosineExpanded) { + auto dataset_view = this->dataset(); + if (dataset_view.extent(0) > 0) { compute_dataset_norms_(res); } + } } /** @@ -320,7 +539,7 @@ struct index : ann::index { void update_graph(raft::resources const& res, raft::device_matrix_view knn_graph) { - raft_index_->update_graph(res, knn_graph); + graph_view_ = knn_graph; } /** @@ -331,83 +550,2209 @@ struct index : ann::index { void update_graph(raft::resources const& res, raft::host_matrix_view knn_graph) { - raft_index_->update_graph(res, knn_graph); - } + RAFT_LOG_DEBUG("Copying CAGRA knn graph from host to device"); - auto get_raft_index() const -> const raft::neighbors::cagra::index* - { - return raft_index_.get(); + if ((graph_.extent(0) != knn_graph.extent(0)) || (graph_.extent(1) != knn_graph.extent(1))) { + // clear existing memory before allocating to prevent OOM errors on large graphs + if (graph_.size()) { graph_ = raft::make_device_matrix(res, 0, 0); } + graph_ = + raft::make_device_matrix(res, knn_graph.extent(0), knn_graph.extent(1)); + } + raft::copy(graph_.data_handle(), + knn_graph.data_handle(), + knn_graph.size(), + raft::resource::get_cuda_stream(res)); + graph_view_ = graph_.view(); } - auto get_raft_index() -> raft::neighbors::cagra::index* { return raft_index_.get(); } private: - std::unique_ptr> raft_index_; + cuvs::distance::DistanceType metric_; + raft::device_matrix graph_; + raft::device_matrix_view graph_view_; + std::unique_ptr> dataset_; + // only float distances supported at the moment + std::optional> dataset_norms_; + + void compute_dataset_norms_(raft::resources const& res); }; +/** + * @} + */ -// Using device and host_matrix_view avoids needing to typedef multiple mdspans based on accessors -#define CUVS_INST_CAGRA_FUNCS(T, IdxT) \ - auto build(raft::resources const& handle, \ - const cuvs::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset) \ - ->cuvs::neighbors::cagra::index; \ - \ - auto build(raft::resources const& handle, \ - const cuvs::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset) \ - ->cuvs::neighbors::cagra::index; \ - \ - void build_device(raft::resources const& handle, \ - const cuvs::neighbors::cagra::index_params& params, \ - raft::device_matrix_view dataset, \ - cuvs::neighbors::cagra::index& idx); \ - \ - void build_host(raft::resources const& handle, \ - const cuvs::neighbors::cagra::index_params& params, \ - raft::host_matrix_view dataset, \ - cuvs::neighbors::cagra::index& idx); \ - \ - void search(raft::resources const& handle, \ - cuvs::neighbors::cagra::search_params const& params, \ - const cuvs::neighbors::cagra::index& index, \ - raft::device_matrix_view queries, \ - raft::device_matrix_view neighbors, \ - raft::device_matrix_view distances); \ - void serialize_file(raft::resources const& handle, \ - const std::string& filename, \ - const cuvs::neighbors::cagra::index& index, \ - bool include_dataset = true); \ - \ - void deserialize_file(raft::resources const& handle, \ - const std::string& filename, \ - cuvs::neighbors::cagra::index* index); \ - void serialize(raft::resources const& handle, \ - std::string& str, \ - const cuvs::neighbors::cagra::index& index, \ - bool include_dataset = true); \ - \ - void deserialize(raft::resources const& handle, \ - const std::string& str, \ - cuvs::neighbors::cagra::index* index); - -CUVS_INST_CAGRA_FUNCS(float, uint32_t); -CUVS_INST_CAGRA_FUNCS(int8_t, uint32_t); -CUVS_INST_CAGRA_FUNCS(uint8_t, uint32_t); - -#undef CUVS_INST_CAGRA_FUNCS - -#define CUVS_INST_CAGRA_OPTIMIZE(IdxT) \ - void optimize_device(raft::resources const& res, \ - raft::device_matrix_view knn_graph, \ - raft::host_matrix_view new_graph); \ - \ - void optimize_host(raft::resources const& res, \ - raft::host_matrix_view knn_graph, \ - raft::host_matrix_view new_graph); - -CUVS_INST_CAGRA_OPTIMIZE(uint32_t); - -#undef CUVS_INST_CAGRA_OPTIMIZE - -/** @} */ +/** + * @defgroup cagra_cpp_index_build CAGRA index build functions + * @{ + */ -} // namespace cuvs::neighbors::cagra +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (device) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (host) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (device) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (host) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (device) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (host) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (device) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::device_matrix_view dataset) + -> cuvs::neighbors::cagra::index; + +/** + * @brief Build the index from the dataset for efficient search. + * + * The build consist of two steps: build an intermediate knn-graph, and optimize it to + * create the final graph. The index_params struct controls the node degree of these + * graphs. + * + * The following distance metrics are supported: + * - L2 + * - InnerProduct (currently only supported with IVF-PQ as the build algorithm) + * - CosineExpanded (dataset norms are computed as float regardless of input data type) + * + * Usage example: + * @code{.cpp} + * using namespace cuvs::neighbors; + * // use default index parameters + * cagra::index_params index_params; + * // create and fill the index from a [N, D] dataset + * auto index = cagra::build(res, index_params, dataset); + * // use default search parameters + * cagra::search_params search_params; + * // search K nearest neighbours + * auto neighbors = raft::make_device_matrix(res, n_queries, k); + * auto distances = raft::make_device_matrix(res, n_queries, k); + * cagra::search(res, search_params, index, queries, neighbors.view(), distances.view()); + * @endcode + * + * @param[in] res + * @param[in] params parameters for building the index + * @param[in] dataset a matrix view (host) to a row-major matrix [n_rows, dim] + * + * @return the constructed cagra index + */ +auto build(raft::resources const& res, + const cuvs::neighbors::cagra::index_params& params, + raft::host_matrix_view dataset) + -> cuvs::neighbors::cagra::index; +/** + * @} + */ + +/** + * @defgroup cagra_cpp_index_extend CAGRA extend functions + * @{ + */ + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_device_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on device memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::device_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_host_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on host memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::host_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_device_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on device memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::device_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_host_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on host memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::host_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_host_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on host memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::device_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); + +/** @brief Add new vectors to a CAGRA index + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto additional_dataset = raft::make_host_matrix(handle,add_size,dim); + * // set_additional_dataset(additional_dataset.view()); + * + * cagra::extend_params params; + * cagra::extend(res, params, raft::make_const_mdspan(additional_dataset.view()), index); + * @endcode + * + * @param[in] handle raft resources + * @param[in] params extend params + * @param[in] additional_dataset additional dataset on host memory + * @param[in,out] idx CAGRA index + * @param[out] new_dataset_buffer_view memory buffer view for the dataset including the additional + * part. The data will be copied from the current index in this function. The num rows must be the + * sum of the original and additional datasets, cols must be the dimension of the dataset, and the + * stride must be the same as the original index dataset. This view will be stored in the output + * index. It is the caller's responsibility to ensure that dataset stays alive as long as the index. + * This option is useful when users want to manage the memory space for the dataset themselves. + * @param[out] new_graph_buffer_view memory buffer view for the graph including the additional part. + * The data will be copied from the current index in this function. The num rows must be the sum of + * the original and additional datasets and cols must be the graph degree. This view will be stored + * in the output index. It is the caller's responsibility to ensure that dataset stays alive as long + * as the index. This option is useful when users want to manage the memory space for the graph + * themselves. + */ +void extend( + raft::resources const& handle, + const cagra::extend_params& params, + raft::host_matrix_view additional_dataset, + cuvs::neighbors::cagra::index& idx, + std::optional> + new_dataset_buffer_view = std::nullopt, + std::optional> new_graph_buffer_view = std::nullopt); +/** + * @} + */ + +/** + * @defgroup cagra_cpp_index_search CAGRA search functions + * @{ + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ + +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ + +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @brief Search ANN using the constructed index. + * + * See the [cagra::build](#cagra::build) documentation for a usage example. + * + * @param[in] res raft resources + * @param[in] params configure the search + * @param[in] index cagra index + * @param[in] queries a device matrix view to a row-major matrix [n_queries, index->dim()] + * @param[out] neighbors a device matrix view to the indices of the neighbors in the source dataset + * [n_queries, k] + * @param[out] distances a device matrix view to the distances to the selected neighbors [n_queries, + * k] + * @param[in] sample_filter an optional device filter function object that greenlights samples + * for a given query. (none_sample_filter for no filtering) + */ +void search(raft::resources const& res, + cuvs::neighbors::cagra::search_params const& params, + const cuvs::neighbors::cagra::index& index, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& sample_filter = + cuvs::neighbors::filtering::none_sample_filter{}); + +/** + * @} + */ + +/** + * @defgroup cagra_cpp_serialize CAGRA serialize functions + * @{ + */ + +/** + * Save the index to file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + * + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, filename, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::cagra::index* index); + +/** + * Write the index to an output stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from input stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, is, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::cagra::index* index); +/** + * Save the index to file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + * + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, filename, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::cagra::index* index); + +/** + * Write the index to an output stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from input stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, is, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::cagra::index* index); + +/** + * Save the index to file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, filename, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::cagra::index* index); + +/** + * Write the index to an output stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from input stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, is, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::cagra::index* index); + +/** + * Save the index to file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, filename, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from file. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, filename, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] filename the name of the file that stores the index + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + const std::string& filename, + cuvs::neighbors::cagra::index* index); + +/** + * Write the index to an output stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] include_dataset Whether or not to write out the dataset to the file. + */ +void serialize(raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + bool include_dataset = true); + +/** + * Load index from input stream + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an input stream + * std::istream is(std::cin.rdbuf()); + * cuvs::neighbors::cagra::index index; + * cuvs::neighbors::cagra::deserialize(handle, is, &index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] is input stream + * @param[out] index the cagra index + */ +void deserialize(raft::resources const& handle, + std::istream& is, + cuvs::neighbors::cagra::index* index); + +/** + * Write the CAGRA built index as a base layer HNSW index to an output stream + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Save a CAGRA build index in hnswlib base-layer-only serialized format + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); + * @endcode + * + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Write the CAGRA built index as a base layer HNSW index to an output stream + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Save a CAGRA build index in hnswlib base-layer-only serialized format + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); + * @endcode + * + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Write the CAGRA built index as a base layer HNSW index to an output stream + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Save a CAGRA build index in hnswlib base-layer-only serialized format + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); + * @endcode + * + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Write the CAGRA built index as a base layer HNSW index to an output stream + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create an output stream + * std::ostream os(std::cout.rdbuf()); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, os, index); + * @endcode + * + * @param[in] handle the raft handle + * @param[in] os output stream + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + std::ostream& os, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * Save a CAGRA build index in hnswlib base-layer-only serialized format + * NOTE: The saved index can only be read by the hnswlib wrapper in cuVS, + * as the serialization format is not compatible with the original hnswlib. + * + * Experimental, both the API and the serialization format are subject to change. + * + * @code{.cpp} + * #include + * #include + * + * raft::resources handle; + * + * // create a string with a filepath + * std::string filename("/path/to/index"); + * // create an index with `auto index = cuvs::neighbors::cagra::build(...);` + * cuvs::neighbors::cagra::serialize_to_hnswlib(handle, filename, index); + * @endcode + * + * + * @param[in] handle the raft handle + * @param[in] filename the file name for saving the index + * @param[in] index CAGRA index + * @param[in] dataset [optional] host array that stores the dataset, required if the index + * does not contain the dataset. + * + */ +void serialize_to_hnswlib( + raft::resources const& handle, + const std::string& filename, + const cuvs::neighbors::cagra::index& index, + std::optional> dataset = + std::nullopt); + +/** + * @} + */ + +/** + * @defgroup cagra_cpp_index_merge CAGRA index build functions + * @{ + */ + +/** @brief Merge multiple CAGRA indices into a single index. + * + * This function merges multiple CAGRA indices into one, combining both the datasets and graph + * structures. + * + * @note: When device memory is sufficient, the dataset attached to the returned index is allocated + * in device memory by default; otherwise, host memory is used automatically. + * + * @note: This API only supports physical merge (`merge_strategy = MERGE_STRATEGY_PHYSICAL`), and + * attempting a logical merge here will throw an error. + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto dataset0 = raft::make_host_matrix(handle, size0, dim); + * auto dataset1 = raft::make_host_matrix(handle, size1, dim); + * + * auto index0 = cagra::build(res, index_params, dataset0); + * auto index1 = cagra::build(res, index_params, dataset1); + * + * std::vector*> indices{&index0, &index1}; + * cagra::merge_params params{index_params}; + * + * auto merged_index = cagra::merge(res, params, indices); + * @endcode + * + * @param[in] res RAFT resources used for the merge operation. + * @param[in] params Parameters that control the merging process. + * @param[in] indices A vector of pointers to the CAGRA indices to merge. All indices must: + * - Have attached datasets with the same dimension. + * + * @return A new CAGRA index containing the merged indices, graph, and dataset. + */ +auto merge(raft::resources const& res, + const cuvs::neighbors::cagra::merge_params& params, + std::vector*>& indices) + -> cuvs::neighbors::cagra::index; + +/** @brief Merge multiple CAGRA indices into a single index. + * + * This function merges multiple CAGRA indices into one, combining both the datasets and graph + * structures. + * + * @note: When device memory is sufficient, the dataset attached to the returned index is allocated + * in device memory by default; otherwise, host memory is used automatically. + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto dataset0 = raft::make_host_matrix(handle, size0, dim); + * auto dataset1 = raft::make_host_matrix(handle, size1, dim); + * + * auto index0 = cagra::build(res, index_params, dataset0); + * auto index1 = cagra::build(res, index_params, dataset1); + * + * std::vector*> indices{&index0, &index1}; + * cagra::merge_params params{index_params}; + * + * auto merged_index = cagra::merge(res, params, indices); + * @endcode + * + * @param[in] res RAFT resources used for the merge operation. + * @param[in] params Parameters that control the merging process. + * @param[in] indices A vector of pointers to the CAGRA indices to merge. All indices must: + * - Have attached datasets with the same dimension. + * + * @return A new CAGRA index containing the merged indices, graph, and dataset. + */ +auto merge(raft::resources const& res, + const cuvs::neighbors::cagra::merge_params& params, + std::vector*>& indices) + -> cuvs::neighbors::cagra::index; + +/** @brief Merge multiple CAGRA indices into a single index. + * + * This function merges multiple CAGRA indices into one, combining both the datasets and graph + * structures. + * + * @note: When device memory is sufficient, the dataset attached to the returned index is allocated + * in device memory by default; otherwise, host memory is used automatically. + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto dataset0 = raft::make_host_matrix(handle, size0, dim); + * auto dataset1 = raft::make_host_matrix(handle, size1, dim); + * + * auto index0 = cagra::build(res, index_params, dataset0); + * auto index1 = cagra::build(res, index_params, dataset1); + * + * std::vector*> indices{&index0, &index1}; + * cagra::merge_params params{index_params}; + * + * auto merged_index = cagra::merge(res, params, indices); + * @endcode + * + * @param[in] res RAFT resources used for the merge operation. + * @param[in] params Parameters that control the merging process. + * @param[in] indices A vector of pointers to the CAGRA indices to merge. All indices must: + * - Have attached datasets with the same dimension. + * + * @return A new CAGRA index containing the merged indices, graph, and dataset. + */ +auto merge(raft::resources const& res, + const cuvs::neighbors::cagra::merge_params& params, + std::vector*>& indices) + -> cuvs::neighbors::cagra::index; + +/** @brief Merge multiple CAGRA indices into a single index. + * + * This function merges multiple CAGRA indices into one, combining both the datasets and graph + * structures. + * + * @note: When device memory is sufficient, the dataset attached to the returned index is allocated + * in device memory by default; otherwise, host memory is used automatically. + * + * Usage example: + * @code{.cpp} + * using namespace raft::neighbors; + * auto dataset0 = raft::make_host_matrix(handle, size0, dim); + * auto dataset1 = raft::make_host_matrix(handle, size1, dim); + * + * auto index0 = cagra::build(res, index_params, dataset0); + * auto index1 = cagra::build(res, index_params, dataset1); + * + * std::vector*> indices{&index0, &index1}; + * cagra::merge_params params{index_params}; + * + * auto merged_index = cagra::merge(res, params, indices); + * @endcode + * + * @param[in] res RAFT resources used for the merge operation. + * @param[in] params Parameters that control the merging process. + * @param[in] indices A vector of pointers to the CAGRA indices to merge. All indices must: + * - Have attached datasets with the same dimension. + * + * @return A new CAGRA index containing the merged indices, graph, and dataset. + */ +auto merge(raft::resources const& res, + const cuvs::neighbors::cagra::merge_params& params, + std::vector*>& indices) + -> cuvs::neighbors::cagra::index; +/** + * @} + */ + +/// \defgroup mg_cpp_index_build ANN MG index build + +/// \ingroup mg_cpp_index_build +/** + * @brief Builds a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index_params configure the index building + * @param[in] index_dataset a row-major matrix on host [n_rows, dim] + * + * @return the constructed CAGRA MG index + */ +auto build(const raft::resources& clique, + const cuvs::neighbors::mg_index_params& index_params, + raft::host_matrix_view index_dataset) + -> cuvs::neighbors::mg_index, float, uint32_t>; + +/// \ingroup mg_cpp_index_build +/** + * @brief Builds a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index_params configure the index building + * @param[in] index_dataset a row-major matrix on host [n_rows, dim] + * + * @return the constructed CAGRA MG index + */ +auto build(const raft::resources& clique, + const cuvs::neighbors::mg_index_params& index_params, + raft::host_matrix_view index_dataset) + -> cuvs::neighbors::mg_index, half, uint32_t>; + +/// \ingroup mg_cpp_index_build +/** + * @brief Builds a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index_params configure the index building + * @param[in] index_dataset a row-major matrix on host [n_rows, dim] + * + * @return the constructed CAGRA MG index + */ +auto build(const raft::resources& clique, + const cuvs::neighbors::mg_index_params& index_params, + raft::host_matrix_view index_dataset) + -> cuvs::neighbors::mg_index, int8_t, uint32_t>; + +/// \ingroup mg_cpp_index_build +/** + * @brief Builds a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index_params configure the index building + * @param[in] index_dataset a row-major matrix on host [n_rows, dim] + * + * @return the constructed CAGRA MG index + */ +auto build(const raft::resources& clique, + const cuvs::neighbors::mg_index_params& index_params, + raft::host_matrix_view index_dataset) + -> cuvs::neighbors::mg_index, uint8_t, uint32_t>; + +/// \defgroup mg_cpp_index_extend ANN MG index extend + +/// \ingroup mg_cpp_index_extend +/** + * @brief Extends a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::cagra::extend(clique, index, new_vectors, std::nullopt); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] new_vectors a row-major matrix on host [n_rows, dim] + * @param[in] new_indices optional vector on host [n_rows], + * `std::nullopt` means default continuous range `[0...n_rows)` + * + */ +void extend(const raft::resources& clique, + cuvs::neighbors::mg_index, float, uint32_t>& index, + raft::host_matrix_view new_vectors, + std::optional> new_indices); + +/// \ingroup mg_cpp_index_extend +/** + * @brief Extends a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::cagra::extend(clique, index, new_vectors, std::nullopt); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] new_vectors a row-major matrix on host [n_rows, dim] + * @param[in] new_indices optional vector on host [n_rows], + * `std::nullopt` means default continuous range `[0...n_rows)` + * + */ +void extend(const raft::resources& clique, + cuvs::neighbors::mg_index, half, uint32_t>& index, + raft::host_matrix_view new_vectors, + std::optional> new_indices); + +/// \ingroup mg_cpp_index_extend +/** + * @brief Extends a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::cagra::extend(clique, index, new_vectors, std::nullopt); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] new_vectors a row-major matrix on host [n_rows, dim] + * @param[in] new_indices optional vector on host [n_rows], + * `std::nullopt` means default continuous range `[0...n_rows)` + * + */ +void extend(const raft::resources& clique, + cuvs::neighbors::mg_index, int8_t, uint32_t>& index, + raft::host_matrix_view new_vectors, + std::optional> new_indices); + +/// \ingroup mg_cpp_index_extend +/** + * @brief Extends a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::cagra::extend(clique, index, new_vectors, std::nullopt); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] new_vectors a row-major matrix on host [n_rows, dim] + * @param[in] new_indices optional vector on host [n_rows], + * `std::nullopt` means default continuous range `[0...n_rows)` + * + */ +void extend(const raft::resources& clique, + cuvs::neighbors::mg_index, uint8_t, uint32_t>& index, + raft::host_matrix_view new_vectors, + std::optional> new_indices); + +/// \defgroup mg_cpp_index_search ANN MG index search + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search(const raft::resources& clique, + const cuvs::neighbors::mg_index, float, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search(const raft::resources& clique, + const cuvs::neighbors::mg_index, half, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search( + const raft::resources& clique, + const cuvs::neighbors::mg_index, int8_t, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search( + const raft::resources& clique, + const cuvs::neighbors::mg_index, uint8_t, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search(const raft::resources& clique, + const cuvs::neighbors::mg_index, float, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search(const raft::resources& clique, + const cuvs::neighbors::mg_index, half, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search( + const raft::resources& clique, + const cuvs::neighbors::mg_index, int8_t, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \ingroup mg_cpp_index_search +/** + * @brief Searches a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * cuvs::neighbors::mg_search_params search_params; + * cuvs::neighbors::cagra::search(clique, index, search_params, queries, neighbors, + * distances); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] search_params configure the index search + * @param[in] queries a row-major matrix on host [n_rows, dim] + * @param[out] neighbors a row-major matrix on host [n_rows, n_neighbors] + * @param[out] distances a row-major matrix on host [n_rows, n_neighbors] + * + */ +void search( + const raft::resources& clique, + const cuvs::neighbors::mg_index, uint8_t, uint32_t>& index, + const cuvs::neighbors::mg_search_params& search_params, + raft::host_matrix_view queries, + raft::host_matrix_view neighbors, + raft::host_matrix_view distances); + +/// \defgroup mg_cpp_serialize ANN MG index serialization + +/// \ingroup mg_cpp_serialize +/** + * @brief Serializes a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "mg_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, index, filename); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] filename path to the file to be serialized + * + */ +void serialize( + const raft::resources& clique, + const cuvs::neighbors::mg_index, float, uint32_t>& index, + const std::string& filename); + +/// \ingroup mg_cpp_serialize +/** + * @brief Serializes a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "mg_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, index, filename); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] filename path to the file to be serialized + * + */ +void serialize(const raft::resources& clique, + const cuvs::neighbors::mg_index, half, uint32_t>& index, + const std::string& filename); + +/// \ingroup mg_cpp_serialize +/** + * @brief Serializes a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "mg_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, index, filename); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] filename path to the file to be serialized + * + */ +void serialize( + const raft::resources& clique, + const cuvs::neighbors::mg_index, int8_t, uint32_t>& index, + const std::string& filename); + +/// \ingroup mg_cpp_serialize +/** + * @brief Serializes a multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "mg_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, index, filename); + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] index the pre-built index + * @param[in] filename path to the file to be serialized + * + */ +void serialize( + const raft::resources& clique, + const cuvs::neighbors::mg_index, uint8_t, uint32_t>& index, + const std::string& filename); + +/// \defgroup mg_cpp_deserialize ANN MG index deserialization + +/// \ingroup mg_cpp_deserialize +/** + * @brief Deserializes a CAGRA multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::mg_index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "mg_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, index, filename); + * auto new_index = cuvs::neighbors::cagra::deserialize(clique, filename); + * + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] filename path to the file to be deserialized + * + */ +template +auto deserialize(const raft::resources& clique, const std::string& filename) + -> cuvs::neighbors::mg_index, T, IdxT>; + +/// \defgroup mg_cpp_distribute ANN MG local index distribution + +/// \ingroup mg_cpp_distribute +/** + * @brief Replicates a locally built and serialized CAGRA index to all GPUs to form a distributed + * multi-GPU index + * + * Usage example: + * @code{.cpp} + * raft::device_resources_snmg clique; + * cuvs::neighbors::cagra::index_params index_params; + * auto index = cuvs::neighbors::cagra::build(clique, index_params, index_dataset); + * const std::string filename = "local_index.cuvs"; + * cuvs::neighbors::cagra::serialize(clique, filename, index); + * auto new_index = cuvs::neighbors::cagra::distribute(clique, filename); + * + * @endcode + * + * @param[in] clique a `raft::resources` object specifying the NCCL clique configuration + * @param[in] filename path to the file to be deserialized : a local index + * + */ +template +auto distribute(const raft::resources& clique, const std::string& filename) + -> cuvs::neighbors::mg_index, T, IdxT>; + +} // namespace cuvs::neighbors::cagra + +#include +#include diff --git a/cpp/include/cuvs/neighbors/cagra_c.h b/cpp/include/cuvs/neighbors/cagra_c.h deleted file mode 100644 index 59861b5029..0000000000 --- a/cpp/include/cuvs/neighbors/cagra_c.h +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Copyright (c) 2024, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include -#include -#include - -/** - * @defgroup cagra_c C API for CUDA ANN Graph-based nearest neighbor search - * @{ - */ - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * @brief Enum to denote which ANN algorithm is used to build CAGRA graph - * - */ -enum cagraGraphBuildAlgo { - /* Use IVF-PQ to build all-neighbors knn graph */ - IVF_PQ, - /* Experimental, use NN-Descent to build all-neighbors knn graph */ - NN_DESCENT -}; - -/** - * @brief Supplemental parameters to build CAGRA Index - * - */ -struct cagraIndexParams { - /** Degree of input graph for pruning. */ - size_t intermediate_graph_degree; - /** Degree of output graph. */ - size_t graph_degree; - /** ANN algorithm to build knn graph. */ - enum cagraGraphBuildAlgo build_algo; - /** Number of Iterations to run if building with NN_DESCENT */ - size_t nn_descent_niter; -}; - -typedef struct cagraIndexParams* cuvsCagraIndexParams_t; - -/** - * @brief Allocate CAGRA Index params, and populate with default values - * - * @param[in] params cuvsCagraIndexParams_t to allocate - * @return cuvsError_t - */ -cuvsError_t cuvsCagraIndexParamsCreate(cuvsCagraIndexParams_t* params); - -/** - * @brief De-allocate CAGRA Index params - * - * @param[in] params - * @return cuvsError_t - */ -cuvsError_t cuvsCagraIndexParamsDestroy(cuvsCagraIndexParams_t index); - -/** - * @brief Enum to denote algorithm used to search CAGRA Index - * - */ -enum cagraSearchAlgo { - /** For large batch sizes. */ - SINGLE_CTA, - /** For small batch sizes. */ - MULTI_CTA, - MULTI_KERNEL, - AUTO -}; - -/** - * @brief Enum to denote Hash Mode used while searching CAGRA index - * - */ -enum cagraHashMode { HASH, SMALL, AUTO_HASH }; - -/** - * @brief Supplemental parameters to search CAGRA index - * - */ -struct cagraSearchParams { - /** Maximum number of queries to search at the same time (batch size). Auto select when 0.*/ - size_t max_queries; - - /** Number of intermediate search results retained during the search. - * - * This is the main knob to adjust trade off between accuracy and search speed. - * Higher values improve the search accuracy. - */ - size_t itopk_size; - - /** Upper limit of search iterations. Auto select when 0.*/ - size_t max_iterations; - - // In the following we list additional search parameters for fine tuning. - // Reasonable default values are automatically chosen. - - /** Which search implementation to use. */ - enum cagraSearchAlgo algo; - - /** Number of threads used to calculate a single distance. 4, 8, 16, or 32. */ - size_t team_size; - - /** Number of graph nodes to select as the starting point for the search in each iteration. aka - * search width?*/ - size_t search_width; - /** Lower limit of search iterations. */ - size_t min_iterations; - - /** Thread block size. 0, 64, 128, 256, 512, 1024. Auto selection when 0. */ - size_t thread_block_size; - /** Hashmap type. Auto selection when AUTO. */ - enum cagraHashMode hashmap_mode; - /** Lower limit of hashmap bit length. More than 8. */ - size_t hashmap_min_bitlen; - /** Upper limit of hashmap fill rate. More than 0.1, less than 0.9.*/ - float hashmap_max_fill_rate; - - /** Number of iterations of initial random seed node selection. 1 or more. */ - uint32_t num_random_samplings; - /** Bit mask used for initial random seed node selection. */ - uint64_t rand_xor_mask; -}; - -typedef struct cagraSearchParams* cuvsCagraSearchParams_t; - -/** - * @brief Allocate CAGRA search params, and populate with default values - * - * @param[in] params cuvsCagraSearchParams_t to allocate - * @return cuvsError_t - */ -cuvsError_t cuvsCagraSearchParamsCreate(cuvsCagraSearchParams_t* params); - -/** - * @brief De-allocate CAGRA search params - * - * @param[in] params - * @return cuvsError_t - */ -cuvsError_t cuvsCagraSearchParamsDestroy(cuvsCagraSearchParams_t params); - -/** - * @brief Struct to hold address of cuvs::neighbors::cagra::index and its active trained dtype - * - */ -typedef struct { - uintptr_t addr; - DLDataType dtype; - -} cagraIndex; - -typedef cagraIndex* cagraIndex_t; - -/** - * @brief Allocate CAGRA index - * - * @param[in] index cagraIndex_t to allocate - * @return cagraError_t - */ -cuvsError_t cagraIndexCreate(cagraIndex_t* index); - -/** - * @brief De-allocate CAGRA index - * - * @param[in] index cagraIndex_t to de-allocate - */ -cuvsError_t cagraIndexDestroy(cagraIndex_t index); - -/** - * @brief Build a CAGRA index with a `DLManagedTensor` which has underlying - * `DLDeviceType` equal to `kDLCUDA`, `kDLCUDAHost`, `kDLCUDAManaged`, - * or `kDLCPU`. Also, acceptable underlying types are: - * 1. `kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32` - * 2. `kDLDataType.code == kDLInt` and `kDLDataType.bits = 8` - * 3. `kDLDataType.code == kDLUInt` and `kDLDataType.bits = 8` - * - * @code {.c} - * #include - * #include - * - * // Create cuvsResources_t - * cuvsResources_t res; - * cuvsError_t res_create_status = cuvsResourcesCreate(&res); - * - * // Assume a populated `DLManagedTensor` type here - * DLManagedTensor dataset; - * - * // Create default index params - * cuvsCagraIndexParams_t params; - * cuvsError_t params_create_status = cuvsCagraIndexParamsCreate(¶ms); - * - * // Create CAGRA index - * cagraIndex_t index; - * cuvsError_t index_create_status = cagraIndexCreate(&index); - * - * // Build the CAGRA Index - * cuvsError_t build_status = cagraBuild(res, params, &dataset, index); - * - * // de-allocate `params`, `index` and `res` - * cuvsError_t params_destroy_status = cuvsCagraIndexParamsDestroy(params); - * cuvsError_t index_destroy_status = cagraIndexDestroy(index); - * cuvsError_t res_destroy_status = cuvsResourcesDestroy(res); - * @endcode - * - * @param[in] res cuvsResources_t opaque C handle - * @param[in] params cuvsCagraIndexParams_t used to build CAGRA index - * @param[in] dataset DLManagedTensor* training dataset - * @param[out] index cagraIndex_t Newly built CAGRA index - * @return cuvsError_t - */ -cuvsError_t cagraBuild(cuvsResources_t res, - cuvsCagraIndexParams_t params, - DLManagedTensor* dataset, - cagraIndex_t index); - -/** - * @brief Search a CAGRA index with a `DLManagedTensor` which has underlying - * `DLDeviceType` equal to `kDLCUDA`, `kDLCUDAHost`, `kDLCUDAManaged`. - * It is also important to note that the CAGRA Index must have been built - * with the same type of `queries`, such that `index.dtype.code == - * queries.dl_tensor.dtype.code` Types for input are: - * 1. `queries`: kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32` - * 2. `neighbors`: `kDLDataType.code == kDLUInt` and `kDLDataType.bits = 32` - * 3. `distances`: `kDLDataType.code == kDLFloat` and `kDLDataType.bits = 32` - * - * @code {.c} - * #include - * #include - * - * // Create cuvsResources_t - * cuvsResources_t res; - * cuvsError_t res_create_status = cuvsResourcesCreate(&res); - * - * // Assume a populated `DLManagedTensor` type here - * DLManagedTensor dataset; - * DLManagedTensor queries; - * DLManagedTensor neighbors; - * - * // Create default search params - * cuvsCagraSearchParams_t params; - * cuvsError_t params_create_status = cuvsCagraSearchParamsCreate(¶ms); - * - * // Search the `index` built using `cagraBuild` - * cuvsError_t search_status = cagraSearch(res, params, index, queries, neighbors, distances); - * - * // de-allocate `params` and `res` - * cuvsError_t params_destroy_status = cuvsCagraSearchParamsDestroy(params); - * cuvsError_t res_destroy_status = cuvsResourcesDestroy(res); - * @endcode - * - * @param[in] res cuvsResources_t opaque C handle - * @param[in] params cuvsCagraSearchParams_t used to search CAGRA index - * @param[in] index cagraIndex which has been returned by `cagraBuild` - * @param[in] queries DLManagedTensor* queries dataset to search - * @param[out] neighbors DLManagedTensor* output `k` neighbors for queries - * @param[out] distances DLManagedTensor* output `k` distances for queries - */ -cuvsError_t cagraSearch(cuvsResources_t res, - cuvsCagraSearchParams_t params, - cagraIndex_t index, - DLManagedTensor* queries, - DLManagedTensor* neighbors, - DLManagedTensor* distances); - -#ifdef __cplusplus -} -#endif - -/** - * @} - */ diff --git a/cpp/include/cuvs/neighbors/cagra_index_wrapper.hpp b/cpp/include/cuvs/neighbors/cagra_index_wrapper.hpp new file mode 100644 index 0000000000..c7223d12cf --- /dev/null +++ b/cpp/include/cuvs/neighbors/cagra_index_wrapper.hpp @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +// Forward declarations to avoid circular dependencies +namespace cuvs::neighbors::cagra { +template +struct index; +struct merge_params; +} // namespace cuvs::neighbors::cagra + +namespace cuvs::neighbors::cagra { + +/** + * @brief Wrapper for CAGRA index implementing IndexWrapper. + * + * This class wraps a CAGRA index and provides compatibility with the IndexBase interface. + * It serves as a bridge to help the CAGRA index implementation transition from its + * original design to the new object-oriented polymorphic design based on IndexBase. + * + * The wrapper enables: + * - CAGRA index to work seamlessly with the new polymorphic IndexBase interface + * - Gradual migration from the original CAGRA API to the unified index architecture + * - Compatibility with composite index patterns and other polymorphic usage scenarios + * - Preservation of existing CAGRA functionality while adopting the new design patterns + * + * This allows existing CAGRA users to benefit from the new architecture without + * requiring immediate changes to their existing code, while new users can adopt + * the unified interface from the start. + */ +template +class IndexWrapper : public cuvs::neighbors::IndexWrapper { + public: + using base_type = cuvs::neighbors::IndexWrapper; + using value_type = typename base_type::value_type; + using index_type = typename base_type::index_type; + using out_index_type = typename base_type::out_index_type; + using matrix_index_type = typename base_type::matrix_index_type; + + explicit IndexWrapper(cuvs::neighbors::cagra::index* idx); + + void search( + const raft::resources& handle, + const cuvs::neighbors::search_params& params, + raft::device_matrix_view queries, + raft::device_matrix_view neighbors, + raft::device_matrix_view distances, + const cuvs::neighbors::filtering::base_filter& filter = + cuvs::neighbors::filtering::none_sample_filter{}) const override; + + index_type size() const noexcept override; + + cuvs::distance::DistanceType metric() const noexcept override; + + /** + * @brief Merge this CAGRA index with other CAGRA indices. + * + * This method provides merge capability for CAGRA indices. It supports both + * physical merge (calling native CAGRA merge) and logical merge (creating + * CompositeIndex with wrapped indices). + * + * @param[in] handle RAFT resources for executing operations + * @param[in] params Merge parameters containing strategy and CAGRA-specific settings + * @param[in] other_indices Vector of other indices to merge with this one + * @return Shared pointer to merged index + */ + std::shared_ptr> merge( + const raft::resources& handle, + const cuvs::neighbors::merge_params& params, + const std::vector< + std::shared_ptr>>& + other_indices) const override; + + protected: + const cuvs::neighbors::search_params& convert_search_params( + const cuvs::neighbors::search_params& params) const override + { + // For CAGRA, we expect the params to be cagra::search_params + // This is handled in the search method via static_cast + return params; + } + + private: + cuvs::neighbors::cagra::index* index_; +}; + +/** + * @brief Factory function for creating a wrapped CAGRA index. + * + * This function creates a shared pointer to an IndexWrapper that wraps a CAGRA index, + * enabling it to work with the polymorphic IndexBase interface and composite operations. + * + * @tparam T Data type + * @tparam IdxT Index type + * @tparam OutputIdxT Output index type + * @param index Pointer to the CAGRA index + * @return Shared pointer to the wrapped index + * + * @par Example usage: + * @code{.cpp} + * // Create multiple CAGRA indices + * auto cagra_index1 = cuvs::neighbors::cagra::build(res, params, dataset1); + * auto cagra_index2 = cuvs::neighbors::cagra::build(res, params, dataset2); + * + * // Wrap them for polymorphic usage + * auto wrapped_index1 = cuvs::neighbors::cagra::make_index_wrapper(&cagra_index1); + * auto wrapped_index2 = cuvs::neighbors::cagra::make_index_wrapper(&cagra_index2); + * + * // Merge indices using the composite merge function + * std::vector>> indices; + * indices.push_back(wrapped_index1); + * indices.push_back(wrapped_index2); + * + * cuvs::neighbors::cagra::merge_params merge_params; + * auto merged_index = cuvs::neighbors::composite::merge(res, merge_params, indices); + * @endcode + */ +template +inline auto make_index_wrapper(cuvs::neighbors::cagra::index* index) + -> std::shared_ptr> +{ + return std::make_shared>(index); +} + +} // namespace cuvs::neighbors::cagra diff --git a/cpp/include/cuvs/neighbors/cagra_optimize.hpp b/cpp/include/cuvs/neighbors/cagra_optimize.hpp new file mode 100644 index 0000000000..ef0ae90061 --- /dev/null +++ b/cpp/include/cuvs/neighbors/cagra_optimize.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace cuvs::neighbors::cagra::helpers { + +/** + * @brief Optimize a KNN graph into a CAGRA graph. + * + * This function optimizes a k-NN graph to create a CAGRA graph. + * The input/output graphs must be on host memory. + * + * Usage example: + * @code{.cpp} + * raft::resources res; + * auto h_knn = raft::make_host_matrix(N, K_in); + * // Fill h_knn with KNN graph + * auto h_out = raft::make_host_matrix(N, K_out); + * cuvs::neighbors::cagra::helpers::optimize(res, h_knn.view(), h_out.view()); + * @endcode + * + * @param[in] handle RAFT resources + * @param[in] knn_graph Input KNN graph on host [n_rows, k_in] + * @param[out] new_graph Output CAGRA graph on host [n_rows, k_out] + */ +void optimize(raft::resources const& handle, + raft::host_matrix_view knn_graph, + raft::host_matrix_view new_graph); + +} // namespace cuvs::neighbors::cagra::helpers diff --git a/cpp/include/cuvs/neighbors/common.hpp b/cpp/include/cuvs/neighbors/common.hpp new file mode 100644 index 0000000000..4ed79dd695 --- /dev/null +++ b/cpp/include/cuvs/neighbors/common.hpp @@ -0,0 +1,901 @@ +/* + * Copyright (c) 2024-2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include // get_device_for_address +#include // rounding up + +#include +#include +#include + +#include +#include +#include + +#ifdef __cpp_lib_bitops +#include +#endif + +namespace cuvs::neighbors { + +/** Parameters for VPQ compression. */ +struct vpq_params { + /** + * The bit length of the vector element after compression by PQ. + * + * Possible values: [4, 5, 6, 7, 8]. + * + * Hint: the smaller the 'pq_bits', the smaller the index size and the better the search + * performance, but the lower the recall. + */ + uint32_t pq_bits = 8; + /** + * The dimensionality of the vector after compression by PQ. + * When zero, an optimal value is selected using a heuristic. + * + * TODO: at the moment `dim` must be a multiple `pq_dim`. + */ + uint32_t pq_dim = 0; + /** + * Vector Quantization (VQ) codebook size - number of "coarse cluster centers". + * When zero, an optimal value is selected using a heuristic. + */ + uint32_t vq_n_centers = 0; + /** The number of iterations searching for kmeans centers (both VQ & PQ phases). */ + uint32_t kmeans_n_iters = 25; + /** + * The fraction of data to use during iterative kmeans building (VQ phase). + * When zero, an optimal value is selected using a heuristic. + */ + double vq_kmeans_trainset_fraction = 0; + /** + * The fraction of data to use during iterative kmeans building (PQ phase). + * When zero, an optimal value is selected using a heuristic. + */ + double pq_kmeans_trainset_fraction = 0; +}; + +/** + * @defgroup neighbors_index Approximate Nearest Neighbors Types + * @{ + */ + +/** The base for approximate KNN index structures. */ +struct index {}; + +/** The base for KNN index parameters. */ +struct index_params { + /** Distance type. */ + cuvs::distance::DistanceType metric = cuvs::distance::DistanceType::L2Expanded; + /** The argument used by some distance metrics. */ + float metric_arg = 2.0f; +}; + +struct search_params {}; + +/** + * @brief Strategy for merging indices. + * + * This enum is declared separately to avoid namespace pollution when including common.hpp. + * It provides a generic merge strategy that can be used across different index types. + */ +enum class MergeStrategy { + /** Merge indices physically by combining their data structures */ + MERGE_STRATEGY_PHYSICAL = 0, + /** Merge indices logically by creating a composite wrapper */ + MERGE_STRATEGY_LOGICAL = 1 +}; + +/** Base merge parameters with polymorphic interface. */ +struct merge_params { + virtual ~merge_params() = default; + + virtual MergeStrategy strategy() const = 0; +}; + +/** @} */ // end group neighbors_index + +/** Two-dimensional dataset; maybe owning, maybe compressed, maybe strided. */ +template +struct dataset { + using index_type = IdxT; + /** Size of the dataset. */ + [[nodiscard]] virtual auto n_rows() const noexcept -> index_type = 0; + /** Dimensionality of the dataset. */ + [[nodiscard]] virtual auto dim() const noexcept -> uint32_t = 0; + /** Whether the object owns the data. */ + [[nodiscard]] virtual auto is_owning() const noexcept -> bool = 0; + virtual ~dataset() noexcept = default; +}; + +template +struct empty_dataset : public dataset { + using index_type = IdxT; + uint32_t suggested_dim; + explicit empty_dataset(uint32_t dim) noexcept : suggested_dim(dim) {} + [[nodiscard]] auto n_rows() const noexcept -> index_type final { return 0; } + [[nodiscard]] auto dim() const noexcept -> uint32_t final { return suggested_dim; } + [[nodiscard]] auto is_owning() const noexcept -> bool final { return true; } +}; + +template +struct strided_dataset : public dataset { + using index_type = IdxT; + using value_type = DataT; + using view_type = raft::device_matrix_view; + [[nodiscard]] auto n_rows() const noexcept -> index_type final { return view().extent(0); } + [[nodiscard]] auto dim() const noexcept -> uint32_t final + { + return static_cast(view().extent(1)); + } + /** Leading dimension of the dataset. */ + [[nodiscard]] constexpr auto stride() const noexcept -> uint32_t + { + auto v = view(); + return static_cast(v.stride(0) > 0 ? v.stride(0) : v.extent(1)); + } + /** Get the view of the data. */ + [[nodiscard]] virtual auto view() const noexcept -> view_type = 0; +}; + +template +struct non_owning_dataset : public strided_dataset { + using index_type = IdxT; + using value_type = DataT; + using typename strided_dataset::view_type; + view_type data; + explicit non_owning_dataset(view_type v) noexcept : data(v) {} + [[nodiscard]] auto is_owning() const noexcept -> bool final { return false; } + [[nodiscard]] auto view() const noexcept -> view_type final { return data; }; +}; + +template +struct owning_dataset : public strided_dataset { + using index_type = IdxT; + using value_type = DataT; + using typename strided_dataset::view_type; + using storage_type = + raft::mdarray, LayoutPolicy, ContainerPolicy>; + using mapping_type = typename view_type::mapping_type; + storage_type data; + mapping_type view_mapping; + owning_dataset(storage_type&& store, mapping_type view_mapping) noexcept + : data{std::move(store)}, view_mapping{view_mapping} + { + } + + [[nodiscard]] auto is_owning() const noexcept -> bool final { return true; } + [[nodiscard]] auto view() const noexcept -> view_type final + { + return view_type{data.data_handle(), view_mapping}; + }; +}; + +template +struct is_strided_dataset : std::false_type {}; + +template +struct is_strided_dataset> : std::true_type {}; + +template +struct is_strided_dataset> : std::true_type {}; + +template +struct is_strided_dataset> + : std::true_type {}; + +template +inline constexpr bool is_strided_dataset_v = is_strided_dataset::value; + +/** + * @brief Contstruct a strided matrix from any mdarray or mdspan. + * + * This function constructs a non-owning view if the input satisfied two conditions: + * + * 1) The data is accessible from the current device + * 2) The memory layout is the same as expected (row-major matrix with the required stride) + * + * Otherwise, this function constructs an owning device matrix and copies the data. + * When the data is copied, padding elements are filled with zeroes. + * + * @tparam SrcT the source mdarray or mdspan + * + * @param[in] res raft resources handle + * @param[in] src the source mdarray or mdspan + * @param[in] required_stride the leading dimension (in elements) + * @return maybe owning current-device-accessible strided matrix + */ +template +auto make_strided_dataset(const raft::resources& res, const SrcT& src, uint32_t required_stride) + -> std::unique_ptr> +{ + using extents_type = typename SrcT::extents_type; + using value_type = typename SrcT::value_type; + using index_type = typename SrcT::index_type; + using layout_type = typename SrcT::layout_type; + static_assert(extents_type::rank() == 2, "The input must be a matrix."); + static_assert(std::is_same_v || + std::is_same_v> || + std::is_same_v, + "The input must be row-major"); + RAFT_EXPECTS(src.extent(1) <= required_stride, + "The input row length must be not larger than the desired stride."); + cudaPointerAttributes ptr_attrs; + RAFT_CUDA_TRY(cudaPointerGetAttributes(&ptr_attrs, src.data_handle())); + auto* device_ptr = reinterpret_cast(ptr_attrs.devicePointer); + const uint32_t src_stride = src.stride(0) > 0 ? src.stride(0) : src.extent(1); + const bool device_accessible = device_ptr != nullptr; + const bool row_major = src.stride(1) <= 1; + const bool stride_matches = required_stride == src_stride; + + if (device_accessible && row_major && stride_matches) { + // Everything matches: make a non-owning dataset + return std::make_unique>( + raft::make_device_strided_matrix_view( + device_ptr, src.extent(0), src.extent(1), required_stride)); + } + // Something is wrong: have to make a copy and produce an owning dataset + auto out_layout = + raft::make_strided_layout(src.extents(), std::array{required_stride, 1}); + auto out_array = + raft::make_device_matrix(res, src.extent(0), required_stride); + + using out_mdarray_type = decltype(out_array); + using out_layout_type = typename out_mdarray_type::layout_type; + using out_container_policy_type = typename out_mdarray_type::container_policy_type; + using out_owning_type = + owning_dataset; + + RAFT_CUDA_TRY(cudaMemsetAsync(out_array.data_handle(), + 0, + out_array.size() * sizeof(value_type), + raft::resource::get_cuda_stream(res))); + RAFT_CUDA_TRY(cudaMemcpy2DAsync(out_array.data_handle(), + sizeof(value_type) * required_stride, + src.data_handle(), + sizeof(value_type) * src_stride, + sizeof(value_type) * src.extent(1), + src.extent(0), + cudaMemcpyDefault, + raft::resource::get_cuda_stream(res))); + + return std::make_unique(std::move(out_array), out_layout); +} + +/** + * @brief Contstruct a strided matrix from any mdarray. + * + * This function constructs an owning device matrix and copies the data. + * When the data is copied, padding elements are filled with zeroes. + * + * @tparam DataT + * @tparam IdxT + * @tparam LayoutPolicy + * @tparam ContainerPolicy + * + * @param[in] res raft resources handle + * @param[in] src the source mdarray or mdspan + * @param[in] required_stride the leading dimension (in elements) + * @return owning current-device-accessible strided matrix + */ +template +auto make_strided_dataset( + const raft::resources& res, + raft::mdarray, LayoutPolicy, ContainerPolicy>&& src, + uint32_t required_stride) -> std::unique_ptr> +{ + using value_type = DataT; + using index_type = IdxT; + using layout_type = LayoutPolicy; + using container_policy_type = ContainerPolicy; + static_assert(std::is_same_v || + std::is_same_v> || + std::is_same_v, + "The input must be row-major"); + RAFT_EXPECTS(src.extent(1) <= required_stride, + "The input row length must be not larger than the desired stride."); + const uint32_t src_stride = src.stride(0) > 0 ? src.stride(0) : src.extent(1); + const bool stride_matches = required_stride == src_stride; + + auto out_layout = + raft::make_strided_layout(src.extents(), std::array{required_stride, 1}); + + using out_mdarray_type = raft::device_matrix; + using out_layout_type = typename out_mdarray_type::layout_type; + using out_container_policy_type = typename out_mdarray_type::container_policy_type; + using out_owning_type = + owning_dataset; + + if constexpr (std::is_same_v && + std::is_same_v) { + if (stride_matches) { + // Everything matches, we can own the mdarray + return std::make_unique(std::move(src), out_layout); + } + } + // Something is wrong: have to make a copy and produce an owning dataset + auto out_array = + raft::make_device_matrix(res, src.extent(0), required_stride); + + RAFT_CUDA_TRY(cudaMemsetAsync(out_array.data_handle(), + 0, + out_array.size() * sizeof(value_type), + raft::resource::get_cuda_stream(res))); + RAFT_CUDA_TRY(cudaMemcpy2DAsync(out_array.data_handle(), + sizeof(value_type) * required_stride, + src.data_handle(), + sizeof(value_type) * src_stride, + sizeof(value_type) * src.extent(1), + src.extent(0), + cudaMemcpyDefault, + raft::resource::get_cuda_stream(res))); + + return std::make_unique(std::move(out_array), out_layout); +} + +/** + * @brief Contstruct a strided matrix from any mdarray or mdspan. + * + * A variant `make_strided_dataset` that allows specifying the byte alignment instead of the + * explicit stride length. + * + * @tparam SrcT the source mdarray or mdspan + * + * @param[in] res raft resources handle + * @param[in] src the source mdarray or mdspan + * @param[in] align_bytes the required byte alignment for the dataset rows. + * @return maybe owning current-device-accessible strided matrix + */ +template +auto make_aligned_dataset(const raft::resources& res, SrcT src, uint32_t align_bytes = 16) + -> std::unique_ptr> +{ + using source_type = std::remove_cv_t>; + using value_type = typename source_type::value_type; + constexpr size_t kSize = sizeof(value_type); + uint32_t required_stride = + raft::round_up_safe(src.extent(1) * kSize, std::lcm(align_bytes, kSize)) / kSize; + return make_strided_dataset(res, std::forward(src), required_stride); +} +/** + * @brief VPQ compressed dataset. + * + * The dataset is compressed using two level quantization + * + * 1. Vector Quantization + * 2. Product Quantization of residuals + * + * @tparam MathT the type of elements in the codebooks + * @tparam IdxT type of the vector indices (represent dataset.extent(0)) + * + */ +template +struct vpq_dataset : public dataset { + using index_type = IdxT; + using math_type = MathT; + /** Vector Quantization codebook - "coarse cluster centers". */ + raft::device_matrix vq_code_book; + /** Product Quantization codebook - "fine cluster centers". */ + raft::device_matrix pq_code_book; + /** Compressed dataset. */ + raft::device_matrix data; + + vpq_dataset(raft::device_matrix&& vq_code_book, + raft::device_matrix&& pq_code_book, + raft::device_matrix&& data) + : vq_code_book{std::move(vq_code_book)}, + pq_code_book{std::move(pq_code_book)}, + data{std::move(data)} + { + } + + [[nodiscard]] auto n_rows() const noexcept -> index_type final { return data.extent(0); } + [[nodiscard]] auto dim() const noexcept -> uint32_t final { return vq_code_book.extent(1); } + [[nodiscard]] auto is_owning() const noexcept -> bool final { return true; } + + /** Row length of the encoded data in bytes. */ + [[nodiscard]] constexpr inline auto encoded_row_length() const noexcept -> uint32_t + { + return data.extent(1); + } + /** The number of "coarse cluster centers" */ + [[nodiscard]] constexpr inline auto vq_n_centers() const noexcept -> uint32_t + { + return vq_code_book.extent(0); + } + /** The bit length of an encoded vector element after compression by PQ. */ + [[nodiscard]] constexpr inline auto pq_bits() const noexcept -> uint32_t + { + /* + NOTE: pq_bits and the book size + + Normally, we'd store `pq_bits` as a part of the index. + However, we know there's an invariant `pq_n_centers = 1 << pq_bits`, i.e. the codebook size is + the same as the number of possible code values. Hence, we don't store the pq_bits and derive it + from the array dimensions instead. + */ + auto pq_width = pq_n_centers(); +#ifdef __cpp_lib_bitops + return std::countr_zero(pq_width); +#else + uint32_t pq_bits = 0; + while (pq_width > 1) { + pq_bits++; + pq_width >>= 1; + } + return pq_bits; +#endif + } + /** The dimensionality of an encoded vector after compression by PQ. */ + [[nodiscard]] constexpr inline auto pq_dim() const noexcept -> uint32_t + { + return raft::div_rounding_up_unsafe(dim(), pq_len()); + } + /** Dimensionality of a subspaces, i.e. the number of vector components mapped to a subspace */ + [[nodiscard]] constexpr inline auto pq_len() const noexcept -> uint32_t + { + return pq_code_book.extent(1); + } + /** The number of vectors in a PQ codebook (`1 << pq_bits`). */ + [[nodiscard]] constexpr inline auto pq_n_centers() const noexcept -> uint32_t + { + return pq_code_book.extent(0); + } +}; + +template +struct is_vpq_dataset : std::false_type {}; + +template +struct is_vpq_dataset> : std::true_type {}; + +template +inline constexpr bool is_vpq_dataset_v = is_vpq_dataset::value; + +namespace filtering { + +/** + * @defgroup neighbors_filtering Filtering for ANN Types + * @{ + */ + +enum class FilterType { None, Bitmap, Bitset }; + +struct base_filter { + ~base_filter() = default; + virtual FilterType get_filter_type() const = 0; +}; + +/* A filter that filters nothing. This is the default behavior. */ +struct none_sample_filter : public base_filter { + /** \cond */ + constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()( + // query index + const uint32_t query_ix, + // the current inverted list index + const uint32_t cluster_ix, + // the index of the current sample inside the current inverted list + const uint32_t sample_ix) const; + + constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()( + // query index + const uint32_t query_ix, + // the index of the current sample + const uint32_t sample_ix) const; + /** \endcond */ + FilterType get_filter_type() const override { return FilterType::None; } +}; + +/** + * @brief Filter used to convert the cluster index and sample index + * of an IVF search into a sample index. This can be used as an + * intermediate filter. + * + * @tparam index_t Indexing type + * @tparam filter_t + */ +template +struct ivf_to_sample_filter : public base_filter { + const index_t* const* inds_ptrs_; + const filter_t next_filter_; + + ivf_to_sample_filter(const index_t* const* inds_ptrs, const filter_t next_filter); + + /** \cond */ + /** If the original filter takes three arguments, then don't modify the arguments. + * If the original filter takes two arguments, then we are using `inds_ptr_` to obtain the sample + * index. + */ + inline _RAFT_HOST_DEVICE bool operator()( + // query index + const uint32_t query_ix, + // the current inverted list index + const uint32_t cluster_ix, + // the index of the current sample inside the current inverted list + const uint32_t sample_ix) const; + + FilterType get_filter_type() const override { return next_filter_.get_filter_type(); } + /** \endcond */ +}; + +/** + * @brief Filter an index with a bitmap + * + * @tparam bitmap_t Data type of the bitmap + * @tparam index_t Indexing type + */ +template +struct bitmap_filter : public base_filter { + using view_t = cuvs::core::bitmap_view; + + // View of the bitset to use as a filter + const view_t bitmap_view_; + + bitmap_filter(const view_t bitmap_for_filtering); + /** \cond */ + inline _RAFT_HOST_DEVICE bool operator()( + // query index + const uint32_t query_ix, + // the index of the current sample + const uint32_t sample_ix) const; + /** \endcond */ + + FilterType get_filter_type() const override { return FilterType::Bitmap; } + + view_t view() const { return bitmap_view_; } + + template + void to_csr(raft::resources const& handle, csr_matrix_t& csr); +}; + +/** + * @brief Filter an index with a bitset + * + * @tparam bitset_t Data type of the bitset + * @tparam index_t Indexing type + */ +template +struct bitset_filter : public base_filter { + using view_t = cuvs::core::bitset_view; + + // View of the bitset to use as a filter + const view_t bitset_view_; + + /** \cond */ + _RAFT_HOST_DEVICE bitset_filter(const view_t bitset_for_filtering); + constexpr __forceinline__ _RAFT_HOST_DEVICE bool operator()( + // query index + const uint32_t query_ix, + // the index of the current sample + const uint32_t sample_ix) const; + /** \endcond */ + + FilterType get_filter_type() const override { return FilterType::Bitset; } + + view_t view() const { return bitset_view_; } + + template + void to_csr(raft::resources const& handle, csr_matrix_t& csr); +}; + +/** @} */ // end group neighbors_filtering + +/** + * If the filtering depends on the index of a sample, then the following + * filter template can be used: + * + * template + * struct index_ivf_sample_filter { + * using index_type = IdxT; + * + * const index_type* const* inds_ptr = nullptr; + * + * index_ivf_sample_filter() {} + * index_ivf_sample_filter(const index_type* const* _inds_ptr) + * : inds_ptr{_inds_ptr} {} + * index_ivf_sample_filter(const index_ivf_sample_filter&) = default; + * index_ivf_sample_filter(index_ivf_sample_filter&&) = default; + * index_ivf_sample_filter& operator=(const index_ivf_sample_filter&) = default; + * index_ivf_sample_filter& operator=(index_ivf_sample_filter&&) = default; + * + * inline _RAFT_HOST_DEVICE bool operator()( + * const uint32_t query_ix, + * const uint32_t cluster_ix, + * const uint32_t sample_ix) const { + * index_type database_idx = inds_ptr[cluster_ix][sample_ix]; + * + * // return true or false, depending on the database_idx + * return true; + * } + * }; + * + * Initialize it as: + * using filter_type = index_ivf_sample_filter; + * filter_type filter(cuvs_ivfpq_index.inds_ptrs().data_handle()); + * + * Use it as: + * cuvs::neighbors::ivf_pq::search_with_filtering( + * ...regular parameters here..., + * filter + * ); + * + * Another example would be the following filter that greenlights samples according + * to a contiguous bit mask vector. + * + * template + * struct bitmask_ivf_sample_filter { + * using index_type = IdxT; + * + * const index_type* const* inds_ptr = nullptr; + * const uint64_t* const bit_mask_ptr = nullptr; + * const int64_t bit_mask_stride_64 = 0; + * + * bitmask_ivf_sample_filter() {} + * bitmask_ivf_sample_filter( + * const index_type* const* _inds_ptr, + * const uint64_t* const _bit_mask_ptr, + * const int64_t _bit_mask_stride_64) + * : inds_ptr{_inds_ptr}, + * bit_mask_ptr{_bit_mask_ptr}, + * bit_mask_stride_64{_bit_mask_stride_64} {} + * bitmask_ivf_sample_filter(const bitmask_ivf_sample_filter&) = default; + * bitmask_ivf_sample_filter(bitmask_ivf_sample_filter&&) = default; + * bitmask_ivf_sample_filter& operator=(const bitmask_ivf_sample_filter&) = default; + * bitmask_ivf_sample_filter& operator=(bitmask_ivf_sample_filter&&) = default; + * + * inline _RAFT_HOST_DEVICE bool operator()( + * const uint32_t query_ix, + * const uint32_t cluster_ix, + * const uint32_t sample_ix) const { + * const index_type database_idx = inds_ptr[cluster_ix][sample_ix]; + * const uint64_t bit_mask_element = + * bit_mask_ptr[query_ix * bit_mask_stride_64 + database_idx / 64]; + * const uint64_t masked_bool = + * bit_mask_element & (1ULL << (uint64_t)(database_idx % 64)); + * const bool is_bit_set = (masked_bool != 0); + * + * return is_bit_set; + * } + * }; + */ +} // namespace filtering + +namespace ivf { + +/** + * Default value filled in the `indices` array. + * One may encounter it trying to access a record within a list that is outside of the + * `size` bound or whenever the list is allocated but not filled-in yet. + */ +template +constexpr static IdxT kInvalidRecord = + (std::is_signed_v ? IdxT{0} : std::numeric_limits::max()) - 1; + +/** The data for a single IVF list. */ +template