From bd2289ba2591717fe2fdc9c910d96bc9ee52b08c Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Thu, 23 Oct 2025 18:22:29 -0700 Subject: [PATCH 01/32] null check --- cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h index 3ad01dc2b4..0e5562221f 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h +++ b/cpp/bench/ann/src/cuvs/cuvs_ann_bench_param_parser.h @@ -60,7 +60,7 @@ extern template class cuvs::bench::cuvs_cagra; template void parse_dynamic_batching_params(const nlohmann::json& conf, ParamT& param) { - if (!conf.value("dynamic_batching", false)) { return; } + if (conf.is_null() || !conf.value("dynamic_batching", false)) { return; } param.dynamic_batching = true; if (conf.contains("dynamic_batching_max_batch_size")) { param.dynamic_batching_max_batch_size = conf.at("dynamic_batching_max_batch_size"); From d8d92569aeb792ea7d29a19b014824d128f195a8 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Thu, 23 Oct 2025 18:45:54 -0700 Subject: [PATCH 02/32] all tests --- .../config/algos/cuvs_brute_force.yaml | 3 + .../config/algos/cuvs_cagra_diskann.yaml | 7 + .../cuvs_bench/config/algos/cuvs_vamana.yaml | 8 + .../cuvs_bench/cuvs_bench/tests/test_cli.py | 257 ++++++++++++++++++ 4 files changed, 275 insertions(+) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_brute_force.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_brute_force.yaml index df6855fb42..d86d13fa79 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_brute_force.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_brute_force.yaml @@ -3,3 +3,6 @@ groups: base: build: search: + test: + build: + search: diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_cagra_diskann.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_cagra_diskann.yaml index e086e8230b..50fe2acf6a 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_cagra_diskann.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_cagra_diskann.yaml @@ -9,3 +9,10 @@ groups: graph_build_algo: ["NN_DESCENT"] search: L_search: [10, 20, 30, 40, 50, 100, 200, 300] + test: + build: + graph_degree: [32] + intermediate_graph_degree: [32] + graph_build_algo: ["NN_DESCENT"] + search: + L_search: [10] diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml index b840962d92..d75c337047 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml @@ -8,3 +8,11 @@ groups: search: L_search: [10, 20, 30, 40, 50, 100, 200, 300] num_threads: [32] + test: + build: + graph_degree: [32] + visited_size: [128] + alpha: [1.2] + search: + L_search: [10] + num_threads: [32] diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 48f65d8492..c3e2958203 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -429,3 +429,260 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): assert ( file_path.stat().st_size > 0 ), f"Expected file {filename} is empty." + + +def test_cuvs_ivf_pq_algorithm(temp_datasets_dir: Path): + """ + Test the cuvs_ivf_pq algorithm build and search operations. + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_ivf_pq", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "latency", + "--force", + ] + result = runner.invoke(run_main, run_args) + assert ( + result.exit_code == 0 + ), f"Run command failed with output:\n{result.output}" + + # Verify build and search result files exist + build_file = temp_datasets_dir / "test-data/result/build/cuvs_ivf_pq,test.csv" + assert build_file.exists(), "cuvs_ivf_pq build file doesn't exist" + assert build_file.stat().st_size > 0, "cuvs_ivf_pq build file is empty" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_ivf_pq search file doesn't exist" + assert search_file.stat().st_size > 0, "cuvs_ivf_pq search file is empty" + + +def test_cuvs_brute_force_algorithm(temp_datasets_dir: Path): + """ + Test the cuvs_brute_force algorithm build and search operations. + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_brute_force", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "latency", + "--force", + ] + result = runner.invoke(run_main, run_args) + assert ( + result.exit_code == 0 + ), f"Run command failed with output:\n{result.output}" + + # Verify search result files exist (brute force may not have a build phase) + search_file = temp_datasets_dir / "test-data/result/search/cuvs_brute_force,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_brute_force search file doesn't exist" + assert search_file.stat().st_size > 0, "cuvs_brute_force search file is empty" + + +def test_cuvs_vamana_algorithm(temp_datasets_dir: Path): + """ + Test the cuvs_vamana algorithm build and search operations. + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_vamana", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "latency", + "--force", + ] + result = runner.invoke(run_main, run_args) + # Note: This may fail if vamana is not enabled in the build + if result.exit_code == 0: + # Verify build and search result files exist + build_file = temp_datasets_dir / "test-data/result/build/cuvs_vamana,test.csv" + assert build_file.exists(), "cuvs_vamana build file doesn't exist" + assert build_file.stat().st_size > 0, "cuvs_vamana build file is empty" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_vamana search file doesn't exist" + assert search_file.stat().st_size > 0, "cuvs_vamana search file is empty" + + +def test_cuvs_cagra_diskann_algorithm(temp_datasets_dir: Path): + """ + Test the cuvs_cagra_diskann algorithm build and search operations. + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_cagra_diskann", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "latency", + "--force", + ] + result = runner.invoke(run_main, run_args) + # Note: This may fail if diskann is not enabled in the build + if result.exit_code == 0: + # Verify build and search result files exist + build_file = temp_datasets_dir / "test-data/result/build/cuvs_cagra_diskann,test.csv" + assert build_file.exists(), "cuvs_cagra_diskann build file doesn't exist" + assert build_file.stat().st_size > 0, "cuvs_cagra_diskann build file is empty" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_cagra_diskann,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_cagra_diskann search file doesn't exist" + assert search_file.stat().st_size > 0, "cuvs_cagra_diskann search file is empty" + + +@pytest.mark.skipif( + not pytest.importorskip("nccl", reason="NCCL not available"), + reason="Multi-GPU tests require NCCL", +) +def test_multi_gpu_algorithms(temp_datasets_dir: Path): + """ + Test multi-GPU algorithms (cuvs_mg_*). + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + + # Test cuvs_mg_ivf_flat + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_mg_ivf_flat", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "latency", + "--force", + ] + result = runner.invoke(run_main, run_args) + if result.exit_code == 0: + # Verify files exist + build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_ivf_flat,test.csv" + assert build_file.exists(), "cuvs_mg_ivf_flat build file doesn't exist" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_ivf_flat,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_mg_ivf_flat search file doesn't exist" + + # Test cuvs_mg_ivf_pq + run_args[7] = "cuvs_mg_ivf_pq" + result = runner.invoke(run_main, run_args) + if result.exit_code == 0: + build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_ivf_pq,test.csv" + assert build_file.exists(), "cuvs_mg_ivf_pq build file doesn't exist" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_ivf_pq,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_mg_ivf_pq search file doesn't exist" + + # Test cuvs_mg_cagra + run_args[7] = "cuvs_mg_cagra" + result = runner.invoke(run_main, run_args) + if result.exit_code == 0: + build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_cagra,test.csv" + assert build_file.exists(), "cuvs_mg_cagra build file doesn't exist" + + search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_cagra,test,k10,bs100,raw.csv" + assert search_file.exists(), "cuvs_mg_cagra search file doesn't exist" + + +def test_multiple_algorithms_integration(temp_datasets_dir: Path): + """ + Integration test running multiple algorithms together. + """ + dataset_path_arg = str(temp_datasets_dir) + + from cuvs_bench.run.__main__ import main as run_main + + runner = CliRunner() + run_args = [ + "--dataset", + "test-data", + "--dataset-path", + dataset_path_arg, + "--algorithms", + "cuvs_ivf_flat,cuvs_ivf_pq,cuvs_brute_force,cuvs_cagra", + "--batch-size", + "100", + "-k", + "10", + "--groups", + "test", + "-m", + "throughput", # Test throughput mode as well + "--force", + ] + result = runner.invoke(run_main, run_args) + assert ( + result.exit_code == 0 + ), f"Multiple algorithms run failed with output:\n{result.output}" + + # Verify that results were created for all algorithms + algorithms = ["cuvs_ivf_flat", "cuvs_ivf_pq", "cuvs_brute_force", "cuvs_cagra"] + for algo in algorithms: + search_file = temp_datasets_dir / f"test-data/result/search/{algo},test,k10,bs100,throughput.csv" + assert search_file.exists(), f"{algo} throughput search file doesn't exist" + assert search_file.stat().st_size > 0, f"{algo} throughput search file is empty" From fa932cc149372fe82fb1cd12c1560dde10e21880 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 11:38:08 -0700 Subject: [PATCH 03/32] syntax error --- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index c3e2958203..32396b8e29 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -81,7 +81,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,", # noqa: E501 "--batch-size", "100", "-k", @@ -158,6 +158,20 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, + "test-data/result/build/cuvs_ivf_pq,test.csv": { + "header": common_build_header + + [ + "GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 1, + }, + # Note: cuvs_brute_force typically doesn't have a build phase # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -351,6 +365,82 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 2, }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + } for rel_path, expectations in expected_files.items(): From b9acca7d59bda8562684fd344d522e9c5c53591a Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 11:43:05 -0700 Subject: [PATCH 04/32] add cuvs_ivf_pq pytest --- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 48f65d8492..7def73cc19 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -81,7 +81,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,", # noqa: E501 "--batch-size", "100", "-k", @@ -158,6 +158,20 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, + "test-data/result/build/cuvs_ivf_pq,test.csv": { + "header": common_build_header + + [ + "GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 1, + }, + # Note: cuvs_brute_force typically doesn't have a build phase # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -351,6 +365,82 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 2, }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "GPU", + "end_to_end", + "internalDistanceDtype", + "k", + "n_queries", + "nprobe", + "refine_ratio", + "smemLutDtype", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "codebook_kind", + "niter", + "nlist", + "pq_bits", + "pq_dim", + "ratio", + ], + "rows": 3, + }, + } for rel_path, expectations in expected_files.items(): From 119b0babffe7a8a4d542e2ac5b5bc54dae4880a9 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 11:44:21 -0700 Subject: [PATCH 05/32] format --- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 7def73cc19..62208311f7 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -171,7 +171,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, - # Note: cuvs_brute_force typically doesn't have a build phase # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -440,7 +439,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 3, }, - } for rel_path, expectations in expected_files.items(): From 37462e45f5fc1a6b8838a72f2cb2af8fd56757dd Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 12:08:04 -0700 Subject: [PATCH 06/32] correct order in header --- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 62208311f7..a93dd39cba 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -81,7 +81,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq" # noqa: E501 "--batch-size", "100", "-k", From 4ae79f0cf77a972b83289df7e2d34d43be4d45c1 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 12:36:57 -0700 Subject: [PATCH 07/32] format and docs --- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index a93dd39cba..4bc598aee3 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -62,7 +62,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): python -m cuvs_bench.run --dataset test-data --dataset-path datasets/ \ --algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,\ - cuvs_cagra,ggnn,cuvs_cagra_hnswlib, \ + cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \ --batch-size 100 -k 10 --groups test -m latency --force It then verifies that the set of expected result files @@ -81,7 +81,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq" # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501 "--batch-size", "100", "-k", @@ -466,7 +466,7 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): python -m cuvs_bench.plot --dataset test-data --dataset-path datasets/ \ --algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq, \ - cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib \ + cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \ --batch-size 100 -k 10 --groups test -m latency and then verifies that the following files are produced in the @@ -490,7 +490,7 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): "--output-filepath", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501 "--batch-size", "100", "-k", From 0c93b2a441087738bb11c43eacf17a0727f1e59a Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 15:21:31 -0700 Subject: [PATCH 08/32] correct header and params --- .../cuvs_bench/config/algos/cuvs_ivf_pq.yaml | 14 +++++++------- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 19 ++++++------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml index 154ae4b149..dc75b66b59 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml @@ -42,12 +42,12 @@ groups: test: build: nlist: [1024] - pq_dim: [64] - pq_bits: [8] - ratio: [10] - niter: [25] + pq_dim: [16] + pq_bits: [6] + ratio: [1] + niter: [20] search: - nprobe: [1] + nprobe: [5] internalDistanceDtype: ["float"] - smemLutDtype: ["float", "fp8", "half"] - refine_ratio: [1] + smemLutDtype: ["half"] + refine_ratio: [1, 2] diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 4bc598aee3..d29a562253 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -162,7 +162,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "header": common_build_header + [ "GPU", - "codebook_kind", "niter", "nlist", "pq_bits", @@ -369,75 +368,69 @@ def test_run_command_creates_results(temp_datasets_dir: Path): + [ "GPU", "end_to_end", - "internalDistanceDtype", "k", "n_queries", "nprobe", "refine_ratio", - "smemLutDtype", "total_queries", + "search_label", "build time", "build threads", "build cpu_time", "build GPU", - "codebook_kind", "niter", "nlist", "pq_bits", "pq_dim", "ratio", ], - "rows": 3, + "rows": 2, }, "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,latency.csv": { "header": common_search_header + [ "GPU", "end_to_end", - "internalDistanceDtype", "k", "n_queries", "nprobe", "refine_ratio", - "smemLutDtype", "total_queries", + "search_label", "build time", "build threads", "build cpu_time", "build GPU", - "codebook_kind", "niter", "nlist", "pq_bits", "pq_dim", "ratio", ], - "rows": 3, + "rows": 2, }, "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,throughput.csv": { "header": common_search_header + [ "GPU", "end_to_end", - "internalDistanceDtype", "k", "n_queries", "nprobe", "refine_ratio", - "smemLutDtype", "total_queries", + "search_label", "build time", "build threads", "build cpu_time", "build GPU", - "codebook_kind", "niter", "nlist", "pq_bits", "pq_dim", "ratio", ], - "rows": 3, + "rows": 2, }, } From 47f879b7d47f85d5b443c7eb3615253a7afabac9 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Fri, 24 Oct 2025 15:28:54 -0700 Subject: [PATCH 09/32] sparams to get increasing pareto --- python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml index dc75b66b59..1f07080999 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_ivf_pq.yaml @@ -47,7 +47,7 @@ groups: ratio: [1] niter: [20] search: - nprobe: [5] + nprobe: [1, 5] internalDistanceDtype: ["float"] smemLutDtype: ["half"] - refine_ratio: [1, 2] + refine_ratio: [1] From 9c8f72ce1b0db2f826c0182db3d21286dcdf4b4f Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 27 Oct 2025 09:12:47 -0700 Subject: [PATCH 10/32] update pytests --- .../cuvs_bench/config/algos/cuvs_vamana.yaml | 1 - .../cuvs_bench/cuvs_bench/tests/test_cli.py | 257 ------------------ 2 files changed, 258 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml index d75c337047..cee6c9eaf8 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml @@ -15,4 +15,3 @@ groups: alpha: [1.2] search: L_search: [10] - num_threads: [32] diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 32396b8e29..7def73cc19 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -519,260 +519,3 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): assert ( file_path.stat().st_size > 0 ), f"Expected file {filename} is empty." - - -def test_cuvs_ivf_pq_algorithm(temp_datasets_dir: Path): - """ - Test the cuvs_ivf_pq algorithm build and search operations. - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_ivf_pq", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "latency", - "--force", - ] - result = runner.invoke(run_main, run_args) - assert ( - result.exit_code == 0 - ), f"Run command failed with output:\n{result.output}" - - # Verify build and search result files exist - build_file = temp_datasets_dir / "test-data/result/build/cuvs_ivf_pq,test.csv" - assert build_file.exists(), "cuvs_ivf_pq build file doesn't exist" - assert build_file.stat().st_size > 0, "cuvs_ivf_pq build file is empty" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_ivf_pq,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_ivf_pq search file doesn't exist" - assert search_file.stat().st_size > 0, "cuvs_ivf_pq search file is empty" - - -def test_cuvs_brute_force_algorithm(temp_datasets_dir: Path): - """ - Test the cuvs_brute_force algorithm build and search operations. - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_brute_force", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "latency", - "--force", - ] - result = runner.invoke(run_main, run_args) - assert ( - result.exit_code == 0 - ), f"Run command failed with output:\n{result.output}" - - # Verify search result files exist (brute force may not have a build phase) - search_file = temp_datasets_dir / "test-data/result/search/cuvs_brute_force,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_brute_force search file doesn't exist" - assert search_file.stat().st_size > 0, "cuvs_brute_force search file is empty" - - -def test_cuvs_vamana_algorithm(temp_datasets_dir: Path): - """ - Test the cuvs_vamana algorithm build and search operations. - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_vamana", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "latency", - "--force", - ] - result = runner.invoke(run_main, run_args) - # Note: This may fail if vamana is not enabled in the build - if result.exit_code == 0: - # Verify build and search result files exist - build_file = temp_datasets_dir / "test-data/result/build/cuvs_vamana,test.csv" - assert build_file.exists(), "cuvs_vamana build file doesn't exist" - assert build_file.stat().st_size > 0, "cuvs_vamana build file is empty" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_vamana search file doesn't exist" - assert search_file.stat().st_size > 0, "cuvs_vamana search file is empty" - - -def test_cuvs_cagra_diskann_algorithm(temp_datasets_dir: Path): - """ - Test the cuvs_cagra_diskann algorithm build and search operations. - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_cagra_diskann", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "latency", - "--force", - ] - result = runner.invoke(run_main, run_args) - # Note: This may fail if diskann is not enabled in the build - if result.exit_code == 0: - # Verify build and search result files exist - build_file = temp_datasets_dir / "test-data/result/build/cuvs_cagra_diskann,test.csv" - assert build_file.exists(), "cuvs_cagra_diskann build file doesn't exist" - assert build_file.stat().st_size > 0, "cuvs_cagra_diskann build file is empty" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_cagra_diskann,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_cagra_diskann search file doesn't exist" - assert search_file.stat().st_size > 0, "cuvs_cagra_diskann search file is empty" - - -@pytest.mark.skipif( - not pytest.importorskip("nccl", reason="NCCL not available"), - reason="Multi-GPU tests require NCCL", -) -def test_multi_gpu_algorithms(temp_datasets_dir: Path): - """ - Test multi-GPU algorithms (cuvs_mg_*). - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - - # Test cuvs_mg_ivf_flat - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_mg_ivf_flat", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "latency", - "--force", - ] - result = runner.invoke(run_main, run_args) - if result.exit_code == 0: - # Verify files exist - build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_ivf_flat,test.csv" - assert build_file.exists(), "cuvs_mg_ivf_flat build file doesn't exist" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_ivf_flat,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_mg_ivf_flat search file doesn't exist" - - # Test cuvs_mg_ivf_pq - run_args[7] = "cuvs_mg_ivf_pq" - result = runner.invoke(run_main, run_args) - if result.exit_code == 0: - build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_ivf_pq,test.csv" - assert build_file.exists(), "cuvs_mg_ivf_pq build file doesn't exist" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_ivf_pq,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_mg_ivf_pq search file doesn't exist" - - # Test cuvs_mg_cagra - run_args[7] = "cuvs_mg_cagra" - result = runner.invoke(run_main, run_args) - if result.exit_code == 0: - build_file = temp_datasets_dir / "test-data/result/build/cuvs_mg_cagra,test.csv" - assert build_file.exists(), "cuvs_mg_cagra build file doesn't exist" - - search_file = temp_datasets_dir / "test-data/result/search/cuvs_mg_cagra,test,k10,bs100,raw.csv" - assert search_file.exists(), "cuvs_mg_cagra search file doesn't exist" - - -def test_multiple_algorithms_integration(temp_datasets_dir: Path): - """ - Integration test running multiple algorithms together. - """ - dataset_path_arg = str(temp_datasets_dir) - - from cuvs_bench.run.__main__ import main as run_main - - runner = CliRunner() - run_args = [ - "--dataset", - "test-data", - "--dataset-path", - dataset_path_arg, - "--algorithms", - "cuvs_ivf_flat,cuvs_ivf_pq,cuvs_brute_force,cuvs_cagra", - "--batch-size", - "100", - "-k", - "10", - "--groups", - "test", - "-m", - "throughput", # Test throughput mode as well - "--force", - ] - result = runner.invoke(run_main, run_args) - assert ( - result.exit_code == 0 - ), f"Multiple algorithms run failed with output:\n{result.output}" - - # Verify that results were created for all algorithms - algorithms = ["cuvs_ivf_flat", "cuvs_ivf_pq", "cuvs_brute_force", "cuvs_cagra"] - for algo in algorithms: - search_file = temp_datasets_dir / f"test-data/result/search/{algo},test,k10,bs100,throughput.csv" - assert search_file.exists(), f"{algo} throughput search file doesn't exist" - assert search_file.stat().st_size > 0, f"{algo} throughput search file is empty" From 3a8909dda4bf7301ec6ae77d0a11bc7156bd3996 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 27 Oct 2025 09:28:36 -0700 Subject: [PATCH 11/32] add new algos to current set of tests --- .../config/algos/diskann_memory.yaml | 8 + .../cuvs_bench/cuvs_bench/tests/test_cli.py | 148 +++++++++++++++++- 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml index b62f057084..9cee8e5ab5 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml @@ -10,3 +10,11 @@ groups: num_threads: [32] search: L_search: [10, 20, 30, 40, 50, 100, 200, 300] + test: + build: + R: [32] + L_build: [128] + alpha: [1.2] + num_threads: [32] + search: + L_search: [10] diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index d29a562253..1d8e66c3f3 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -62,7 +62,8 @@ def test_run_command_creates_results(temp_datasets_dir: Path): python -m cuvs_bench.run --dataset test-data --dataset-path datasets/ \ --algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,\ - cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \ + cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,\ + diskann_memory \ --batch-size 100 -k 10 --groups test -m latency --force It then verifies that the set of expected result files @@ -81,7 +82,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501 "--batch-size", "100", "-k", @@ -170,6 +171,27 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, + "test-data/result/build/cuvs_vamana,test.csv": { + "header": common_build_header + + [ + "GPU", + "alpha", + "graph_degree", + "num_threads", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/build/diskann_memory,test.csv": { + "header": common_build_header + + [ + "L_build", + "R", + "alpha", + "num_threads", + ], + "rows": 1, + }, # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -432,6 +454,123 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 2, }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "num_threads", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "build num_threads", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "num_threads", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "build num_threads", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "num_threads", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "build num_threads", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build num_threads", + ], + "rows": 1, + }, } for rel_path, expectations in expected_files.items(): @@ -459,7 +598,8 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): python -m cuvs_bench.plot --dataset test-data --dataset-path datasets/ \ --algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq, \ - cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \ + cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,\ + cuvs_vamana,diskann_memory \ --batch-size 100 -k 10 --groups test -m latency and then verifies that the following files are produced in the @@ -483,7 +623,7 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): "--output-filepath", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501 + "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501 "--batch-size", "100", "-k", From 0e49c6b4b6fea4aa3a799c37408b5f1028996117 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 27 Oct 2025 10:44:14 -0700 Subject: [PATCH 12/32] correct headers --- cpp/bench/ann/src/diskann/diskann_wrapper.h | 2 +- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cpp/bench/ann/src/diskann/diskann_wrapper.h b/cpp/bench/ann/src/diskann/diskann_wrapper.h index 6e7e0dc1c7..2cefd8ca06 100644 --- a/cpp/bench/ann/src/diskann/diskann_wrapper.h +++ b/cpp/bench/ann/src/diskann/diskann_wrapper.h @@ -242,7 +242,7 @@ class diskann_ssd : public algo { template diskann_ssd::diskann_ssd(Metric metric, int dim, const build_param& param) : algo(metric, dim) { - // Currently set the indexing RAM budget and the search RAM budget to max value to avoid sharding + // Currently we set the indexing RAM budget and the search RAM budget to max value to avoid sharding uint32_t build_dram_budget = std::numeric_limits::max(); uint32_t search_dram_budget = std::numeric_limits::max(); index_build_params_str = diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 1d8e66c3f3..20bf036bdc 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -524,6 +524,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", + "num_threads", "total_queries", "build time", "build threads", @@ -531,7 +532,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "L_build", "R", "alpha", - "build num_threads", + "build_num_threads", ], "rows": 1, }, @@ -542,6 +543,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", + "num_threads", "total_queries", "build time", "build threads", @@ -549,7 +551,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "L_build", "R", "alpha", - "build num_threads", + "build_num_threads", ], "rows": 1, }, @@ -560,6 +562,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", + "num_threads", "total_queries", "build time", "build threads", @@ -567,7 +570,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "L_build", "R", "alpha", - "build num_threads", + "build_num_threads", ], "rows": 1, }, From 42d4c0ff0234c03047ec09cc1d807eddd752714e Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Thu, 29 Jan 2026 12:33:16 -0800 Subject: [PATCH 13/32] style --- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 26ea3bc6a5..98ba7240c4 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # From 513cb7e3eb5ce0e0120010ef557bf8223a749347 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Sat, 31 Jan 2026 17:23:08 -0800 Subject: [PATCH 14/32] fix data_export and tests --- python/cuvs_bench/cuvs_bench/run/data_export.py | 5 +++-- python/cuvs_bench/cuvs_bench/tests/test_cli.py | 10 ---------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/run/data_export.py b/python/cuvs_bench/cuvs_bench/run/data_export.py index 707677a083..b2d8c4aff0 100644 --- a/python/cuvs_bench/cuvs_bench/run/data_export.py +++ b/python/cuvs_bench/cuvs_bench/run/data_export.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # @@ -213,9 +213,10 @@ def convert_json_to_csv_search(dataset, dataset_path): write["build GPU"] = None for col_idx in range(start_idx, len(build_df.columns)): col_name = build_df.columns[col_idx] - write[col_name] = None if col_name == "num_threads": write["build_num_threads"] = None + else: + write[col_name] = None for s_index, search_row in write.iterrows(): for b_index, build_row in build_df.iterrows(): if search_row["index_name"] == build_row["index_name"]: diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 98ba7240c4..51d78941f7 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -166,7 +166,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "GPU", "alpha", "graph_degree", - "num_threads", "visited_size", ], "rows": 1, @@ -451,7 +450,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", @@ -459,7 +457,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "build GPU", "alpha", "graph_degree", - "build num_threads", "visited_size", ], "rows": 1, @@ -472,7 +469,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", @@ -480,7 +476,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "build GPU", "alpha", "graph_degree", - "build num_threads", "visited_size", ], "rows": 1, @@ -493,7 +488,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", @@ -501,7 +495,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "build GPU", "alpha", "graph_degree", - "build num_threads", "visited_size", ], "rows": 1, @@ -513,7 +506,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", @@ -532,7 +524,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", @@ -551,7 +542,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "end_to_end", "k", "n_queries", - "num_threads", "total_queries", "build time", "build threads", From 0c7cd2e5499ea0f4817871c4ec4b26059c2693fe Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 16 Mar 2026 10:25:04 -0700 Subject: [PATCH 15/32] cmake change --- conda/environments/bench_ann_cuda-129_arch-x86_64.yaml | 2 +- cpp/cmake/thirdparty/get_diskann.cmake | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml index c44e2e58fb..eab0fc87c0 100644 --- a/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml @@ -48,4 +48,4 @@ dependencies: - setuptools>=77.0.0 - sysroot_linux-64==2.28 - wheel -name: bench_ann_cuda-129_arch-x86_64 +# name: bench_ann_cuda-129_arch-x86_64 diff --git a/cpp/cmake/thirdparty/get_diskann.cmake b/cpp/cmake/thirdparty/get_diskann.cmake index c872f279a8..f96ec5f73d 100644 --- a/cpp/cmake/thirdparty/get_diskann.cmake +++ b/cpp/cmake/thirdparty/get_diskann.cmake @@ -21,9 +21,13 @@ function(find_and_configure_diskann) CPM_VAR cpm_args ) + # DiskANN is CPU-only, so ensure CUDA architectures don't affect its build + # Pass CMAKE_CUDA_ARCHITECTURES as empty via CPM_ARGS to prevent CUDA initialization issues rapids_cpm_find(diskann ${version} ${find_args} GLOBAL_TARGETS diskann CPM_ARGS ${cpm_args} + CMAKE_ARGS + "-DCMAKE_CUDA_ARCHITECTURES=" OPTIONS "PYBIND OFF" "UNIT_TEST OFF" From f89d67205cefe0c82a68bebfb516c4053517218e Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 16 Mar 2026 13:32:14 -0700 Subject: [PATCH 16/32] check arm cpu --- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 155 ++++++++++-------- 1 file changed, 89 insertions(+), 66 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 51d78941f7..fac9955f6e 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # +import platform from pathlib import Path import pandas as pd @@ -11,6 +12,11 @@ from cuvs_bench.get_dataset.__main__ import main +def is_arm_cpu(): + """Check if running on ARM Linux architecture.""" + return platform.machine() in ("aarch64", "arm64") + + @pytest.fixture(scope="session") def temp_datasets_dir(tmp_path_factory): return tmp_path_factory.mktemp("datasets") @@ -64,6 +70,12 @@ def test_run_command_creates_results(temp_datasets_dir: Path): from cuvs_bench.run.__main__ import main as run_main + # Skip diskann_memory on ARM Linux (not supported) + skip_diskann = is_arm_cpu() + algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana" + if not skip_diskann: + algorithms += ",diskann_memory" + runner = CliRunner() run_args = [ "--dataset", @@ -71,7 +83,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "--dataset-path", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501 + algorithms, "--batch-size", "100", "-k", @@ -106,6 +118,8 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ] # --- Verify that the expected result files exist and are not empty --- + skip_diskann = is_arm_cpu() + expected_files = { # Build files: "test-data/result/build/cuvs_ivf_flat,test.csv": { @@ -170,16 +184,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, - "test-data/result/build/diskann_memory,test.csv": { - "header": common_build_header - + [ - "L_build", - "R", - "alpha", - "num_threads", - ], - "rows": 1, - }, # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -499,61 +503,75 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, - "test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, - "test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, - "test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, } + + if not skip_diskann: + expected_files.update({ + "test-data/result/build/diskann_memory,test.csv": { + "header": common_build_header + + [ + "L_build", + "R", + "alpha", + "num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + }) for rel_path, expectations in expected_files.items(): file_path = temp_datasets_dir / rel_path @@ -596,6 +614,11 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): from cuvs_bench.plot.__main__ import main as plot_main + skip_diskann = is_arm_cpu() + algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana" + if not skip_diskann: + algorithms += ",diskann_memory" + runner = CliRunner() args = [ "--dataset", @@ -605,7 +628,7 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): "--output-filepath", dataset_path_arg, "--algorithms", - "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501 + algorithms, "--batch-size", "100", "-k", From 6ed5e06875dd5ca416eefd04297d3134e794ada0 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 16 Mar 2026 13:35:15 -0700 Subject: [PATCH 17/32] undo cmake changes and conda env --- conda/environments/bench_ann_cuda-129_arch-x86_64.yaml | 2 +- cpp/cmake/thirdparty/get_diskann.cmake | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml index eab0fc87c0..c44e2e58fb 100644 --- a/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml +++ b/conda/environments/bench_ann_cuda-129_arch-x86_64.yaml @@ -48,4 +48,4 @@ dependencies: - setuptools>=77.0.0 - sysroot_linux-64==2.28 - wheel -# name: bench_ann_cuda-129_arch-x86_64 +name: bench_ann_cuda-129_arch-x86_64 diff --git a/cpp/cmake/thirdparty/get_diskann.cmake b/cpp/cmake/thirdparty/get_diskann.cmake index f96ec5f73d..c872f279a8 100644 --- a/cpp/cmake/thirdparty/get_diskann.cmake +++ b/cpp/cmake/thirdparty/get_diskann.cmake @@ -21,13 +21,9 @@ function(find_and_configure_diskann) CPM_VAR cpm_args ) - # DiskANN is CPU-only, so ensure CUDA architectures don't affect its build - # Pass CMAKE_CUDA_ARCHITECTURES as empty via CPM_ARGS to prevent CUDA initialization issues rapids_cpm_find(diskann ${version} ${find_args} GLOBAL_TARGETS diskann CPM_ARGS ${cpm_args} - CMAKE_ARGS - "-DCMAKE_CUDA_ARCHITECTURES=" OPTIONS "PYBIND OFF" "UNIT_TEST OFF" From 350db1b0998c1e403ef90c2d5192d6a7f24874b3 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 16 Mar 2026 13:40:04 -0700 Subject: [PATCH 18/32] style check --- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 138 +++++++++--------- 1 file changed, 70 insertions(+), 68 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index fac9955f6e..591ec2912b 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -119,7 +119,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path): # --- Verify that the expected result files exist and are not empty --- skip_diskann = is_arm_cpu() - + expected_files = { # Build files: "test-data/result/build/cuvs_ivf_flat,test.csv": { @@ -504,74 +504,76 @@ def test_run_command_creates_results(temp_datasets_dir: Path): "rows": 1, }, } - + if not skip_diskann: - expected_files.update({ - "test-data/result/build/diskann_memory,test.csv": { - "header": common_build_header - + [ - "L_build", - "R", - "alpha", - "num_threads", - ], - "rows": 1, - }, - "test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, - "test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, - "test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": { - "header": common_search_header - + [ - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "L_build", - "R", - "alpha", - "build_num_threads", - ], - "rows": 1, - }, - }) + expected_files.update( + { + "test-data/result/build/diskann_memory,test.csv": { + "header": common_build_header + + [ + "L_build", + "R", + "alpha", + "num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + "test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "L_build", + "R", + "alpha", + "build_num_threads", + ], + "rows": 1, + }, + } + ) for rel_path, expectations in expected_files.items(): file_path = temp_datasets_dir / rel_path From 47586de79d05e19f0d10158063f4a17558de820f Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Mon, 16 Mar 2026 14:34:20 -0700 Subject: [PATCH 19/32] add vamana to the list --- .../cuvs_bench/cuvs_bench/tests/test_cli.py | 143 +++++++++--------- 1 file changed, 71 insertions(+), 72 deletions(-) diff --git a/python/cuvs_bench/cuvs_bench/tests/test_cli.py b/python/cuvs_bench/cuvs_bench/tests/test_cli.py index 591ec2912b..c0d8f07abc 100644 --- a/python/cuvs_bench/cuvs_bench/tests/test_cli.py +++ b/python/cuvs_bench/cuvs_bench/tests/test_cli.py @@ -70,11 +70,10 @@ def test_run_command_creates_results(temp_datasets_dir: Path): from cuvs_bench.run.__main__ import main as run_main - # Skip diskann_memory on ARM Linux (not supported) skip_diskann = is_arm_cpu() - algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana" + algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq" if not skip_diskann: - algorithms += ",diskann_memory" + algorithms += ",cuvs_vamana,diskann_memory" runner = CliRunner() run_args = [ @@ -174,16 +173,6 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 1, }, - "test-data/result/build/cuvs_vamana,test.csv": { - "header": common_build_header - + [ - "GPU", - "alpha", - "graph_degree", - "visited_size", - ], - "rows": 1, - }, # Search files: "test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": { "header": common_search_header @@ -446,68 +435,78 @@ def test_run_command_creates_results(temp_datasets_dir: Path): ], "rows": 2, }, - "test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv": { - "header": common_search_header - + [ - "GPU", - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "build GPU", - "alpha", - "graph_degree", - "visited_size", - ], - "rows": 1, - }, - "test-data/result/search/cuvs_vamana,test,k10,bs100,latency.csv": { - "header": common_search_header - + [ - "GPU", - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "build GPU", - "alpha", - "graph_degree", - "visited_size", - ], - "rows": 1, - }, - "test-data/result/search/cuvs_vamana,test,k10,bs100,throughput.csv": { - "header": common_search_header - + [ - "GPU", - "L_search", - "end_to_end", - "k", - "n_queries", - "total_queries", - "build time", - "build threads", - "build cpu_time", - "build GPU", - "alpha", - "graph_degree", - "visited_size", - ], - "rows": 1, - }, } if not skip_diskann: expected_files.update( { + "test-data/result/build/cuvs_vamana,test.csv": { + "header": common_build_header + + [ + "GPU", + "alpha", + "graph_degree", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,latency.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "visited_size", + ], + "rows": 1, + }, + "test-data/result/search/cuvs_vamana,test,k10,bs100,throughput.csv": { + "header": common_search_header + + [ + "GPU", + "L_search", + "end_to_end", + "k", + "n_queries", + "total_queries", + "build time", + "build threads", + "build cpu_time", + "build GPU", + "alpha", + "graph_degree", + "visited_size", + ], + "rows": 1, + }, "test-data/result/build/diskann_memory,test.csv": { "header": common_build_header + [ @@ -617,9 +616,9 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path): from cuvs_bench.plot.__main__ import main as plot_main skip_diskann = is_arm_cpu() - algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana" + algorithms = "faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq" if not skip_diskann: - algorithms += ",diskann_memory" + algorithms += ",cuvs_vamana,diskann_memory" runner = CliRunner() args = [ From e9f0da795e1cb711ff5fe4168e97e47a27b63b49 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 09:12:59 -0700 Subject: [PATCH 20/32] single thread build --- python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml index 9cee8e5ab5..7321d772df 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml @@ -15,6 +15,5 @@ groups: R: [32] L_build: [128] alpha: [1.2] - num_threads: [32] search: L_search: [10] From 6f8d28985346c14221fa6c524ff58babe47c4445 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 10:04:25 -0700 Subject: [PATCH 21/32] fix single thread config --- python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml index 7321d772df..1ec67cb044 100644 --- a/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml +++ b/python/cuvs_bench/cuvs_bench/config/algos/diskann_memory.yaml @@ -15,5 +15,6 @@ groups: R: [32] L_build: [128] alpha: [1.2] + num_threads: [1] search: L_search: [10] From 268ab86daa50dec327519248e94fddae5067fb96 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 11:09:53 -0700 Subject: [PATCH 22/32] sync stream after vamana build --- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 2 ++ cpp/bench/ann/src/diskann/diskann_benchmark.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index 7d7292bd50..6897416a3c 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace cuvs::bench { @@ -82,6 +83,7 @@ void cuvs_vamana::build(const T* dataset, size_t nrow) dataset_is_on_host ? cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_host) : cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_device))); + raft::resource::sync_stream(handle_); } template diff --git a/cpp/bench/ann/src/diskann/diskann_benchmark.cpp b/cpp/bench/ann/src/diskann/diskann_benchmark.cpp index 4129f3cdab..cc437389cb 100644 --- a/cpp/bench/ann/src/diskann/diskann_benchmark.cpp +++ b/cpp/bench/ann/src/diskann/diskann_benchmark.cpp @@ -26,7 +26,7 @@ void parse_build_param(const nlohmann::json& conf, { param.R = conf.at("R"); param.L_build = conf.at("L_build"); - if (conf.contains("alpha")) { param.num_threads = conf.at("alpha"); } + if (conf.contains("alpha")) { param.alpha = conf.at("alpha"); } if (conf.contains("num_threads")) { param.num_threads = conf.at("num_threads"); } } From 46ef644f8c03d5196a2ec70715905939c348e446 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 11:40:08 -0700 Subject: [PATCH 23/32] do not skip state --- cpp/bench/ann/src/common/benchmark.hpp | 2 +- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index 22859e9ab8..2d5fb1405e 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -169,7 +169,7 @@ void bench_build(::benchmark::State& state, } state.counters.insert({{"index_size", index_size}}); - if (state.skipped()) { return; } + // if (state.skipped()) { return; } make_sure_parent_dir_exists(index.file); algo->save(index.file); } diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index 6897416a3c..99bc1725ec 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once From f2c41daba372d8a02e7561ac7a5077f636b82a57 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 11:50:58 -0700 Subject: [PATCH 24/32] fix style --- cpp/bench/ann/src/common/benchmark.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index 2d5fb1405e..787ba6b72c 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once From 284984a8a251511040e0add404f593b68884cbd5 Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 15:07:05 -0700 Subject: [PATCH 25/32] add sync stream in dataset serialization --- cpp/bench/ann/src/common/benchmark.hpp | 2 +- cpp/src/neighbors/detail/vamana/vamana_serialize.cuh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index 787ba6b72c..4cfc2ec6b3 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -169,7 +169,7 @@ void bench_build(::benchmark::State& state, } state.counters.insert({{"index_size", index_size}}); - // if (state.skipped()) { return; } + if (state.skipped()) { return; } make_sure_parent_dir_exists(index.file); algo->save(index.file); } diff --git a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh index 887c9eb448..84155e059e 100644 --- a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh +++ b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh @@ -72,6 +72,7 @@ void serialize_dataset(raft::resources const& res, strided_dataset->n_rows() * strided_dataset->dim()), raft::make_device_vector_view(strided_dataset->view().data_handle(), strided_dataset->n_rows() * strided_dataset->dim())); + raft::resource::sync_stream(res); to_file(dataset_base_file, h_dataset); } else { RAFT_LOG_DEBUG("dynamic_cast to strided_dataset failed"); @@ -91,6 +92,7 @@ void serialize_dataset(raft::resources const& res, try { auto h_dataset = raft::make_host_matrix(dataset.extent(0), dataset.extent(1)); raft::copy(res, h_dataset.view(), dataset); + raft::resource::sync_stream(res); to_file(dataset_base_file, h_dataset); } catch (std::bad_alloc& e) { RAFT_LOG_INFO("Failed to serialize dataset"); From 5edb4e5ad577f731cd0107938d0dbc94dda69b9d Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 15:09:30 -0700 Subject: [PATCH 26/32] rm sync stream in vamana wrapper --- cpp/bench/ann/src/common/benchmark.hpp | 2 +- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index 4cfc2ec6b3..22859e9ab8 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index 99bc1725ec..7d7292bd50 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -13,7 +13,6 @@ #include #include #include -#include namespace cuvs::bench { @@ -83,7 +82,6 @@ void cuvs_vamana::build(const T* dataset, size_t nrow) dataset_is_on_host ? cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_host) : cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_device))); - raft::resource::sync_stream(handle_); } template From aa022a23942bf60cf8c8a14a1d7baa526b9ce6fa Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 15:42:23 -0700 Subject: [PATCH 27/32] one more fix to serialize-dataset --- .../detail/vamana/vamana_serialize.cuh | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh index 84155e059e..21a48ba942 100644 --- a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh +++ b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh @@ -65,13 +65,19 @@ void serialize_dataset(raft::resources const& res, const auto* strided_dataset = dynamic_cast*>(dataset); if (strided_dataset) { - auto h_dataset = - raft::make_host_matrix(strided_dataset->n_rows(), strided_dataset->dim()); - raft::copy(res, - raft::make_host_vector_view(h_dataset.data_handle(), - strided_dataset->n_rows() * strided_dataset->dim()), - raft::make_device_vector_view(strided_dataset->view().data_handle(), - strided_dataset->n_rows() * strided_dataset->dim())); + auto nrows = strided_dataset->n_rows(); + auto dim = strided_dataset->dim(); + auto stride = strided_dataset->stride(); + auto d_data = strided_dataset->view(); + auto h_dataset = raft::make_host_matrix(nrows, dim); + RAFT_CUDA_TRY(cudaMemcpy2DAsync(h_dataset.data_handle(), + sizeof(T) * dim, + d_data.data_handle(), + sizeof(T) * stride, + sizeof(T) * dim, + nrows, + cudaMemcpyDefault, + raft::resource::get_cuda_stream(res))); raft::resource::sync_stream(res); to_file(dataset_base_file, h_dataset); } else { From 23019c8478ebed5e67c0359f03671c1715dd899d Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Wed, 25 Mar 2026 15:45:42 -0700 Subject: [PATCH 28/32] style --- cpp/src/neighbors/detail/vamana/vamana_serialize.cuh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh index 21a48ba942..e39a9e7ae4 100644 --- a/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh +++ b/cpp/src/neighbors/detail/vamana/vamana_serialize.cuh @@ -65,10 +65,10 @@ void serialize_dataset(raft::resources const& res, const auto* strided_dataset = dynamic_cast*>(dataset); if (strided_dataset) { - auto nrows = strided_dataset->n_rows(); - auto dim = strided_dataset->dim(); - auto stride = strided_dataset->stride(); - auto d_data = strided_dataset->view(); + auto nrows = strided_dataset->n_rows(); + auto dim = strided_dataset->dim(); + auto stride = strided_dataset->stride(); + auto d_data = strided_dataset->view(); auto h_dataset = raft::make_host_matrix(nrows, dim); RAFT_CUDA_TRY(cudaMemcpy2DAsync(h_dataset.data_handle(), sizeof(T) * dim, From 7ebf54f2001979d8a80d3d7aad25a43d8ad7c0bf Mon Sep 17 00:00:00 2001 From: Tarang Jain Date: Sat, 28 Mar 2026 15:04:28 -0700 Subject: [PATCH 29/32] lower value for L_load --- cpp/bench/ann/src/diskann/diskann_wrapper.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/bench/ann/src/diskann/diskann_wrapper.h b/cpp/bench/ann/src/diskann/diskann_wrapper.h index b99c3c6354..bb17c9c354 100644 --- a/cpp/bench/ann/src/diskann/diskann_wrapper.h +++ b/cpp/bench/ann/src/diskann/diskann_wrapper.h @@ -165,7 +165,7 @@ void diskann_memory::load(const std::string& index_file) initialize_index_(0); int load_threads = (bench_mode_ == Mode::kThroughput) ? cuvs::bench::benchmark_n_threads : 1; - this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 2000); + this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 100); } template From f231647cf3d75508fd4ada15c67f3655194f8927 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 13 May 2026 16:39:31 -0700 Subject: [PATCH 30/32] debug statements --- cpp/bench/ann/src/common/benchmark.hpp | 6 +- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 91 ++++++++++++++++++- .../cuvs_bench/backends/cpp_gbench.py | 31 +++++++ 3 files changed, 126 insertions(+), 2 deletions(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index a588b1e2a6..c642d37f31 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -171,7 +171,11 @@ void bench_build(::benchmark::State& state, if (state.skipped()) { return; } make_sure_parent_dir_exists(index.file); - algo->save(index.file); + try { + algo->save(index.file); + } catch (const std::exception& e) { + state.SkipWithError("Failed to save an index: " + std::string(e.what())); + } } template diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index 7d7292bd50..b9273690c2 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -10,12 +10,30 @@ #include #include +#include +#include #include #include #include +#include +#include +#include namespace cuvs::bench { +inline bool cuvs_vamana_artifact_exists(const std::string& file) +{ + std::error_code ec; + return std::filesystem::exists(file, ec); +} + +inline std::uintmax_t cuvs_vamana_artifact_size(const std::string& file) +{ + std::error_code ec; + auto size = std::filesystem::file_size(file, ec); + return ec ? 0 : size; +} + template class cuvs_vamana : public algo, public algo_gpu { public: @@ -78,10 +96,31 @@ void cuvs_vamana::build(const T* dataset, size_t nrow) dataset, raft::make_extents(nrow, this->dim_)); bool dataset_is_on_host = raft::get_device_for_address(dataset) == -1; + log_info( + "cuvs_vamana build start: nrow=%zu dim=%d graph_degree=%u visited_size=%u alpha=%f " + "dataset_memory=%s", + nrow, + this->dim_, + static_cast(vamana_index_params_.graph_degree), + static_cast(vamana_index_params_.visited_size), + static_cast(vamana_index_params_.alpha), + dataset_is_on_host ? "host" : "device"); + vamana_index_ = std::make_shared>(std::move( dataset_is_on_host ? cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_host) : cuvs::neighbors::vamana::build(handle_, vamana_index_params_, dataset_view_device))); + + log_info("cuvs_vamana build finish: index_size=%zu index_dim=%u graph_degree=%u", + static_cast(vamana_index_->size()), + static_cast(vamana_index_->dim()), + static_cast(vamana_index_->graph_degree())); + + if (vamana_index_->size() == 0 || vamana_index_->dim() == 0) { + throw std::runtime_error("cuvs_vamana build produced an empty index: size=" + + std::to_string(static_cast(vamana_index_->size())) + + ", dim=" + std::to_string(vamana_index_->dim())); + } } template @@ -95,13 +134,63 @@ void cuvs_vamana::set_search_param(const search_param_base& param, template void cuvs_vamana::save(const std::string& file) const { + if (!vamana_index_) { throw std::runtime_error("cuvs_vamana save called before build"); } + + log_info("cuvs_vamana save start: file=%s index_size=%zu index_dim=%u graph_degree=%u", + file.c_str(), + static_cast(vamana_index_->size()), + static_cast(vamana_index_->dim()), + static_cast(vamana_index_->graph_degree())); + cuvs::neighbors::vamana::serialize(handle_, file, *vamana_index_); + + const auto data_file = file + ".data"; + const auto index_bytes = cuvs_vamana_artifact_size(file); + const auto data_bytes = cuvs_vamana_artifact_size(data_file); + const auto index_exists = cuvs_vamana_artifact_exists(file); + const auto data_exists = cuvs_vamana_artifact_exists(data_file); + + log_info( + "cuvs_vamana save artifacts: index_file=%s exists=%d bytes=%zu data_file=%s " + "exists=%d bytes=%zu", + file.c_str(), + index_exists ? 1 : 0, + static_cast(index_bytes), + data_file.c_str(), + data_exists ? 1 : 0, + static_cast(data_bytes)); + + if (!index_exists) { + throw std::runtime_error("cuvs_vamana serialize did not write index file: " + file); + } + if (!data_exists) { + throw std::runtime_error("cuvs_vamana serialize did not write dataset sidecar: " + data_file); + } + if (index_bytes == 0 || data_bytes == 0) { + throw std::runtime_error("cuvs_vamana serialize wrote empty artifact(s): index_file=" + file + + ", data_file=" + data_file); + } } template void cuvs_vamana::load(const std::string& file) { + const auto data_file = file + ".data"; + const auto index_bytes = cuvs_vamana_artifact_size(file); + const auto data_bytes = cuvs_vamana_artifact_size(data_file); + + log_info( + "cuvs_vamana load start: index_file=%s exists=%d bytes=%zu data_file=%s exists=%d " + "bytes=%zu", + file.c_str(), + cuvs_vamana_artifact_exists(file) ? 1 : 0, + static_cast(index_bytes), + data_file.c_str(), + cuvs_vamana_artifact_exists(data_file) ? 1 : 0, + static_cast(data_bytes)); + diskann_memory_search_->load(file); + log_info("cuvs_vamana load finish: file=%s", file.c_str()); } template diff --git a/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py b/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py index a888e957e5..f12708a062 100644 --- a/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py +++ b/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py @@ -67,6 +67,25 @@ def __init__(self, config: Dict[str, Any]): f"C++ benchmark executable not found: {self.executable_path}" ) + @staticmethod + def _gbench_error_message( + benchmarks: List[Dict[str, Any]], + phase: str, + output_path: Optional[Path] = None, + ) -> Optional[str]: + """Return a compact message if Google Benchmark reported row errors.""" + errors = [b for b in benchmarks if b.get("error_occurred")] + if not errors: + return None + + location = f" in {output_path}" if output_path else "" + details = [] + for bench in errors: + name = bench.get("name", "") + message = bench.get("error_message", "") + details.append(f"{name}: {message}") + return f"{phase} benchmark error(s){location}: " + "; ".join(details) + def build( self, dataset: Dataset, @@ -254,6 +273,12 @@ def build( "No benchmarks found in Google Benchmark output" ) + error_message = self._gbench_error_message( + benchmarks, "Build", benchmark_out + ) + if error_message: + raise RuntimeError(error_message) + # Return aggregated result total_build_time = sum(b.get("real_time", 0) for b in benchmarks) @@ -528,6 +553,12 @@ def search( "No benchmarks found in Google Benchmark output" ) + error_message = self._gbench_error_message( + benchmarks, "Search", output_json + ) + if error_message: + raise RuntimeError(error_message) + # Aggregate metrics across all benchmarks total_search_time = sum(b.get("real_time", 0) for b in benchmarks) avg_recall = ( From e239ea109f5526682ffd1c9bb825c89da20d2053 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Wed, 13 May 2026 17:55:46 -0700 Subject: [PATCH 31/32] more debug prints --- cpp/bench/ann/src/common/benchmark.hpp | 204 +++++++++++++++++- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 9 + cpp/bench/ann/src/diskann/diskann_wrapper.h | 122 ++++++++++- .../cuvs_bench/backends/cpp_gbench.py | 66 ++++++ .../cuvs_bench/cuvs_bench/run/data_export.py | 47 ++++ 5 files changed, 435 insertions(+), 13 deletions(-) diff --git a/cpp/bench/ann/src/common/benchmark.hpp b/cpp/bench/ann/src/common/benchmark.hpp index c642d37f31..5bf39ac79c 100644 --- a/cpp/bench/ann/src/common/benchmark.hpp +++ b/cpp/bench/ann/src/common/benchmark.hpp @@ -116,10 +116,25 @@ void bench_build(::benchmark::State& state, cuvs::bench::benchmark_thread_id = state.thread_index(); cuvs::bench::benchmark_n_threads = state.threads(); dump_parameters(state, index.build_param); + + if (state.thread_index() == 0) { + log_info( + "bench_build start: algo=%s index=%s file=%s force_overwrite=%d threads=%d dim=%d " + "base_set_size=%zu", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str(), + force_overwrite ? 1 : 0, + state.threads(), + dataset->dim(), + dataset->base_set_size()); + } + if (file_exists(index.file)) { if (force_overwrite) { - log_info("Overwriting file: %s", index.file.c_str()); + log_info("bench_build overwriting existing index file: %s", index.file.c_str()); } else { + log_info("bench_build skipped existing index file: %s", index.file.c_str()); return state.SkipWithMessage( "Index file already exists (use --force to overwrite the index)."); } @@ -127,15 +142,39 @@ void bench_build(::benchmark::State& state, std::unique_ptr> algo; try { + if (state.thread_index() == 0) { + log_info( + "bench_build create algo start: algo=%s index=%s", index.algo.c_str(), index.name.c_str()); + } algo = create_algo(index.algo, dataset->distance(), dataset->dim(), index.build_param); + if (state.thread_index() == 0) { + log_info( + "bench_build create algo finish: algo=%s index=%s", index.algo.c_str(), index.name.c_str()); + } } catch (const std::exception& e) { + log_warn("bench_build create algo failed: algo=%s index=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + e.what()); return state.SkipWithError("Failed to create an algo: " + std::string(e.what())); } const auto algo_property = parse_algo_property(algo->get_preference(), index.build_param); + if (state.thread_index() == 0) { + log_info("bench_build algo properties: algo=%s dataset_memory_type=%d query_memory_type=%d", + index.algo.c_str(), + static_cast(algo_property.dataset_memory_type), + static_cast(algo_property.query_memory_type)); + } const T* base_set = dataset->base_set(algo_property.dataset_memory_type); std::size_t index_size = dataset->base_set_size(); + if (state.thread_index() == 0) { + log_info("bench_build base_set ready: algo=%s pointer=%p index_size=%zu", + index.algo.c_str(), + static_cast(base_set), + index_size); + } cuda_timer gpu_timer{algo}; { @@ -155,11 +194,24 @@ void bench_build(::benchmark::State& state, */ [[maybe_unused]] auto gpu_all = gpu_timer.lap(no_lap_sync); for (auto _ : state) { + if (state.thread_index() == 0) { + log_info( + "bench_build iteration start: algo=%s index=%s", index.algo.c_str(), index.name.c_str()); + } [[maybe_unused]] auto ntx_lap = nvtx.lap(); [[maybe_unused]] auto gpu_lap = gpu_timer.lap(!no_lap_sync); try { algo->build(base_set, index_size); + if (state.thread_index() == 0) { + log_info("bench_build iteration finish: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); + } } catch (const std::exception& e) { + log_warn("bench_build iteration failed: algo=%s index=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + e.what()); state.SkipWithError(std::string(e.what())); } } @@ -172,8 +224,22 @@ void bench_build(::benchmark::State& state, if (state.skipped()) { return; } make_sure_parent_dir_exists(index.file); try { + log_info("bench_build save start: algo=%s index=%s file=%s", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str()); algo->save(index.file); + log_info("bench_build save finish: algo=%s index=%s file=%s exists=%d", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str(), + file_exists(index.file) ? 1 : 0); } catch (const std::exception& e) { + log_warn("bench_build save failed: algo=%s index=%s file=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str(), + e.what()); state.SkipWithError("Failed to save an index: " + std::string(e.what())); } } @@ -201,10 +267,26 @@ void bench_search(::benchmark::State& state, // Round down the query data to a multiple of the batch size to loop over full batches of data const std::size_t query_set_size = (dataset->query_set_size() / n_queries) * n_queries; + if (state.thread_index() == 0) { + log_info( + "bench_search start: algo=%s index=%s file=%s search_param_ix=%zu threads=%d " + "raw_queries=%zu query_set_size=%zu k=%u n_queries=%zu", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str(), + search_param_ix, + state.threads(), + dataset->query_set_size(), + query_set_size, + k, + n_queries); + } + if (dataset->query_set_size() < n_queries) { std::stringstream msg; msg << "Not enough queries in benchmark set. Expected " << n_queries << ", actual " << dataset->query_set_size(); + if (state.thread_index() == 0) { log_warn("bench_search skipped: %s", msg.str().c_str()); } state.SkipWithError(msg.str()); return; } @@ -219,9 +301,18 @@ void bench_search(::benchmark::State& state, const T* query_set = nullptr; if (!file_exists(index.file)) { + if (state.thread_index() == 0) { + log_warn("bench_search missing index file: algo=%s index=%s file=%s", + index.algo.c_str(), + index.name.c_str(), + index.file.c_str()); + } state.SkipWithError("Index file is missing. Run the benchmark in the build mode first."); return; } + if (state.thread_index() == 0) { + log_info("bench_search index file exists: file=%s", index.file.c_str()); + } /** * Make sure the first thread loads the algo and dataset @@ -231,6 +322,9 @@ void bench_search(::benchmark::State& state, // algo is static to cache it between close search runs to save time on index loading static std::string index_file = ""; if (index.file != index_file) { + log_info("bench_search cache reset: previous_file=%s new_file=%s", + index_file.c_str(), + index.file.c_str()); current_algo.reset(); index_file = index.file; } @@ -239,26 +333,63 @@ void bench_search(::benchmark::State& state, algo* a; try { if (!current_algo || (a = dynamic_cast*>(current_algo.get())) == nullptr) { + log_info("bench_search create algo start: algo=%s index=%s dim=%d", + index.algo.c_str(), + index.name.c_str(), + dataset->dim()); auto ualgo = create_algo(index.algo, dataset->distance(), dataset->dim(), index.build_param); a = ualgo.get(); + log_info( + "bench_search load algo start: algo=%s file=%s", index.algo.c_str(), index_file.c_str()); a->load(index_file); + log_info( + "bench_search load algo finish: algo=%s file=%s", index.algo.c_str(), index_file.c_str()); current_algo = std::move(ualgo); + } else { + log_info("bench_search using cached algo: algo=%s file=%s", + index.algo.c_str(), + index_file.c_str()); } + log_info("bench_search create search_param start: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); search_param = create_search_param(index.algo, sp_json); + log_info("bench_search create search_param finish: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); } catch (const std::exception& e) { + log_warn("bench_search load/create failed: algo=%s index=%s file=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + index_file.c_str(), + e.what()); state.SkipWithError("Failed to create an algo: " + std::string(e.what())); return; } current_algo_props = std::make_unique(std::move(parse_algo_property(a->get_preference(), sp_json))); + log_info("bench_search algo properties: algo=%s dataset_memory_type=%d query_memory_type=%d", + index.algo.c_str(), + static_cast(current_algo_props->dataset_memory_type), + static_cast(current_algo_props->query_memory_type)); if (search_param->needs_dataset()) { try { + log_info("bench_search set_search_dataset start: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); a->set_search_dataset(dataset->base_set(current_algo_props->dataset_memory_type), dataset->base_set_size()); + log_info("bench_search set_search_dataset finish: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); } catch (const std::exception& ex) { + log_warn("bench_search set_search_dataset failed: algo=%s index=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + ex.what()); state.SkipWithError("The algorithm '" + index.name + "' requires the base set, but it's not available. " + "Exception: " + std::string(ex.what())); @@ -266,14 +397,28 @@ void bench_search(::benchmark::State& state, } } try { + log_info("bench_search set_search_param start: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); a->set_search_param(*search_param, dataset->filter_bitset(current_algo_props->dataset_memory_type)); + log_info("bench_search set_search_param finish: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); } catch (const std::exception& ex) { + log_warn("bench_search set_search_param failed: algo=%s index=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + ex.what()); state.SkipWithError("An error occurred setting search parameters: " + std::string(ex.what())); return; } query_set = dataset->query_set(current_algo_props->query_memory_type); + log_info("bench_search query_set acquired: algo=%s pointer=%p query_memory_type=%d", + index.algo.c_str(), + static_cast(query_set), + static_cast(current_algo_props->query_memory_type)); load_barrier.arrive(state.threads()); } else { // All other threads will wait for the first thread to initialize the algo. @@ -297,6 +442,12 @@ void bench_search(::benchmark::State& state, auto* neighbors_ptr = reinterpret_cast(result_buf.data(current_algo_props->query_memory_type)); auto* distances_ptr = reinterpret_cast(neighbors_ptr + result_elem_count); + if (state.thread_index() == 0) { + log_info("bench_search result buffer ready: algo=%s result_elem_count=%zu memory_type=%d", + index.algo.c_str(), + result_elem_count, + static_cast(current_algo_props->query_memory_type)); + } { nvtx_stats nvtx_stats{state}; @@ -304,8 +455,21 @@ void bench_search(::benchmark::State& state, std::unique_ptr> a{nullptr}; try { + if (state.thread_index() == 0) { + log_info( + "bench_search algo copy start: algo=%s index=%s", index.algo.c_str(), index.name.c_str()); + } dynamic_cast*>(current_algo.get())->copy().swap(a); + if (state.thread_index() == 0) { + log_info("bench_search algo copy finish: algo=%s index=%s", + index.algo.c_str(), + index.name.c_str()); + } } catch (const std::exception& e) { + log_warn("bench_search algo copy failed: algo=%s index=%s error=%s", + index.algo.c_str(), + index.name.c_str(), + e.what()); state.SkipWithError("Algo::copy: " + std::string(e.what())); return; } @@ -325,6 +489,16 @@ void bench_search(::benchmark::State& state, neighbors_ptr + out_offset * k, distances_ptr + out_offset * k); } catch (const std::exception& e) { + log_warn( + "bench_search benchmark loop failed: algo=%s index=%s batch_offset=%td " + "out_offset=%td n_queries=%zu k=%u error=%s", + index.algo.c_str(), + index.name.c_str(), + batch_offset, + out_offset, + n_queries, + k, + e.what()); state.SkipWithError("Benchmark loop: " + std::string(e.what())); break; } @@ -351,11 +525,27 @@ void bench_search(::benchmark::State& state, // This will be the total number of queries across all threads state.counters.insert({{"total_queries", queries_processed}}); - if (state.skipped()) { return; } + if (state.skipped()) { + if (state.thread_index() == 0) { + log_warn("bench_search skipped before recall: algo=%s index=%s queries_processed=%zu", + index.algo.c_str(), + index.name.c_str(), + queries_processed); + } + return; + } // Each thread calculates recall on their partition of queries. // evaluate recall if (dataset->max_k() >= k && dataset->gt_maps().has_value()) { + if (state.thread_index() == 0) { + log_info("bench_search recall start: algo=%s index=%s max_k=%u k=%u queries_processed=%zu", + index.algo.c_str(), + index.name.c_str(), + dataset->max_k(), + k, + queries_processed); + } // gt_maps[i] is a hash map of {id, neighbor_rank} for query i const auto& gt_maps = dataset->gt_maps(); result_buf.transfer_data(MemoryType::kHost, current_algo_props->query_memory_type); @@ -414,6 +604,16 @@ void bench_search(::benchmark::State& state, batch_offset = (batch_offset + queries_stride) % query_set_size; } double actual_recall = static_cast(match_count) / static_cast(total_count); + if (state.thread_index() == 0) { + log_info( + "bench_search recall finish: algo=%s index=%s match_count=%zu total_count=%zu " + "recall=%f", + index.algo.c_str(), + index.name.c_str(), + match_count, + total_count, + actual_recall); + } /* NOTE: recall in the throughput mode & filtering When filtering is enabled, `total_count` may vary between individual threads, but we still take diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index b9273690c2..d62d0dc34a 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -197,6 +198,14 @@ template void cuvs_vamana::search( const T* queries, int batch_size, int k, algo_base::index_type* neighbors, float* distances) const { + static std::atomic search_logged{false}; + bool expected = false; + if (search_logged.compare_exchange_strong(expected, true)) { + log_info("cuvs_vamana search first call: batch_size=%d k=%d query_ptr=%p", + batch_size, + k, + static_cast(queries)); + } diskann_memory_search_->search(queries, batch_size, k, neighbors, distances); } diff --git a/cpp/bench/ann/src/diskann/diskann_wrapper.h b/cpp/bench/ann/src/diskann/diskann_wrapper.h index bb17c9c354..c8510bd761 100644 --- a/cpp/bench/ann/src/diskann/diskann_wrapper.h +++ b/cpp/bench/ann/src/diskann/diskann_wrapper.h @@ -16,14 +16,32 @@ #include #include +#include #include #include #include +#include #include +#include +#include +#include #include namespace cuvs::bench { +inline bool diskann_memory_artifact_exists(const std::string& file) +{ + std::error_code ec; + return std::filesystem::exists(file, ec); +} + +inline std::uintmax_t diskann_memory_artifact_size(const std::string& file) +{ + std::error_code ec; + auto size = std::filesystem::file_size(file, ec); + return ec ? 0 : size; +} + diskann::Metric parse_metric_to_diskann(cuvs::bench::Metric metric) { if (metric == cuvs::bench::Metric::kInnerProduct) { @@ -80,9 +98,13 @@ class diskann_memory : public algo { private: std::shared_ptr diskann_index_write_params_{nullptr}; uint32_t max_points_; - uint32_t build_pq_bytes_ = 0; - int num_threads_; - uint32_t L_search_; + uint32_t build_pq_bytes_ = 0; + uint32_t configured_build_pq_bytes_ = 0; + uint32_t graph_degree_ = 0; + uint32_t L_build_ = 0; + float alpha_ = 0.0f; + int num_threads_ = 0; + uint32_t L_search_ = 0; Mode bench_mode_; std::string index_path_prefix_; std::shared_ptr> mem_index_{nullptr}; @@ -94,6 +116,10 @@ diskann_memory::diskann_memory(Metric metric, int dim, const build_param& par : algo(metric, dim) { assert(this->dim_ > 0); + configured_build_pq_bytes_ = param.build_pq_bytes; + graph_degree_ = param.R; + L_build_ = param.L_build; + alpha_ = param.alpha; num_threads_ = param.num_threads; diskann_index_write_params_ = std::make_shared( diskann::IndexWriteParametersBuilder(param.L_build, param.R) @@ -124,8 +150,25 @@ void diskann_memory::initialize_index_(size_t max_points) template void diskann_memory::build(const T* dataset, size_t nrow) { + log_info( + "diskann_memory build start: nrow=%zu dim=%d graph_degree=%u L_build=%u alpha=%f " + "configured_build_pq_bytes=%u build_pq_bytes=%u threads=%d", + nrow, + this->dim_, + static_cast(graph_degree_), + static_cast(L_build_), + static_cast(alpha_), + static_cast(configured_build_pq_bytes_), + static_cast(build_pq_bytes_), + num_threads_); initialize_index_(nrow); - mem_index_->build(dataset, nrow, std::vector()); + try { + mem_index_->build(dataset, nrow, std::vector()); + } catch (const std::exception& e) { + log_warn("diskann_memory build failed: nrow=%zu dim=%d error=%s", nrow, this->dim_, e.what()); + throw; + } + log_info("diskann_memory build finish: nrow=%zu dim=%d", nrow, this->dim_); } template @@ -134,25 +177,66 @@ void diskann_memory::set_search_param(const search_param_base& param, const v if (filter_bitset != nullptr) { throw std::runtime_error("Filtering is not supported yet."); } auto sp = dynamic_cast(param); L_search_ = sp.L_search; + log_info("diskann_memory set_search_param: L_search=%u dim=%d", + static_cast(L_search_), + this->dim_); } template void diskann_memory::search( const T* queries, int batch_size, int k, algo_base::index_type* indices, float* distances) const { + static std::atomic search_logged{false}; + bool expected = false; + if (search_logged.compare_exchange_strong(expected, true)) { + log_info( + "diskann_memory search first call: prefix=%s batch_size=%d k=%d L_search=%u dim=%d " + "mode=%s benchmark_threads=%d", + index_path_prefix_.c_str(), + batch_size, + k, + static_cast(L_search_), + this->dim_, + bench_mode_ == Mode::kThroughput ? "throughput" : "latency", + cuvs::bench::benchmark_n_threads); + } + for (int i = 0; i < batch_size; i++) { - mem_index_->search(queries + i * this->dim_, - static_cast(k), - L_search_, - reinterpret_cast(indices + i * k), - distances + i * k); + try { + mem_index_->search(queries + i * this->dim_, + static_cast(k), + L_search_, + reinterpret_cast(indices + i * k), + distances + i * k); + } catch (const std::exception& e) { + log_warn( + "diskann_memory search failed: prefix=%s query_in_batch=%d batch_size=%d k=%d " + "L_search=%u error=%s", + index_path_prefix_.c_str(), + i, + batch_size, + k, + static_cast(L_search_), + e.what()); + throw; + } } } template void diskann_memory::save(const std::string& index_file) const { - this->mem_index_->save(index_file.c_str()); + log_info("diskann_memory save start: file=%s dim=%d", index_file.c_str(), this->dim_); + try { + this->mem_index_->save(index_file.c_str()); + } catch (const std::exception& e) { + log_warn("diskann_memory save failed: file=%s error=%s", index_file.c_str(), e.what()); + throw; + } + log_info("diskann_memory save finish: file=%s exists=%d bytes=%zu", + index_file.c_str(), + diskann_memory_artifact_exists(index_file) ? 1 : 0, + static_cast(diskann_memory_artifact_size(index_file))); } template @@ -165,7 +249,23 @@ void diskann_memory::load(const std::string& index_file) initialize_index_(0); int load_threads = (bench_mode_ == Mode::kThroughput) ? cuvs::bench::benchmark_n_threads : 1; - this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 100); + log_info( + "diskann_memory load start: prefix=%s dim=%d benchmark_threads=%d load_threads=%d " + "mode=%s build_threads=%d", + index_path_prefix_.c_str(), + this->dim_, + cuvs::bench::benchmark_n_threads, + load_threads, + bench_mode_ == Mode::kThroughput ? "throughput" : "latency", + num_threads_); + try { + this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 100); + } catch (const std::exception& e) { + log_warn( + "diskann_memory load failed: prefix=%s error=%s", index_path_prefix_.c_str(), e.what()); + throw; + } + log_info("diskann_memory load finish: prefix=%s", index_path_prefix_.c_str()); } template diff --git a/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py b/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py index f12708a062..e1e009be5c 100644 --- a/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py +++ b/python/cuvs_bench/cuvs_bench/backends/cpp_gbench.py @@ -86,6 +86,22 @@ def _gbench_error_message( details.append(f"{name}: {message}") return f"{phase} benchmark error(s){location}: " + "; ".join(details) + @staticmethod + def _index_debug_summary( + indexes: List[IndexConfig], + ) -> List[Dict[str, Any]]: + """Return the C++ benchmark index payload fields useful in CI logs.""" + return [ + { + "name": idx.name, + "algo": idx.algo, + "file": idx.file, + "build_param": idx.build_param, + "search_params": idx.search_params, + } + for idx in indexes + ] + def build( self, dataset: Dataset, @@ -249,6 +265,17 @@ def build( start_time = time.perf_counter() try: + print("[cuvs-bench] C++ build launch", flush=True) + print(f" executable: {self.executable_path}", flush=True) + print(f" temp_config: {temp_config_path}", flush=True) + print(f" benchmark_out: {benchmark_out}", flush=True) + print(f" data_prefix: {self.data_prefix}", flush=True) + print( + " indexes: " + + json.dumps(self._index_debug_summary(indexes), default=str), + flush=True, + ) + print(f" command: {' '.join(cmd)}", flush=True) subprocess.run( cmd, check=True, @@ -272,11 +299,17 @@ def build( raise ValueError( "No benchmarks found in Google Benchmark output" ) + print( + f"[cuvs-bench] C++ build parsed {len(benchmarks)} " + f"benchmark row(s) from {benchmark_out}", + flush=True, + ) error_message = self._gbench_error_message( benchmarks, "Build", benchmark_out ) if error_message: + print(f"[cuvs-bench] {error_message}", flush=True) raise RuntimeError(error_message) # Return aggregated result @@ -298,6 +331,11 @@ def build( ) except subprocess.CalledProcessError as e: + print( + f"[cuvs-bench] C++ build subprocess failed: " + f"returncode={e.returncode} stderr={e.stderr}", + flush=True, + ) return BuildResult( index_path=first_index.file, build_time_seconds=time.perf_counter() - start_time, @@ -309,6 +347,7 @@ def build( ) except Exception as e: + print(f"[cuvs-bench] C++ build exception: {e}", flush=True) return BuildResult( index_path=first_index.file, build_time_seconds=time.perf_counter() - start_time, @@ -515,6 +554,21 @@ def search( start_time = time.perf_counter() try: + print("[cuvs-bench] C++ search launch", flush=True) + print(f" executable: {self.executable_path}", flush=True) + print(f" temp_config: {temp_config_path}", flush=True) + print(f" temp_output_json: {temp_output_json}", flush=True) + print(f" output_json: {output_json}", flush=True) + print(f" data_prefix: {self.data_prefix}", flush=True) + print( + f" total_search_configs: {total_search_configs}", flush=True + ) + print( + " indexes: " + + json.dumps(self._index_debug_summary(indexes), default=str), + flush=True, + ) + print(f" command: {' '.join(cmd)}", flush=True) subprocess.run( cmd, check=True, @@ -552,11 +606,17 @@ def search( raise ValueError( "No benchmarks found in Google Benchmark output" ) + print( + f"[cuvs-bench] C++ search parsed {len(benchmarks)} " + f"benchmark row(s) from {output_json}", + flush=True, + ) error_message = self._gbench_error_message( benchmarks, "Search", output_json ) if error_message: + print(f"[cuvs-bench] {error_message}", flush=True) raise RuntimeError(error_message) # Aggregate metrics across all benchmarks @@ -599,6 +659,11 @@ def search( ) except subprocess.CalledProcessError as e: + print( + f"[cuvs-bench] C++ search subprocess failed: " + f"returncode={e.returncode} stderr={e.stderr}", + flush=True, + ) return SearchResult( neighbors=np.array([]), distances=np.array([]), @@ -612,6 +677,7 @@ def search( ) except Exception as e: + print(f"[cuvs-bench] C++ search exception: {e}", flush=True) return SearchResult( neighbors=np.array([]), distances=np.array([]), diff --git a/python/cuvs_bench/cuvs_bench/run/data_export.py b/python/cuvs_bench/cuvs_bench/run/data_export.py index b2d8c4aff0..b869bd57bf 100644 --- a/python/cuvs_bench/cuvs_bench/run/data_export.py +++ b/python/cuvs_bench/cuvs_bench/run/data_export.py @@ -70,6 +70,12 @@ def read_json_files(dataset, dataset_path, method): DataFrame of JSON content. """ dir_path = os.path.join(dataset_path, dataset, "result", method) + if not os.path.isdir(dir_path): + print( + f"[cuvs-bench] Export skipped: result directory does not exist: " + f"{dir_path}" + ) + return for file in os.listdir(dir_path): if file.endswith(".json"): file_path = os.path.join(dir_path, file) @@ -103,6 +109,36 @@ def clean_algo_name(algo_name): return name.removesuffix(".json") +def log_missing_export_columns(file, df, required_columns, method): + """ + Print enough context to diagnose why a JSON result cannot become CSV. + """ + missing = [name for name in required_columns if name not in df.columns] + if not missing: + return False + + print( + f"[cuvs-bench] Cannot export {method} CSV for {file}: " + f"missing column(s) {missing}; available columns: {list(df.columns)}" + ) + + if "error_occurred" in df.columns: + error_rows = df[ + df["error_occurred"].map( + lambda value: value is True + or str(value).lower() in {"true", "1"} + ) + ] + for _, row in error_rows.iterrows(): + print( + "[cuvs-bench] Google Benchmark error row: " + f"name={row.get('name', '')} " + f"message={row.get('error_message', '')}" + ) + + return True + + def write_csv(file, algo_name, df, extra_columns=None, skip_cols=None): """ Write a DataFrame to CSV with specified columns skipped. @@ -120,6 +156,9 @@ def write_csv(file, algo_name, df, extra_columns=None, skip_cols=None): skip_cols : set, optional Set of columns to skip when writing to CSV (default is None). """ + if log_missing_export_columns(file, df, ["name", "real_time"], "build"): + return + df["name"] = df["name"].str.split("/").str[0] write_data = pd.DataFrame( { @@ -183,6 +222,14 @@ def convert_json_to_csv_search(dataset, dataset_path): f"{','.join(algo_name)}.csv", ) algo_name = clean_algo_name(algo_name) + if log_missing_export_columns( + file, + df, + ["name", "Recall", "items_per_second", "Latency"], + "search", + ): + continue + df["name"] = df["name"].str.split("/").str[0] write = pd.DataFrame( { From a17986ea2d35ba9d5c572630b73a0f918fb82a97 Mon Sep 17 00:00:00 2001 From: tarang-jain Date: Thu, 14 May 2026 11:15:48 -0700 Subject: [PATCH 32/32] more debug prints --- cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h | 14 ++++++++++- cpp/bench/ann/src/diskann/diskann_wrapper.h | 25 +++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h index d62d0dc34a..e8fdf502c2 100644 --- a/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h +++ b/cpp/bench/ann/src/cuvs/cuvs_vamana_wrapper.h @@ -190,7 +190,19 @@ void cuvs_vamana::load(const std::string& file) cuvs_vamana_artifact_exists(data_file) ? 1 : 0, static_cast(data_bytes)); - diskann_memory_search_->load(file); + try { + log_info("cuvs_vamana load delegating to diskann_memory: file=%s", file.c_str()); + diskann_memory_search_->load(file); + } catch (const std::exception& e) { + log_warn("cuvs_vamana load failed while loading diskann_memory: file=%s error=%s", + file.c_str(), + e.what()); + throw; + } catch (...) { + log_warn("cuvs_vamana load failed while loading diskann_memory: file=%s unknown error", + file.c_str()); + throw; + } log_info("cuvs_vamana load finish: file=%s", file.c_str()); } diff --git a/cpp/bench/ann/src/diskann/diskann_wrapper.h b/cpp/bench/ann/src/diskann/diskann_wrapper.h index c8510bd761..09edf9b6b7 100644 --- a/cpp/bench/ann/src/diskann/diskann_wrapper.h +++ b/cpp/bench/ann/src/diskann/diskann_wrapper.h @@ -246,11 +246,9 @@ void diskann_memory::load(const std::string& index_file) bench_mode_ = (cuvs::bench::benchmark_n_threads > 1) ? Mode::kThroughput : Mode::kLatency; - initialize_index_(0); - int load_threads = (bench_mode_ == Mode::kThroughput) ? cuvs::bench::benchmark_n_threads : 1; log_info( - "diskann_memory load start: prefix=%s dim=%d benchmark_threads=%d load_threads=%d " + "diskann_memory load enter: prefix=%s dim=%d benchmark_threads=%d load_threads=%d " "mode=%s build_threads=%d", index_path_prefix_.c_str(), this->dim_, @@ -258,12 +256,33 @@ void diskann_memory::load(const std::string& index_file) load_threads, bench_mode_ == Mode::kThroughput ? "throughput" : "latency", num_threads_); + + try { + log_info("diskann_memory initialize_index start: prefix=%s max_points=0", + index_path_prefix_.c_str()); + initialize_index_(0); + log_info("diskann_memory initialize_index finish: prefix=%s", index_path_prefix_.c_str()); + } catch (const std::exception& e) { + log_warn("diskann_memory initialize_index failed: prefix=%s error=%s", + index_path_prefix_.c_str(), + e.what()); + throw; + } catch (...) { + log_warn("diskann_memory initialize_index failed: prefix=%s unknown error", + index_path_prefix_.c_str()); + throw; + } + + log_info("diskann_memory load start: prefix=%s", index_path_prefix_.c_str()); try { this->mem_index_->load(index_path_prefix_.c_str(), load_threads, 100); } catch (const std::exception& e) { log_warn( "diskann_memory load failed: prefix=%s error=%s", index_path_prefix_.c_str(), e.what()); throw; + } catch (...) { + log_warn("diskann_memory load failed: prefix=%s unknown error", index_path_prefix_.c_str()); + throw; } log_info("diskann_memory load finish: prefix=%s", index_path_prefix_.c_str()); }