Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions cpp/src/neighbors/all_neighbors/all_neighbors_batched.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#pragma once
#include "all_neighbors_builder.cuh"
#include "raft/core/logger_macros.hpp"
#include <cuvs/cluster/kmeans.hpp>
#include <cuvs/distance/distance.hpp>
#include <cuvs/neighbors/all_neighbors.hpp>
Expand Down Expand Up @@ -336,6 +337,17 @@ void multi_gpu_batch_build(const raft::resources& handle,
size_t base_cluster_idx = rank * clusters_per_rank + std::min((size_t)rank, rem);
size_t num_clusters_for_this_rank =
(size_t)rank < rem ? clusters_per_rank + 1 : clusters_per_rank;
if (num_clusters_for_this_rank == 0) {
// Happens in the case when num_ranks > n_clusters.
RAFT_LOG_WARN(
"Rank %d is not used for computation. This happens because the total number of ranks (%d) "
"> n_clusters (%lu). Consider increasing n_clusters or reduce the number of GPUs for "
"better utilization.",
rank,
num_ranks,
params.n_clusters);
continue;
}
for (size_t p = 0; p < num_clusters_for_this_rank; p++) {
num_data_for_this_rank += cluster_sizes(base_cluster_idx + p);
}
Expand Down
Loading