Implement the new tuning API for DeviceTransform

c/parallel/src/kernels/iterators.cpp

@@ -114,7 +114,7 @@ struct {0}_proxy_t {{
 struct {0} {{
   using iterator_category = cuda::std::random_access_iterator_tag;
   using difference_type   = DIFF_T;
-  using value_type        = void;
+  using value_type        = VALUE_T;
   using pointer           = {0}_proxy_t*;
   using reference         = {0}_proxy_t;
   __device__ {0}_proxy_t operator*() const {{ return {{state}}; }}

ci/matrix.yaml

@@ -21,6 +21,9 @@ workflows:
   #       args: '--preset libcudacxx --lit-tests "cuda/utility/basic_any.pass.cpp"' }
   #
   override:
+    - {jobs: ['build'], std: 'minmax', ctk: '12.0', cxx: ['msvc2019', 'msvc14.39']}
+    - {jobs: ['build'], std: 'minmax', ctk: '12.X', cxx: ['msvc2019', 'msvc'     ]}
+    - {jobs: ['build'], std: 'minmax', ctk: '13.0', cxx: ['msvc2019', 'msvc'     ]}
 
   pull_request:
     # Old CTK: Oldest/newest supported host compilers:

cub/benchmarks/bench/transform/babelstream.cu

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 // SPDX-License-Identifier: BSD-3-Clause
 
-// Because CUB cannot inspect the transformation function, we cannot add any tunings based on the results of this
-// benchmark. Its main use is to detect regressions.
-
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 1:4:1
 // %RANGE% TUNE_THREADS tpb 128:1024:128
-// %RANGE% TUNE_ALGORITHM alg 0:2:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
 
 #include "common.h"
 

cub/benchmarks/bench/transform/common.h

@@ -5,12 +5,12 @@
 
 // keep checks at the top so compilation of discarded variants fails really fast
 #include <cub/device/dispatch/dispatch_transform.cuh>
-#if !TUNE_BASE && TUNE_ALGORITHM == 2
+#if !TUNE_BASE && TUNE_ALGORITHM == 3
 #  if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
 #    error "When tuning, this benchmark does not support being compiled for multiple architectures"
 #  endif
 #  if (__CUDA_ARCH_LIST__) < 900
-#    error "Cannot compile algorithm 2 (ublkcp) below sm90"
+#    error "Cannot compile algorithm 3 (ublkcp) below sm90"
 #  endif
 #endif
 
@@ -23,32 +23,47 @@
 
 #include <nvbench_helper.cuh>
 
-template <typename RandomAccessIteratorOut, typename... RandomAccessIteratorsIn>
-#if TUNE_BASE
-using policy_hub_t =
-  cub::detail::transform::policy_hub</* stable address */ false,
-                                     /* dense output */ true,
-                                     ::cuda::std::tuple<RandomAccessIteratorsIn...>,
-                                     RandomAccessIteratorOut>;
-#else
-struct policy_hub_t
+#if !TUNE_BASE
+struct arch_policies
 {
-  struct max_policy : cub::ChainedPolicy<500, max_policy, max_policy>
+  _CCCL_API constexpr auto operator()(cuda::arch_id) const -> cub::detail::transform::transform_arch_policy
   {
-    static constexpr int min_bif = cub::detail::transform::arch_to_min_bytes_in_flight(__CUDA_ARCH_LIST__);
+    const int min_bif = cub::detail::transform::arch_to_min_bytes_in_flight(__CUDA_ARCH_LIST__) + TUNE_BIF_BIAS;
 #  if TUNE_ALGORITHM == 0
-    static constexpr auto algorithm = cub::detail::transform::Algorithm::prefetch;
+    const auto algorithm = cub::detail::transform::Algorithm::prefetch;
+    const auto policy    = prefetch_policy{
+      TUNE_THREADS
+#    ifdef TUNE_ITEMS_PER_THREAD_NO_INPUT
+      ,
+      TUNE_ITEMS_PER_THREAD_NO_INPUT
+#    endif
+    };
+    return {min_bif, algorithm, policy, {}, {}};
 #  elif TUNE_ALGORITHM == 1
-    static constexpr auto algorithm = cub::detail::transform::Algorithm::ublkcp;
+    const auto algorithm = cub::detail::transform::Algorithm::vectorized;
+    const auto policy    = vectorized_policy{
+      TUNE_THREADS,
+      TUNE_VEC_SIZE * TUNE_VECTORS_PER_THREAD,
+      TUNE_VEC_SIZE
+#    ifdef TUNE_ITEMS_PER_THREAD_NO_INPUT
+      ,
+      TUNE_ITEMS_PER_THREAD_NO_INPUT
+#    endif
+    };
+    return {min_bif, algorithm, {}, policy, {}};
+#  elif TUNE_ALGORITHM == 2
+    const auto algorithm = cub::detail::transform::Algorithm::memcpy_async;
+    const auto policy    = async_copy_policy{TUNE_THREADS, cub::detail::transform::ldgsts_size_and_align};
+    return {min_bif, algorithm, {}, {}, policy};
+#  elif TUNE_ALGORITHM == 3s
+    const auto algorithm = cub::detail::transform::Algorithm::ublkcp;
+    const auto policy =
+      async_copy_policy{TUNE_THREADS, cub::detail::transform::bulk_copy_alignment(__CUDA_ARCH_LIST__)};
+    return {min_bif, algorithm, {}, {}, policy};
 #  else
 #    error Policy hub does not yet implement the specified value for algorithm
 #  endif
-
-    using algo_policy =
-      ::cuda::std::_If<algorithm == cub::detail::transform::Algorithm::prefetch,
-                       cub::detail::transform::prefetch_policy_t<TUNE_THREADS>,
-                       cub::detail::transform::async_copy_policy_t<TUNE_THREADS, __CUDA_ARCH_LIST__ == 900 ? 128 : 16>>;
-  };
+  }
 };
 #endif
 
@@ -60,15 +75,17 @@ void bench_transform(nvbench::state& state,
                      TransformOp transform_op)
 {
   state.exec(nvbench::exec_tag::gpu, [&](const nvbench::launch& launch) {
-    cub::detail::transform::dispatch_t<
-      cub::detail::transform::requires_stable_address::no,
-      OffsetT,
-      ::cuda::std::tuple<RandomAccessIteratorsIn...>,
-      RandomAccessIteratorOut,
-      cub::detail::transform::always_true_predicate,
-      TransformOp,
-      policy_hub_t<RandomAccessIteratorOut, RandomAccessIteratorsIn...>>::
-      dispatch(
-        inputs, output, num_items, cub::detail::transform::always_true_predicate{}, transform_op, launch.get_stream());
+    cub::detail::transform::dispatch<cub::detail::transform::requires_stable_address::no>(
+      inputs,
+      output,
+      num_items,
+      cub::detail::transform::always_true_predicate{},
+      transform_op,
+      launch.get_stream()
+#if !TUNE_BASE
+        ,
+      arch_policies{}
+#endif
+    );
   });
 }

cub/benchmarks/bench/transform/complex_cmp.cu

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 // SPDX-License-Identifier: BSD-3-Clause
 
-// Because CUB cannot inspect the transformation function, we cannot add any tunings based on the results of this
-// benchmark. Its main use is to detect regressions.
-
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 1:4:1
 // %RANGE% TUNE_THREADS tpb 128:1024:128
-// %RANGE% TUNE_ALGORITHM alg 0:1:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
 
 #include "common.h"
 

cub/benchmarks/bench/transform/fib.cu

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 // SPDX-License-Identifier: BSD-3-Clause
 
-// Because CUB cannot inspect the transformation function, we cannot add any tunings based on the results of this
-// benchmark. Its main use is to detect regressions.
-
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 1:4:1
 // %RANGE% TUNE_THREADS tpb 128:1024:128
-// %RANGE% TUNE_ALGORITHM alg 0:1:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
 
 #include "common.h"
 

cub/benchmarks/bench/transform/fill.cu

@@ -1,11 +1,17 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-// Because CUB cannot inspect the transformation function, we cannot add any tunings based on the results of this
-// benchmark. Its main use is to detect regressions.
-
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// for filling, we can only use the prefetch and the vectorized algorithm
+// %RANGE% TUNE_ALGORITHM alg 1:2:1
 // %RANGE% TUNE_THREADS tpb 128:1024:128
-// %RANGE% TUNE_ALGORITHM alg 0:3:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_ITEMS_PER_THREAD_NO_INPUT ipt 1:32:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
 
 #include "common.h"
 

cub/benchmarks/bench/transform/grayscale.cu

@@ -1,6 +1,14 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: BSD-3-Clause
 
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 1:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
+
 #include "common.h"
 
 template <typename T>

cub/benchmarks/bench/transform/heavy.cu

@@ -1,11 +1,13 @@
 // SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 // SPDX-License-Identifier: BSD-3-Clause
 
-// Because CUB cannot inspect the transformation function, we cannot add any tunings based on the results of this
-// benchmark. Its main use is to detect regressions.
-
+// %RANGE% TUNE_BIF_BIAS alg -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 1:4:1
 // %RANGE% TUNE_THREADS tpb 128:1024:128
-// %RANGE% TUNE_ALGORITHM alg 0:1:1
+
+// TODO(bgruber): those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE ipt 1:32:1
+// %RANGE% TUNE_VECTORS_PER_THREAD vpt 1:4:1
 
 #include "common.h"
 

cub/cub/device/device_transform.cuh

@@ -17,6 +17,7 @@
 #include <cub/device/dispatch/dispatch_transform.cuh>
 #include <cub/util_namespace.cuh>
 
+#include <cuda/__execution/tune.h>
 #include <cuda/__functional/address_stability.h>
 #include <cuda/__stream/get_stream.h>
 #include <cuda/std/__execution/env.h>
@@ -44,25 +45,31 @@ struct ::cuda::proclaims_copyable_arguments<CUB_NS_QUALIFIER::detail::__return_c
 {};
 
 CUB_NAMESPACE_BEGIN
+namespace detail::transform
+{
+struct get_tuning_query_t
+{};
+} // namespace detail::transform
+
 //! DeviceTransform provides device-wide, parallel operations for transforming elements tuple-wise from multiple input
 //! sequences into an output sequence.
 struct DeviceTransform
 {
 private:
-  template <typename... RandomAccessIteratorsIn,
+  template <detail::transform::requires_stable_address StableAddress = detail::transform::requires_stable_address::no,
+            typename... RandomAccessIteratorsIn,
             typename RandomAccessIteratorOut,
             typename NumItemsT,
             typename Predicate,
             typename TransformOp,
-            typename StableAddress = cuda::std::false_type>
+            typename Env>
   CUB_RUNTIME_FUNCTION static cudaError_t TransformInternal(
     ::cuda::std::tuple<RandomAccessIteratorsIn...> inputs,
     RandomAccessIteratorOut output,
     NumItemsT num_items,
     Predicate predicate,
     TransformOp transform_op,
-    cudaStream_t stream,
-    StableAddress = {})
+    Env env)
   {
     using choose_offset_t = detail::choose_signed_offset<NumItemsT>;
     using offset_t        = typename choose_offset_t::type;
@@ -73,17 +80,32 @@ private:
       return error;
     }
 
-    return detail::transform::dispatch_t < StableAddress::value
-           ? detail::transform::requires_stable_address::yes
-           : detail::transform::requires_stable_address::no,
-           offset_t, ::cuda::std::tuple<RandomAccessIteratorsIn...>, RandomAccessIteratorOut, Predicate,
-           TransformOp > ::dispatch(
-             ::cuda::std::move(inputs),
-             ::cuda::std::move(output),
-             num_items,
-             ::cuda::std::move(predicate),
-             ::cuda::std::move(transform_op),
-             stream);
+    using tuning_env_t =
+      ::cuda::std::execution::__query_result_or_t<Env, ::cuda::execution::__get_tuning_t, ::cuda::std::execution::env<>>;
+    using transform_tuning_t =
+      ::cuda::std::execution::__query_result_or_t<tuning_env_t, detail::transform::get_tuning_query_t, int>;
+
+    if constexpr (!::cuda::std::is_same_v<transform_tuning_t, int>)
+    {
+      return detail::transform::dispatch<StableAddress>(
+        ::cuda::std::move(inputs),
+        ::cuda::std::move(output),
+        static_cast<offset_t>(num_items),
+        ::cuda::std::move(predicate),
+        ::cuda::std::move(transform_op),
+        get_stream(env),
+        transform_tuning_t{});
+    }
+    else
+    {
+      return detail::transform::dispatch<StableAddress>(
+        ::cuda::std::move(inputs),
+        ::cuda::std::move(output),
+        static_cast<offset_t>(num_items),
+        ::cuda::std::move(predicate),
+        ::cuda::std::move(transform_op),
+        get_stream(env));
+    }
   }
 
   template <typename Env>
@@ -146,7 +168,7 @@ public:
       num_items,
       detail::transform::always_true_predicate{},
       ::cuda::std::move(transform_op),
-      get_stream(env));
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
@@ -269,7 +291,7 @@ public:
       num_items,
       detail::transform::always_true_predicate{},
       ::cuda::std::move(generator),
-      get_stream(env));
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
@@ -322,7 +344,7 @@ public:
       num_items,
       detail::transform::always_true_predicate{},
       detail::__return_constant<Value>{::cuda::std::move(value)},
-      get_stream(env));
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
@@ -401,7 +423,7 @@ public:
       num_items,
       ::cuda::std::move(predicate),
       ::cuda::std::move(transform_op),
-      get_stream(env));
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
@@ -489,7 +511,7 @@ public:
       num_items,
       ::cuda::std::move(predicate),
       ::cuda::std::move(transform_op),
-      get_stream(env));
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
@@ -567,14 +589,13 @@ public:
     Env env = {})
   {
     _CCCL_NVTX_RANGE_SCOPE("cub::DeviceTransform::TransformStableArgumentAddresses");
-    return TransformInternal(
+    return TransformInternal<detail::transform::requires_stable_address::yes>(
       ::cuda::std::move(inputs),
       ::cuda::std::move(output),
       num_items,
       detail::transform::always_true_predicate{},
       ::cuda::std::move(transform_op),
-      get_stream(env),
-      ::cuda::std::true_type{});
+      ::cuda::std::move(env));
   }
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document