Skip to content

Commit 03cc7d8

Browse files
xuyanwen2012claude
andcommitted
[ET-VK] Rename gemm_double_buf reference shader to coopmat_mm_ref
The test/custom_ops reference port of NVIDIA's shmem_double_buf4.comp is the comparison baseline for the coopmat shaders, so name it after what it is (the coopmat_mm reference) rather than how it buffers. Shader, yaml, op (etvk.coopmat_mm_ref), impl file, bench labels, and comment references updated; validated on device (fp16 GEMM bench: 6/6 correctness PASS, numbers unchanged). Authored with Claude. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent bfcb14a commit 03cc7d8

9 files changed

Lines changed: 41 additions & 41 deletions

File tree

backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.glsl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
* unsigned int4 nibbles in dotPacked4x8) cancels out and is not needed.
2828
*
2929
* Loop structure follows the NVIDIA double-buffered GEMM reference
30-
* (shmem_double_buf4.comp "store-first" variant; see gemm_double_buf.glsl in
30+
* (shmem_double_buf4.comp "store-first" variant; see coopmat_mm_ref.glsl in
3131
* test/custom_ops): prologue register prefetch, then per chunk
3232
* barrier -> prefetch next chunk -> int8 MMA on the current LDS slice ->
3333
* store temp into the other slice. One barrier per chunk; the prefetch is

backends/vulkan/runtime/graph/ops/glsl/linear_dq8ca_qw_coopmat.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
# WEIGHT_NBITS=8 -> linear_dq8ca_q8csw_coopmat (INT8 per-channel symmetric)
1212
# Requires the VK_COMPONENT_TYPE_SINT8_KHR cooperative matrix property to be
1313
# enumerated on the device.
14-
# Loop structure follows the double-buffered reference (gemm_double_buf) at
14+
# Loop structure follows the double-buffered reference (coopmat_mm_ref) at
1515
# a 128x64 tile with K-step 32, 4 subgroups x 64 threads. The reference's
1616
# subgroup-32 layout is NOT used — the Xclipse PAL compiler crashes in
1717
# vkCreateComputePipelines when int8 WMMA is compiled at forced subgroup

backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.glsl

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
* Bsh, keeping the K-loop a clean fp16 MMA.
2323
*
2424
* Loop structure follows the NVIDIA double-buffered GEMM reference
25-
* (shmem_double_buf4.comp, "store-first" variant; see gemm_double_buf.glsl
25+
* (shmem_double_buf4.comp, "store-first" variant; see coopmat_mm_ref.glsl
2626
* in test/custom_ops — measured 1.5x faster than the previous
2727
* single-buffered skeleton at fp16 on Xclipse 970):
2828
* - PROLOGUE: prefetch tile 0 from global memory into temp registers, then
@@ -110,7 +110,7 @@ ${layout_declare_spec_const(C, "int", "num_groups_arg", "0")}
110110
// the first store per subgroup lands correctly; standalone repro cm_acc2).
111111
${layout_declare_spec_const(C, "int", "out_N_arg", "0")}
112112

113-
// --- Tile geometry (from yaml; defaults match gemm_double_buf) ---
113+
// --- Tile geometry (from yaml; defaults match coopmat_mm_ref) ---
114114
const uint MMA_M = ${MMA_M};
115115
const uint MMA_N = ${MMA_N};
116116
const uint MMA_K = ${MMA_K};

backends/vulkan/runtime/graph/ops/glsl/linear_qw_coopmat.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
# Forces buffer storage for activation/output (coopMatLoad/Store on buffers);
1313
# INT weight storage can be texture2d or buffer (matches the tiled path).
1414
# DTYPE = half only; fp32 activations are not supported.
15-
# Geometry follows the double-buffered reference (gemm_double_buf): 128x128
15+
# Geometry follows the double-buffered reference (coopmat_mm_ref): 128x128
1616
# tile, K-step 16, 8 subgroups x 32 threads (subgroup size 32 forced).
1717

1818
linear_qw_coopmat:

backends/vulkan/runtime/graph/ops/impl/QuantizedLinear.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ void resize_linear_qw_node(
5454
}
5555

5656
// Per-shader coopmat tile geometry (must match each shader's yaml). The
57-
// shaders restructured to the double-buffered reference (gemm_double_buf)
57+
// shaders restructured to the double-buffered reference (coopmat_mm_ref)
5858
// use larger tiles and K-step 16; the rest keep the GemmCoopmat.h 64x64x32
5959
// geometry. All use 256-thread workgroups.
6060
// linear_q4gsw_coopmat 128x128x16, 8 subgroups x 32 (forced)

backends/vulkan/test/custom_ops/glsl/gemm_double_buf.glsl renamed to backends/vulkan/test/custom_ops/glsl/coopmat_mm_ref.glsl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838

3939
layout(std430) buffer;
4040

41-
// Bindings — match add_gemm_double_buf_node: output(0), mat1(1), mat2(2).
41+
// Bindings — match add_coopmat_mm_ref_node: output(0), mat1(1), mat2(2).
4242
layout(set = 0, binding = 0) buffer restrict writeonly t_outputBuffer {
4343
float16_t t_output[]; // fp16 D [M, N]
4444
};

backends/vulkan/test/custom_ops/glsl/gemm_double_buf.yaml renamed to backends/vulkan/test/custom_ops/glsl/coopmat_mm_ref.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
# the ET shader system. Geometry matches the reference's standalone harness:
99
# one workgroup = one 128x128 output tile, K-step 16, 8 subgroups x 32 threads.
1010

11-
gemm_double_buf:
11+
coopmat_mm_ref:
1212
parameter_names_with_default_values:
1313
MMA_M: 16
1414
MMA_N: 16
@@ -20,4 +20,4 @@ gemm_double_buf:
2020
SG_GRID_Y: 2
2121
SUBGROUP_SIZE: 32
2222
shader_variants:
23-
- NAME: gemm_double_buf_half
23+
- NAME: coopmat_mm_ref_half

backends/vulkan/test/custom_ops/impl/GemmDoubleBuf.cpp renamed to backends/vulkan/test/custom_ops/impl/CoopmatMmRef.cpp

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -14,26 +14,26 @@
1414
namespace vkcompute {
1515

1616
// Dispatch for the ported NVIDIA double-buffered coopmat GEMM reference
17-
// (gemm_double_buf.glsl): D[M,N] = A[M,K] x B[K,N], all fp16 buffers,
17+
// (coopmat_mm_ref.glsl): D[M,N] = A[M,K] x B[K,N], all fp16 buffers,
1818
// row-major. One workgroup per 128x128 output tile, 256 threads, subgroup
1919
// size forced to 32 by the shader's REQUIRED_SUBGROUP_SIZE annotation.
2020

21-
constexpr uint32_t kDbTileM = 128;
22-
constexpr uint32_t kDbTileN = 128;
23-
constexpr uint32_t kDbTileK = 16;
24-
constexpr uint32_t kDbInvocations = 256; // 8 subgroups x 32
21+
constexpr uint32_t kRefTileM = 128;
22+
constexpr uint32_t kRefTileN = 128;
23+
constexpr uint32_t kRefTileK = 16;
24+
constexpr uint32_t kRefInvocations = 256; // 8 subgroups x 32
2525

26-
static vkapi::ShaderInfo pick_gemm_double_buf_shader(
26+
static vkapi::ShaderInfo pick_coopmat_mm_ref_shader(
2727
ComputeGraph* graph,
2828
const std::vector<ArgGroup>& args,
2929
const std::vector<ValueRef>& resize_args) {
3030
(void)graph;
3131
(void)args;
3232
(void)resize_args;
33-
return VK_KERNEL_FROM_STR("gemm_double_buf_half");
33+
return VK_KERNEL_FROM_STR("coopmat_mm_ref_half");
3434
}
3535

36-
static utils::uvec3 pick_gemm_double_buf_global_wg_size(
36+
static utils::uvec3 pick_coopmat_mm_ref_global_wg_size(
3737
ComputeGraph* graph,
3838
const vkapi::ShaderInfo& shader,
3939
const std::vector<ArgGroup>& args,
@@ -45,15 +45,15 @@ static utils::uvec3 pick_gemm_double_buf_global_wg_size(
4545
const uint32_t M = out_sizes.at(out_sizes.size() - 2);
4646
const uint32_t N = out_sizes.at(out_sizes.size() - 1);
4747
// Same group-count cancellation trick as GemmCoopmat.cpp: the framework
48-
// divides by the local size, so multiplying tiles_n by kDbInvocations
48+
// divides by the local size, so multiplying tiles_n by kRefInvocations
4949
// yields exactly tiles_n x tiles_m workgroups.
5050
return {
51-
utils::div_up(N, kDbTileN) * kDbInvocations,
52-
utils::div_up(M, kDbTileM),
51+
utils::div_up(N, kRefTileN) * kRefInvocations,
52+
utils::div_up(M, kRefTileM),
5353
1};
5454
}
5555

56-
static utils::uvec3 pick_gemm_double_buf_local_wg_size(
56+
static utils::uvec3 pick_coopmat_mm_ref_local_wg_size(
5757
ComputeGraph* graph,
5858
const vkapi::ShaderInfo& shader,
5959
const utils::uvec3& global_workgroup_size,
@@ -64,10 +64,10 @@ static utils::uvec3 pick_gemm_double_buf_local_wg_size(
6464
(void)global_workgroup_size;
6565
(void)args;
6666
(void)resize_args;
67-
return {kDbInvocations, 1, 1};
67+
return {kRefInvocations, 1, 1};
6868
}
6969

70-
void gemm_double_buf(ComputeGraph& graph, const std::vector<ValueRef>& args) {
70+
void coopmat_mm_ref(ComputeGraph& graph, const std::vector<ValueRef>& args) {
7171
int idx = 0;
7272
const ValueRef mat1 = args.at(idx++);
7373
const ValueRef mat2 = args.at(idx++);
@@ -82,15 +82,15 @@ void gemm_double_buf(ComputeGraph& graph, const std::vector<ValueRef>& args) {
8282
const int32_t N = graph.size_at<int32_t>(-1, out);
8383
const int32_t K = graph.size_at<int32_t>(-1, mat1);
8484
// No partial-tile or K-tail handling in the reference shader.
85-
VK_CHECK_COND(M % static_cast<int32_t>(kDbTileM) == 0);
86-
VK_CHECK_COND(N % static_cast<int32_t>(kDbTileN) == 0);
87-
VK_CHECK_COND(K % static_cast<int32_t>(kDbTileK) == 0);
85+
VK_CHECK_COND(M % static_cast<int32_t>(kRefTileM) == 0);
86+
VK_CHECK_COND(N % static_cast<int32_t>(kRefTileN) == 0);
87+
VK_CHECK_COND(K % static_cast<int32_t>(kRefTileK) == 0);
8888

8989
graph.execute_nodes().emplace_back(new DynamicDispatchNode(
9090
graph,
91-
pick_gemm_double_buf_shader,
92-
pick_gemm_double_buf_global_wg_size,
93-
pick_gemm_double_buf_local_wg_size,
91+
pick_coopmat_mm_ref_shader,
92+
pick_coopmat_mm_ref_global_wg_size,
93+
pick_coopmat_mm_ref_local_wg_size,
9494
// Inputs and Outputs
9595
{{out, vkapi::kWrite}, {{mat1, mat2}, vkapi::kRead}},
9696
// Shader params buffers — none; all geometry is spec constants
@@ -106,7 +106,7 @@ void gemm_double_buf(ComputeGraph& graph, const std::vector<ValueRef>& args) {
106106
}
107107

108108
REGISTER_OPERATORS {
109-
VK_REGISTER_OP(etvk.gemm_double_buf, gemm_double_buf);
109+
VK_REGISTER_OP(etvk.coopmat_mm_ref, coopmat_mm_ref);
110110
}
111111

112112
} // namespace vkcompute

backends/vulkan/test/custom_ops/test_fp16_gemm_bench.cpp

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
// tiled matmul_vec, Texture3D (production default baseline)
99
// coopmat matmul_coopmat (coopmat_mm.glsl), Buffer — our shader, forced
1010
// past the desktop-only gate via test_etvk.test_mm "coopmat"
11-
// dbuf4 gemm_double_buf (NVIDIA shmem_double_buf4 reference port),
11+
// coopmat_ref coopmat_mm_ref (NVIDIA shmem_double_buf4 reference port),
1212
// Buffer — double-buffered shared memory, subgroup size 32
1313
//
1414
// Apples-to-apples: same shapes, same runtime-mat2 row-major [K,N] fp16
@@ -30,9 +30,9 @@ struct GemmConfig {
3030
int64_t N;
3131
};
3232

33-
// Impl rows benchmarked per shape. The dbuf4 tile is 128x128 (vs 64x64 for
33+
// Impl rows benchmarked per shape. The coopmat_ref tile is 128x128 (vs 64x64 for
3434
// coopmat), so correctness shapes must align to 128.
35-
static const std::vector<std::string> kImpls = {"tiled", "coopmat", "dbuf4"};
35+
static const std::vector<std::string> kImpls = {"tiled", "coopmat", "coopmat_ref"};
3636

3737
static TestCase make_case(const GemmConfig& cfg, const std::string& impl) {
3838
const vkapi::ScalarType dt = vkapi::kHalf;
@@ -53,8 +53,8 @@ static TestCase make_case(const GemmConfig& cfg, const std::string& impl) {
5353
ValueSpec output({cfg.M, cfg.N}, dt, storage, utils::kWidthPacked,
5454
DataGenType::ZEROS);
5555

56-
if (impl == "dbuf4") {
57-
tc.set_operator_name("etvk.gemm_double_buf");
56+
if (impl == "coopmat_ref") {
57+
tc.set_operator_name("etvk.coopmat_mm_ref");
5858
tc.add_input_spec(mat1);
5959
tc.add_input_spec(mat2);
6060
} else {
@@ -68,7 +68,7 @@ static TestCase make_case(const GemmConfig& cfg, const std::string& impl) {
6868
}
6969
tc.add_output_spec(output);
7070

71-
// tiled accumulates in fp16 (error grows with K); coopmat/dbuf4 accumulate
71+
// tiled accumulates in fp16 (error grows with K); coopmat/coopmat_ref accumulate
7272
// in fp32, bounded by fp16 input/output rounding only.
7373
if (impl == "tiled") {
7474
tc.set_abs_tolerance(1.0f);
@@ -128,8 +128,8 @@ std::vector<TestCase> generate_cases() {
128128
}
129129
}
130130
}
131-
// Correctness: aligned to the dbuf4 128x128 tile (and coopmat's 64/32);
132-
// the second shape dispatches a 2x2 workgroup grid for dbuf4.
131+
// Correctness: aligned to the coopmat_ref 128x128 tile (and coopmat's 64/32);
132+
// the second shape dispatches a 2x2 workgroup grid for coopmat_ref.
133133
static const std::vector<GemmConfig> kCorrectnessShapes = {
134134
{128, 64, 128}, {256, 128, 256}};
135135
for (const auto& cfg : kCorrectnessShapes) {
@@ -172,7 +172,7 @@ int main() {
172172
for (const auto& st : r.get_shader_timings()) {
173173
if (st.shader_name.find("matmul") != std::string::npos ||
174174
st.shader_name.find("coopmat") != std::string::npos ||
175-
st.shader_name.find("double_buf") != std::string::npos) {
175+
st.shader_name.find("coopmat_mm_ref") != std::string::npos) {
176176
name = st.shader_name;
177177
}
178178
}
@@ -183,7 +183,7 @@ int main() {
183183
<< ") ==========\n";
184184
std::cout << std::left << std::setw(15) << "shape(K,N)" << std::right
185185
<< std::setw(10) << "tiled" << std::setw(10) << "coopmat"
186-
<< std::setw(10) << "dbuf4" << std::setw(12) << "dbuf4/coop"
186+
<< std::setw(12) << "coopmat_ref" << std::setw(10) << "ref/coop"
187187
<< " kernels\n";
188188
size_t idx = 0;
189189
for (const auto& kn : kShapes) {
@@ -201,7 +201,7 @@ int main() {
201201
std::to_string(kn.second) + ")")
202202
<< std::right << std::fixed << std::setprecision(1)
203203
<< std::setw(10) << tiled << std::setw(10) << coop
204-
<< std::setw(10) << dbuf << std::setw(11)
204+
<< std::setw(12) << dbuf << std::setw(9)
205205
<< std::setprecision(2) << (coop > 0 ? dbuf / coop : 0.0f)
206206
<< "x " << c_kernel << " | " << d_kernel << "\n";
207207
}

0 commit comments

Comments
 (0)