Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions backends/vulkan/op_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ def register_copy_op():
def register_unaryop_cpp_ops():
return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_T,
inputs_dtypes=utils.FP_BF16_T,
supports_resize=True,
)

Expand All @@ -250,7 +250,7 @@ def register_unaryop_cpp_ops():
def register_clamp():
return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_INT_T,
inputs_dtypes=utils.FP_BF16_INT_T,
supports_resize=True,
)

Expand All @@ -273,7 +273,7 @@ def register_clamp():
def register_binaryop_cpp_ops():
return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_INT_T,
inputs_dtypes=utils.FP_BF16_INT_T,
supports_resize=True,
supports_highdim=True,
)
Expand All @@ -291,7 +291,7 @@ def _check_div_rounding_mode_is_floor(node: torch.fx.Node) -> bool:
def register_div_tensor_mode():
return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_INT_T,
inputs_dtypes=utils.FP_BF16_INT_T,
supports_resize=True,
supports_highdim=True,
are_node_inputs_supported_fn=_check_div_rounding_mode_is_floor,
Expand Down Expand Up @@ -374,7 +374,7 @@ def is_scalar_value_supported(value: Any, dtype: torch.dtype) -> bool:
def register_binary_scalar_ops():
return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_T,
inputs_dtypes=utils.FP_BF16_T,
supports_resize=True,
supports_highdim=True,
are_node_inputs_supported_fn=lambda node: is_scalar_value_supported(
Expand Down Expand Up @@ -402,6 +402,17 @@ def register_eq_scalar():
# =============================================================================


def _check_bfloat16_conversion(node: torch.fx.Node) -> bool:
"""bfloat16 converts to and from float32, and from integers."""
in_dtype = node.args[0].meta["val"].dtype # type: ignore[union-attr]
out_dtype = node.meta["val"].dtype
if in_dtype == torch.bfloat16:
return out_dtype in {torch.bfloat16, torch.float32}
if out_dtype == torch.bfloat16:
return in_dtype in {torch.float32, torch.int32, torch.int64}
return True


@update_features(exir_ops.edge.aten._to_copy.default)
def register_to_copy():
def pick_to_copy_storage(
Expand All @@ -416,10 +427,11 @@ def pick_to_copy_storage(

return OpFeatures(
inputs_storage=utils.ANY_STORAGE,
inputs_dtypes=utils.FP_INT_BOOL_T,
outputs_dtypes=utils.FP_INT_BOOL_T,
inputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16},
outputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16},
supports_resize=True,
pick_io_storage_fn=pick_to_copy_storage,
are_node_inputs_supported_fn=_check_bfloat16_conversion,
)


Expand Down Expand Up @@ -1332,9 +1344,10 @@ def register_view_copy():
def register_to_dim_order_copy():
return OpFeatures(
inputs_storage=utils.ANY_BUFFER,
inputs_dtypes=utils.FP_INT_BOOL_T,
inputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16},
supports_resize=True,
supports_highdim=True,
are_node_inputs_supported_fn=_check_bfloat16_conversion,
)


Expand Down
35 changes: 24 additions & 11 deletions backends/vulkan/partitioner/vulkan_partitioner.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,15 @@
logger.setLevel(logging.INFO)


def _tensor_dtypes(nodes: Iterable[torch.fx.Node]) -> Set[torch.dtype]:
return {
value.dtype
for node in nodes
for value in tree_leaves(node.meta.get("val"))
if isinstance(value, torch.Tensor)
}


class VulkanSupportedOperators(OperatorSupportBase):
def __init__(
self,
Expand Down Expand Up @@ -106,22 +115,26 @@ def __init__(
)
# Create a set of all nodes that are part of fusable subgraphs for quick lookup
self.fusable_nodes: Set[torch.fx.Node] = set()
self.unsupported_fusable_nodes: Set[torch.fx.Node] = set()
# Maps nodes of fusable patterns that must stay outside Vulkan to the reason.
self.unsupported_fusable_nodes: Dict[torch.fx.Node, str] = {}
for match in self.fusable_subgraphs:
nodes = {
node for node in match.all_nodes if isinstance(node, torch.fx.Node)
}
self.fusable_nodes.update(nodes)
inputs = {arg for node in nodes for arg in node.all_input_nodes}
if not downcast_64_bit and any(
isinstance(value, torch.Tensor)
and value.dtype in (torch.int64, torch.float64)
for node in (nodes | inputs) - (constant_nodes or set())
for value in tree_leaves(node.meta.get("val"))
):
# Keep the whole pattern outside Vulkan instead of splitting a fusion.

# Keep the whole pattern outside Vulkan instead of splitting a fusion.
reason = None
if not downcast_64_bit and _tensor_dtypes(
(nodes | inputs) - (constant_nodes or set())
) & {torch.int64, torch.float64}:
# Constant quantization parameters may disappear during fusion.
self.unsupported_fusable_nodes.update(nodes)
reason = "fusable pattern requires 64-bit tensor downcasting"
elif torch.bfloat16 in _tensor_dtypes(nodes | inputs):
reason = "fused operators do not support bfloat16"
if reason is not None:
self.unsupported_fusable_nodes.update(dict.fromkeys(nodes, reason))

self.nn_module_blocklist = nn_module_blocklist
self.nn_module_allowlist = nn_module_allowlist
Expand Down Expand Up @@ -251,7 +264,7 @@ def _is_node_supported(self, node: torch.fx.Node) -> bool:

def _check_node_support(self, node: torch.fx.Node) -> bool: # noqa: C901
if node in self.unsupported_fusable_nodes:
self.log_skip(node, "fusable pattern requires 64-bit tensor downcasting")
self.log_skip(node, self.unsupported_fusable_nodes[node])
return False

# Asserts and the symbolic comparisons feeding them are removed during
Expand Down Expand Up @@ -302,7 +315,7 @@ def _check_node_support(self, node: torch.fx.Node) -> bool: # noqa: C901

if not self.downcast_64_bit:
native_dtypes = utils.DtypeSetList(
utils.ALL_T - {torch.int64, torch.float64}
utils.VULKAN_T - {torch.int64, torch.float64}
)
dtype_valid, dtype_reason = utils.check_node_dtypes(
node, native_dtypes, native_dtypes
Expand Down
1 change: 1 addition & 0 deletions backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ view_buffer:
generate_variant_forall:
DTYPE:
- VALUE: half
- VALUE: bfloat16
- VALUE: float
- VALUE: double
- VALUE: int8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,8 @@ view_convert_buffer:
- parameter_values: [float, half]
- parameter_values: [half, float]
- parameter_values: [half, int32]
- parameter_values: [int32, bfloat16]
- parameter_values: [float, bfloat16]
- parameter_values: [bfloat16, float]
shader_variants:
- NAME: view_convert_buffer
14 changes: 11 additions & 3 deletions backends/vulkan/runtime/graph/ops/impl/ToCopy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,21 @@ void add_to_copy_node(ComputeGraph& graph, ValueRef in, ValueRef out) {
const vkapi::ScalarType in_dtype = graph.dtype_of(in);
const vkapi::ScalarType out_dtype = graph.dtype_of(out);

// Same-dtype or float<->half conversions can use BlitNode
if (in_dtype == out_dtype ||
(is_float_type(in_dtype) && is_float_type(out_dtype))) {
// Same-dtype or float<->half conversions of textures can use BlitNode
if (!graph.is_buffer_storage(out) &&
(in_dtype == out_dtype ||
(is_float_type(in_dtype) && is_float_type(out_dtype)))) {
graph.execute_nodes().emplace_back(new BlitNode(graph, in, out));
return;
}

// Buffers cannot be blitted. On devices that store bfloat16 as fp32, bfloat16
// casts copy between buffers of one dtype.
if (in_dtype == out_dtype) {
add_view_copy_node(graph, in, out, {}, resize_to_copy_op_node);
return;
}

// Other conversions (e.g. int<->float) use view_convert shaders
add_view_copy_convert_node(graph, in, out, {}, resize_to_copy_op_node);
}
Expand Down
77 changes: 76 additions & 1 deletion backends/vulkan/test/test_vulkan_delegate.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,13 +73,14 @@ def lower_module(
sample_inputs: Tuple[torch.Tensor],
dynamic_shapes=None,
compile_options=None,
skip_dim_order: bool = False,
) -> EdgeProgramManager:
compile_options = dict(compile_options or {})
if dynamic_shapes is not None:
compile_options["require_dynamic_shapes"] = True

edge_compile_config = EdgeCompileConfig(
_skip_dim_order=False, # TODO(T182928844): Delegate dim order op to backend.
_skip_dim_order=skip_dim_order, # TODO(T182928844): Delegate dim order op to backend.
)

program: ExportedProgram = export(
Expand Down Expand Up @@ -349,6 +350,80 @@ def forward(self, x, y):

self.lower_module_and_test_output(add_int_module, sample_inputs)

def test_vulkan_backend_bfloat16_elementwise(self):
class Bfloat16Module(torch.nn.Module):
def __init__(self):
super().__init__()
self.weight = torch.rand(size=(3, 17), dtype=torch.bfloat16)

def forward(self, x, y):
z = torch.sigmoid(x * self.weight) - torch.tanh(x / (y.abs() + 1))
z = torch.clamp(z, -0.75, 0.75) + torch.relu(y) * 2
return torch.pow(z, 2.0)

model = Bfloat16Module().eval()
sample_inputs = (
torch.randn(size=(2, 3, 17), dtype=torch.bfloat16),
torch.randn(size=(2, 1, 17), dtype=torch.bfloat16),
)
et_program = lower_module(model, sample_inputs).to_executorch()
self.check_vk_delegation(et_program)
self.assertEqual(
len(et_program.executorch_program.execution_plan[0].operators), 0
)
self.run_delegated_model_and_check_output(
et_program, model, sample_inputs, atol=2e-2, rtol=2e-2
)

def test_vulkan_backend_bfloat16_casts(self):
class Bfloat16CastModule(torch.nn.Module):
def forward(self, x, i):
x = x.to(torch.bfloat16).to(torch.float32)
return x + i.to(torch.bfloat16).to(torch.float32)

model = Bfloat16CastModule().eval()
sample_inputs = (
torch.randn(size=(2, 3, 17), dtype=torch.float32),
torch.randint(-128, 128, size=(2, 3, 17), dtype=torch.int32),
)
# Skipping dim order lowers the casts to _to_copy instead of
# _to_dim_order_copy.
for skip_dim_order in (False, True):
with self.subTest(skip_dim_order=skip_dim_order):
et_program = lower_module(
model, sample_inputs, skip_dim_order=skip_dim_order
).to_executorch()
self.check_vk_delegation(et_program)
self.assertEqual(
len(et_program.executorch_program.execution_plan[0].operators), 0
)
self.run_delegated_model_and_check_output(
et_program, model, sample_inputs, atol=2e-2, rtol=2e-2
)

def test_vulkan_backend_bfloat16_quantized_linear_not_fused(self):
# Fused operators have no bfloat16 shaders, so the pattern stays on the CPU.
class QuantizedLinearModule(torch.nn.Module):
def __init__(self):
super().__init__()
self.linear = torch.nn.Linear(64, 32, bias=False, dtype=torch.bfloat16)

def forward(self, x):
return self.linear(x)

model = QuantizedLinearModule().eval()
quantize_(
model,
IntxWeightOnlyConfig(weight_dtype=torch.int4, granularity=PerGroup(32)),
)
unwrap_tensor_subclass(model)
sample_inputs = (torch.randn(size=(1, 64), dtype=torch.bfloat16),)
edge_program = lower_module(model, sample_inputs)
self.assertNotIn(
torch.ops.higher_order.executorch_call_delegate,
[node.target for node in edge_program.exported_program().graph.nodes],
)

def test_vulkan_backend_zero_dim_tensor(self):
class ZeroDimModule(torch.nn.Module):
def __init__(self):
Expand Down
9 changes: 8 additions & 1 deletion backends/vulkan/test/vulkan_compute_api_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3986,7 +3986,7 @@ TEST(VulkanComputeGraphTest, buffer_copy_convert_preserves_order_after_resize) {
{utils::kWidthPacked, utils::kHeightPacked, utils::kChannelsPacked}) {
for (const auto out_layout :
{utils::kWidthPacked, utils::kHeightPacked, utils::kChannelsPacked}) {
for (const auto dtype : {vkapi::kFloat, vkapi::kHalf}) {
for (const auto dtype : {vkapi::kFloat, vkapi::kHalf, vkapi::kBFloat16}) {
SCOPED_TRACE(
::testing::Message() << "in_layout=" << in_layout << " out_layout="
<< out_layout << " dtype=" << dtype);
Expand Down Expand Up @@ -4019,6 +4019,13 @@ TEST(VulkanComputeGraphTest, buffer_copy_convert_preserves_order_after_resize) {
for (size_t i = 0; i < data.size(); ++i) {
EXPECT_EQ(float(output[i]), data[i]);
}
} else if (dtype == vkapi::kBFloat16) {
std::vector<executorch::aten::BFloat16> output(data.size());
graph.maybe_cast_and_copy_from_staging(
staging, output.data(), output.size(), dtype);
for (size_t i = 0; i < data.size(); ++i) {
EXPECT_EQ(float(output[i]), data[i]);
}
} else {
std::vector<float> output(data.size());
graph.maybe_cast_and_copy_from_staging(
Expand Down
15 changes: 12 additions & 3 deletions backends/vulkan/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
torch.int32: VkDataType.INT32,
torch.int64: VkDataType.INT64,
torch.float16: VkDataType.FLOAT16,
torch.bfloat16: VkDataType.BFLOAT16,
torch.float32: VkDataType.FLOAT32,
torch.float64: VkDataType.FLOAT64,
}
Expand All @@ -65,12 +66,19 @@
INT_T: DtypeSet = {torch.int32, torch.int64}
QINT8_T: DtypeSet = {torch.int8}
BOOL_T: DtypeSet = {torch.bool}
ALL_T: DtypeSet = set(_VULKAN_DTYPES.keys())
# Every serializable dtype. bfloat16 is opt-in per operator, so ALL_T, the
# default operator dtype set, excludes it. Devices without bfloat16 support store
# and compute bfloat16 tensors in fp32 without rounding to bfloat16, so results
# can differ from eager bfloat16.
VULKAN_T: DtypeSet = set(_VULKAN_DTYPES.keys())
ALL_T: DtypeSet = VULKAN_T - {torch.bfloat16}
NONE_T: DtypeSet = set() # Marker for non-tensor args (skip validation)

# Composite dtype sets for specific operator requirements
FP_INT_T: DtypeSet = FP_T | INT_T
FP_INT_BOOL_T: DtypeSet = FP_T | INT_T | BOOL_T
FP_BF16_T: DtypeSet = FP_T | {torch.bfloat16}
FP_BF16_INT_T: DtypeSet = FP_INT_T | {torch.bfloat16}


class DtypeSetList:
Expand Down Expand Up @@ -1267,8 +1275,9 @@ def filter_invalid_reprs(
if extents_are_valid(extents, texture_limits):
valid_texture_layouts.add(memory_layout)

# High dimensional tensors require buffer storage
if len(tensor_val.shape) > 4:
# High dimensional tensors require buffer storage, as do bfloat16 tensors,
# since there is no bfloat16 image format.
if len(tensor_val.shape) > 4 or tensor_val.dtype == torch.bfloat16:
return TensorRepSet(tensor_repset.valid_buffer_layouts, set())

return TensorRepSet(tensor_repset.valid_buffer_layouts, valid_texture_layouts)
Expand Down
Loading