diff --git a/backends/vulkan/op_registry.py b/backends/vulkan/op_registry.py index 5fd9c91b3c8..efa41b2df0d 100644 --- a/backends/vulkan/op_registry.py +++ b/backends/vulkan/op_registry.py @@ -241,7 +241,7 @@ def register_copy_op(): def register_unaryop_cpp_ops(): return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_T, + inputs_dtypes=utils.FP_BF16_T, supports_resize=True, ) @@ -250,7 +250,7 @@ def register_unaryop_cpp_ops(): def register_clamp(): return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_INT_T, + inputs_dtypes=utils.FP_BF16_INT_T, supports_resize=True, ) @@ -273,7 +273,7 @@ def register_clamp(): def register_binaryop_cpp_ops(): return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_INT_T, + inputs_dtypes=utils.FP_BF16_INT_T, supports_resize=True, supports_highdim=True, ) @@ -291,7 +291,7 @@ def _check_div_rounding_mode_is_floor(node: torch.fx.Node) -> bool: def register_div_tensor_mode(): return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_INT_T, + inputs_dtypes=utils.FP_BF16_INT_T, supports_resize=True, supports_highdim=True, are_node_inputs_supported_fn=_check_div_rounding_mode_is_floor, @@ -374,7 +374,7 @@ def is_scalar_value_supported(value: Any, dtype: torch.dtype) -> bool: def register_binary_scalar_ops(): return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_T, + inputs_dtypes=utils.FP_BF16_T, supports_resize=True, supports_highdim=True, are_node_inputs_supported_fn=lambda node: is_scalar_value_supported( @@ -402,6 +402,17 @@ def register_eq_scalar(): # ============================================================================= +def _check_bfloat16_conversion(node: torch.fx.Node) -> bool: + """bfloat16 converts to and from float32, and from integers.""" + in_dtype = node.args[0].meta["val"].dtype # type: ignore[union-attr] + out_dtype = node.meta["val"].dtype + if in_dtype == torch.bfloat16: + return out_dtype in {torch.bfloat16, torch.float32} + if out_dtype == torch.bfloat16: + return in_dtype in {torch.float32, torch.int32, torch.int64} + return True + + @update_features(exir_ops.edge.aten._to_copy.default) def register_to_copy(): def pick_to_copy_storage( @@ -416,10 +427,11 @@ def pick_to_copy_storage( return OpFeatures( inputs_storage=utils.ANY_STORAGE, - inputs_dtypes=utils.FP_INT_BOOL_T, - outputs_dtypes=utils.FP_INT_BOOL_T, + inputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16}, + outputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16}, supports_resize=True, pick_io_storage_fn=pick_to_copy_storage, + are_node_inputs_supported_fn=_check_bfloat16_conversion, ) @@ -1332,9 +1344,10 @@ def register_view_copy(): def register_to_dim_order_copy(): return OpFeatures( inputs_storage=utils.ANY_BUFFER, - inputs_dtypes=utils.FP_INT_BOOL_T, + inputs_dtypes=utils.FP_INT_BOOL_T | {torch.bfloat16}, supports_resize=True, supports_highdim=True, + are_node_inputs_supported_fn=_check_bfloat16_conversion, ) diff --git a/backends/vulkan/partitioner/vulkan_partitioner.py b/backends/vulkan/partitioner/vulkan_partitioner.py index 3fe5070aded..bbb468ccd7a 100644 --- a/backends/vulkan/partitioner/vulkan_partitioner.py +++ b/backends/vulkan/partitioner/vulkan_partitioner.py @@ -67,6 +67,15 @@ logger.setLevel(logging.INFO) +def _tensor_dtypes(nodes: Iterable[torch.fx.Node]) -> Set[torch.dtype]: + return { + value.dtype + for node in nodes + for value in tree_leaves(node.meta.get("val")) + if isinstance(value, torch.Tensor) + } + + class VulkanSupportedOperators(OperatorSupportBase): def __init__( self, @@ -106,22 +115,26 @@ def __init__( ) # Create a set of all nodes that are part of fusable subgraphs for quick lookup self.fusable_nodes: Set[torch.fx.Node] = set() - self.unsupported_fusable_nodes: Set[torch.fx.Node] = set() + # Maps nodes of fusable patterns that must stay outside Vulkan to the reason. + self.unsupported_fusable_nodes: Dict[torch.fx.Node, str] = {} for match in self.fusable_subgraphs: nodes = { node for node in match.all_nodes if isinstance(node, torch.fx.Node) } self.fusable_nodes.update(nodes) inputs = {arg for node in nodes for arg in node.all_input_nodes} - if not downcast_64_bit and any( - isinstance(value, torch.Tensor) - and value.dtype in (torch.int64, torch.float64) - for node in (nodes | inputs) - (constant_nodes or set()) - for value in tree_leaves(node.meta.get("val")) - ): - # Keep the whole pattern outside Vulkan instead of splitting a fusion. + + # Keep the whole pattern outside Vulkan instead of splitting a fusion. + reason = None + if not downcast_64_bit and _tensor_dtypes( + (nodes | inputs) - (constant_nodes or set()) + ) & {torch.int64, torch.float64}: # Constant quantization parameters may disappear during fusion. - self.unsupported_fusable_nodes.update(nodes) + reason = "fusable pattern requires 64-bit tensor downcasting" + elif torch.bfloat16 in _tensor_dtypes(nodes | inputs): + reason = "fused operators do not support bfloat16" + if reason is not None: + self.unsupported_fusable_nodes.update(dict.fromkeys(nodes, reason)) self.nn_module_blocklist = nn_module_blocklist self.nn_module_allowlist = nn_module_allowlist @@ -251,7 +264,7 @@ def _is_node_supported(self, node: torch.fx.Node) -> bool: def _check_node_support(self, node: torch.fx.Node) -> bool: # noqa: C901 if node in self.unsupported_fusable_nodes: - self.log_skip(node, "fusable pattern requires 64-bit tensor downcasting") + self.log_skip(node, self.unsupported_fusable_nodes[node]) return False # Asserts and the symbolic comparisons feeding them are removed during @@ -302,7 +315,7 @@ def _check_node_support(self, node: torch.fx.Node) -> bool: # noqa: C901 if not self.downcast_64_bit: native_dtypes = utils.DtypeSetList( - utils.ALL_T - {torch.int64, torch.float64} + utils.VULKAN_T - {torch.int64, torch.float64} ) dtype_valid, dtype_reason = utils.check_node_dtypes( node, native_dtypes, native_dtypes diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml index 40ce4602a60..8e1ee187bc0 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/view_buffer.yaml @@ -12,6 +12,7 @@ view_buffer: generate_variant_forall: DTYPE: - VALUE: half + - VALUE: bfloat16 - VALUE: float - VALUE: double - VALUE: int8 diff --git a/backends/vulkan/runtime/graph/ops/glsl/view_convert_buffer.yaml b/backends/vulkan/runtime/graph/ops/glsl/view_convert_buffer.yaml index a770d986fd5..2dfcd40d894 100644 --- a/backends/vulkan/runtime/graph/ops/glsl/view_convert_buffer.yaml +++ b/backends/vulkan/runtime/graph/ops/glsl/view_convert_buffer.yaml @@ -23,5 +23,8 @@ view_convert_buffer: - parameter_values: [float, half] - parameter_values: [half, float] - parameter_values: [half, int32] + - parameter_values: [int32, bfloat16] + - parameter_values: [float, bfloat16] + - parameter_values: [bfloat16, float] shader_variants: - NAME: view_convert_buffer diff --git a/backends/vulkan/runtime/graph/ops/impl/ToCopy.cpp b/backends/vulkan/runtime/graph/ops/impl/ToCopy.cpp index 74ddba85fbf..ee0afce8f6c 100644 --- a/backends/vulkan/runtime/graph/ops/impl/ToCopy.cpp +++ b/backends/vulkan/runtime/graph/ops/impl/ToCopy.cpp @@ -33,13 +33,21 @@ void add_to_copy_node(ComputeGraph& graph, ValueRef in, ValueRef out) { const vkapi::ScalarType in_dtype = graph.dtype_of(in); const vkapi::ScalarType out_dtype = graph.dtype_of(out); - // Same-dtype or float<->half conversions can use BlitNode - if (in_dtype == out_dtype || - (is_float_type(in_dtype) && is_float_type(out_dtype))) { + // Same-dtype or float<->half conversions of textures can use BlitNode + if (!graph.is_buffer_storage(out) && + (in_dtype == out_dtype || + (is_float_type(in_dtype) && is_float_type(out_dtype)))) { graph.execute_nodes().emplace_back(new BlitNode(graph, in, out)); return; } + // Buffers cannot be blitted. On devices that store bfloat16 as fp32, bfloat16 + // casts copy between buffers of one dtype. + if (in_dtype == out_dtype) { + add_view_copy_node(graph, in, out, {}, resize_to_copy_op_node); + return; + } + // Other conversions (e.g. int<->float) use view_convert shaders add_view_copy_convert_node(graph, in, out, {}, resize_to_copy_op_node); } diff --git a/backends/vulkan/test/test_vulkan_delegate.py b/backends/vulkan/test/test_vulkan_delegate.py index e2517cf8fc0..727c5c748f6 100644 --- a/backends/vulkan/test/test_vulkan_delegate.py +++ b/backends/vulkan/test/test_vulkan_delegate.py @@ -73,13 +73,14 @@ def lower_module( sample_inputs: Tuple[torch.Tensor], dynamic_shapes=None, compile_options=None, + skip_dim_order: bool = False, ) -> EdgeProgramManager: compile_options = dict(compile_options or {}) if dynamic_shapes is not None: compile_options["require_dynamic_shapes"] = True edge_compile_config = EdgeCompileConfig( - _skip_dim_order=False, # TODO(T182928844): Delegate dim order op to backend. + _skip_dim_order=skip_dim_order, # TODO(T182928844): Delegate dim order op to backend. ) program: ExportedProgram = export( @@ -349,6 +350,80 @@ def forward(self, x, y): self.lower_module_and_test_output(add_int_module, sample_inputs) + def test_vulkan_backend_bfloat16_elementwise(self): + class Bfloat16Module(torch.nn.Module): + def __init__(self): + super().__init__() + self.weight = torch.rand(size=(3, 17), dtype=torch.bfloat16) + + def forward(self, x, y): + z = torch.sigmoid(x * self.weight) - torch.tanh(x / (y.abs() + 1)) + z = torch.clamp(z, -0.75, 0.75) + torch.relu(y) * 2 + return torch.pow(z, 2.0) + + model = Bfloat16Module().eval() + sample_inputs = ( + torch.randn(size=(2, 3, 17), dtype=torch.bfloat16), + torch.randn(size=(2, 1, 17), dtype=torch.bfloat16), + ) + et_program = lower_module(model, sample_inputs).to_executorch() + self.check_vk_delegation(et_program) + self.assertEqual( + len(et_program.executorch_program.execution_plan[0].operators), 0 + ) + self.run_delegated_model_and_check_output( + et_program, model, sample_inputs, atol=2e-2, rtol=2e-2 + ) + + def test_vulkan_backend_bfloat16_casts(self): + class Bfloat16CastModule(torch.nn.Module): + def forward(self, x, i): + x = x.to(torch.bfloat16).to(torch.float32) + return x + i.to(torch.bfloat16).to(torch.float32) + + model = Bfloat16CastModule().eval() + sample_inputs = ( + torch.randn(size=(2, 3, 17), dtype=torch.float32), + torch.randint(-128, 128, size=(2, 3, 17), dtype=torch.int32), + ) + # Skipping dim order lowers the casts to _to_copy instead of + # _to_dim_order_copy. + for skip_dim_order in (False, True): + with self.subTest(skip_dim_order=skip_dim_order): + et_program = lower_module( + model, sample_inputs, skip_dim_order=skip_dim_order + ).to_executorch() + self.check_vk_delegation(et_program) + self.assertEqual( + len(et_program.executorch_program.execution_plan[0].operators), 0 + ) + self.run_delegated_model_and_check_output( + et_program, model, sample_inputs, atol=2e-2, rtol=2e-2 + ) + + def test_vulkan_backend_bfloat16_quantized_linear_not_fused(self): + # Fused operators have no bfloat16 shaders, so the pattern stays on the CPU. + class QuantizedLinearModule(torch.nn.Module): + def __init__(self): + super().__init__() + self.linear = torch.nn.Linear(64, 32, bias=False, dtype=torch.bfloat16) + + def forward(self, x): + return self.linear(x) + + model = QuantizedLinearModule().eval() + quantize_( + model, + IntxWeightOnlyConfig(weight_dtype=torch.int4, granularity=PerGroup(32)), + ) + unwrap_tensor_subclass(model) + sample_inputs = (torch.randn(size=(1, 64), dtype=torch.bfloat16),) + edge_program = lower_module(model, sample_inputs) + self.assertNotIn( + torch.ops.higher_order.executorch_call_delegate, + [node.target for node in edge_program.exported_program().graph.nodes], + ) + def test_vulkan_backend_zero_dim_tensor(self): class ZeroDimModule(torch.nn.Module): def __init__(self): diff --git a/backends/vulkan/test/vulkan_compute_api_test.cpp b/backends/vulkan/test/vulkan_compute_api_test.cpp index 0ea56829672..18fd53797c2 100644 --- a/backends/vulkan/test/vulkan_compute_api_test.cpp +++ b/backends/vulkan/test/vulkan_compute_api_test.cpp @@ -3986,7 +3986,7 @@ TEST(VulkanComputeGraphTest, buffer_copy_convert_preserves_order_after_resize) { {utils::kWidthPacked, utils::kHeightPacked, utils::kChannelsPacked}) { for (const auto out_layout : {utils::kWidthPacked, utils::kHeightPacked, utils::kChannelsPacked}) { - for (const auto dtype : {vkapi::kFloat, vkapi::kHalf}) { + for (const auto dtype : {vkapi::kFloat, vkapi::kHalf, vkapi::kBFloat16}) { SCOPED_TRACE( ::testing::Message() << "in_layout=" << in_layout << " out_layout=" << out_layout << " dtype=" << dtype); @@ -4019,6 +4019,13 @@ TEST(VulkanComputeGraphTest, buffer_copy_convert_preserves_order_after_resize) { for (size_t i = 0; i < data.size(); ++i) { EXPECT_EQ(float(output[i]), data[i]); } + } else if (dtype == vkapi::kBFloat16) { + std::vector output(data.size()); + graph.maybe_cast_and_copy_from_staging( + staging, output.data(), output.size(), dtype); + for (size_t i = 0; i < data.size(); ++i) { + EXPECT_EQ(float(output[i]), data[i]); + } } else { std::vector output(data.size()); graph.maybe_cast_and_copy_from_staging( diff --git a/backends/vulkan/utils.py b/backends/vulkan/utils.py index dd36ae329d6..d0ba41c33d8 100644 --- a/backends/vulkan/utils.py +++ b/backends/vulkan/utils.py @@ -51,6 +51,7 @@ torch.int32: VkDataType.INT32, torch.int64: VkDataType.INT64, torch.float16: VkDataType.FLOAT16, + torch.bfloat16: VkDataType.BFLOAT16, torch.float32: VkDataType.FLOAT32, torch.float64: VkDataType.FLOAT64, } @@ -65,12 +66,19 @@ INT_T: DtypeSet = {torch.int32, torch.int64} QINT8_T: DtypeSet = {torch.int8} BOOL_T: DtypeSet = {torch.bool} -ALL_T: DtypeSet = set(_VULKAN_DTYPES.keys()) +# Every serializable dtype. bfloat16 is opt-in per operator, so ALL_T, the +# default operator dtype set, excludes it. Devices without bfloat16 support store +# and compute bfloat16 tensors in fp32 without rounding to bfloat16, so results +# can differ from eager bfloat16. +VULKAN_T: DtypeSet = set(_VULKAN_DTYPES.keys()) +ALL_T: DtypeSet = VULKAN_T - {torch.bfloat16} NONE_T: DtypeSet = set() # Marker for non-tensor args (skip validation) # Composite dtype sets for specific operator requirements FP_INT_T: DtypeSet = FP_T | INT_T FP_INT_BOOL_T: DtypeSet = FP_T | INT_T | BOOL_T +FP_BF16_T: DtypeSet = FP_T | {torch.bfloat16} +FP_BF16_INT_T: DtypeSet = FP_INT_T | {torch.bfloat16} class DtypeSetList: @@ -1267,8 +1275,9 @@ def filter_invalid_reprs( if extents_are_valid(extents, texture_limits): valid_texture_layouts.add(memory_layout) - # High dimensional tensors require buffer storage - if len(tensor_val.shape) > 4: + # High dimensional tensors require buffer storage, as do bfloat16 tensors, + # since there is no bfloat16 image format. + if len(tensor_val.shape) > 4 or tensor_val.dtype == torch.bfloat16: return TensorRepSet(tensor_repset.valid_buffer_layouts, set()) return TensorRepSet(tensor_repset.valid_buffer_layouts, valid_texture_layouts)