|
15 | 15 | from executorch.backends.espressif.utils.quantization import quantize_multiplier |
16 | 16 | from executorch.backends.transforms.fused_quant.graph_utils import get_qparams_from_node |
17 | 17 | from executorch.backends.transforms.permute_pass_utils import get_arg |
18 | | -from executorch.backends.transforms.utils import get_constant |
| 18 | +from executorch.backends.transforms.utils import add_constant, get_constant |
19 | 19 | from executorch.exir.dialects._ops import ops as exir_ops |
20 | 20 | from executorch.exir.pass_base import ExportedProgramPassBase, ExportedProgramPassResult |
| 21 | +from executorch.exir.passes.constant_prop_pass import get_propagated_const_tensor_dict |
| 22 | +from torch.export.graph_signature import InputKind |
21 | 23 |
|
22 | 24 |
|
23 | 25 | @dataclass(frozen=True) |
@@ -272,15 +274,185 @@ def _lower_avg_pool2d(program, node) -> bool: |
272 | 274 | return True |
273 | 275 |
|
274 | 276 |
|
| 277 | +def _linear_multipliers(input_params, output_params, weight, scales, bias): |
| 278 | + rescales = [ |
| 279 | + quantize_multiplier(input_params.scale * scale / output_params.scale) |
| 280 | + for scale in scales.flatten().tolist() |
| 281 | + ] |
| 282 | + if any(shift < -30 for _, shift in rescales): |
| 283 | + raise ValueError("requantization shift is outside the ESP-NN range") |
| 284 | + max_input = max(abs(-128 - input_params.zp), abs(127 - input_params.zp)) |
| 285 | + limits = max_input * weight.to(torch.int64).abs().sum(dim=1) |
| 286 | + if bias is not None: |
| 287 | + limits += bias.to(torch.int64).abs() |
| 288 | + if any( |
| 289 | + int(limit) << max(shift, 0) |
| 290 | + > torch.iinfo(torch.int32).max - abs(output_params.zp) |
| 291 | + for limit, (_, shift) in zip(limits, rescales) |
| 292 | + ): |
| 293 | + raise ValueError("accumulation or requantization may overflow int32") |
| 294 | + max_float_input = c_float(max_input * c_float(input_params.scale).value).value |
| 295 | + float_weights = weight.to(torch.float32) * scales |
| 296 | + max_float_outputs = max_float_input * float_weights.abs().to(torch.float64).sum( |
| 297 | + dim=1 |
| 298 | + ) |
| 299 | + if bias is not None: |
| 300 | + bias_scales = input_params.scale * scales.flatten() |
| 301 | + max_float_outputs += ( |
| 302 | + (bias.to(torch.float32) * bias_scales).abs().to(torch.float64) |
| 303 | + ) |
| 304 | + # Bound float32 accumulation rounding, including the bias addition. |
| 305 | + max_float_outputs /= 1 - (weight.shape[1] + 1) * torch.finfo(torch.float32).eps |
| 306 | + if ( |
| 307 | + not math.isfinite(c_float(1.0 / output_params.scale).value) |
| 308 | + or not torch.all(torch.isfinite(max_float_outputs)) |
| 309 | + or torch.any(max_float_outputs > torch.finfo(torch.float32).max) |
| 310 | + ): |
| 311 | + raise ValueError("the float32 reference may overflow") |
| 312 | + multipliers, shifts = zip(*rescales) |
| 313 | + return torch.tensor(multipliers, dtype=torch.int32), torch.tensor( |
| 314 | + shifts, dtype=torch.int32 |
| 315 | + ) |
| 316 | + |
| 317 | + |
| 318 | +def _linear_kernel_args(program, node, constants): |
| 319 | + inp, weight_node, bias_node = ( |
| 320 | + get_arg(node, name) for name in ("inp", "weight", "bias") |
| 321 | + ) |
| 322 | + input_value, output_value = inp.meta.get("val"), node.meta.get("val") |
| 323 | + if not all( |
| 324 | + isinstance(value, torch.Tensor) |
| 325 | + and value.dtype == torch.int8 |
| 326 | + and value.dim() >= 1 |
| 327 | + and all(type(dim) is int for dim in value.shape) |
| 328 | + and 0 < value.numel() <= torch.iinfo(torch.int32).max |
| 329 | + and value.is_contiguous() |
| 330 | + for value in (input_value, output_value) |
| 331 | + ): |
| 332 | + raise ValueError( |
| 333 | + "input and output must be nonempty, static, contiguous int8 tensors" |
| 334 | + ) |
| 335 | + input_params, output_params = ( |
| 336 | + _qparams(program, node, name) for name in ("inp", "out") |
| 337 | + ) |
| 338 | + if input_params is None or output_params is None: |
| 339 | + raise ValueError("input and output require constant per-tensor int8 qparams") |
| 340 | + weight = constants.get(weight_node) |
| 341 | + if ( |
| 342 | + weight is None |
| 343 | + or weight.dtype != torch.int8 |
| 344 | + or weight.dim() != 2 |
| 345 | + or not weight.is_contiguous() |
| 346 | + or weight.numel() > torch.iinfo(torch.int32).max |
| 347 | + or any(not 0 < dim <= 65535 for dim in weight.shape) |
| 348 | + ): |
| 349 | + raise ValueError( |
| 350 | + "weight must be a constant contiguous int8 matrix with dimensions in [1, 65535]" |
| 351 | + ) |
| 352 | + if input_value.shape[-1] != weight.shape[1] or output_value.shape != ( |
| 353 | + *input_value.shape[:-1], |
| 354 | + weight.shape[0], |
| 355 | + ): |
| 356 | + raise ValueError("input, weight and output shapes do not match linear") |
| 357 | + weight_params = get_qparams_from_node(node, "weight") |
| 358 | + if weight_params is None or weight_params.dtype != torch.float32: |
| 359 | + raise ValueError( |
| 360 | + "weight requires float32 dequantization with per-output-channel scales" |
| 361 | + ) |
| 362 | + scales, zero_points = ( |
| 363 | + constants.get(arg) for arg in (weight_params.scale, weight_params.zero_point) |
| 364 | + ) |
| 365 | + if ( |
| 366 | + scales is None |
| 367 | + or scales.dtype != torch.float32 |
| 368 | + or scales.shape != (weight.shape[0], 1) |
| 369 | + or not torch.all(torch.isfinite(scales) & (scales > 0)) |
| 370 | + or zero_points is None |
| 371 | + or zero_points.shape != scales.shape |
| 372 | + or torch.any(zero_points != 0) |
| 373 | + ): |
| 374 | + raise ValueError( |
| 375 | + "weight requires constant per-output-channel scales and zero zero-points" |
| 376 | + ) |
| 377 | + bias = None |
| 378 | + if bias_node is not None: |
| 379 | + bias = constants.get(bias_node) |
| 380 | + bias_params = get_qparams_from_node(node, "bias") |
| 381 | + if ( |
| 382 | + bias is None |
| 383 | + or bias.dtype != torch.int32 |
| 384 | + or not bias.is_contiguous() |
| 385 | + or bias.shape != (weight.shape[0],) |
| 386 | + or bias_params is None |
| 387 | + or bias_params.dtype != torch.float32 |
| 388 | + ): |
| 389 | + raise ValueError( |
| 390 | + "bias must be a constant int32 vector with derived qparams" |
| 391 | + ) |
| 392 | + bias_scales, bias_zero_points = ( |
| 393 | + constants.get(arg) for arg in (bias_params.scale, bias_params.zero_point) |
| 394 | + ) |
| 395 | + if ( |
| 396 | + bias_scales is None |
| 397 | + or not torch.equal(bias_scales, input_params.scale * scales.flatten()) |
| 398 | + or bias_zero_points is None |
| 399 | + or bias_zero_points.shape != bias.shape |
| 400 | + or torch.any(bias_zero_points != 0) |
| 401 | + ): |
| 402 | + raise ValueError( |
| 403 | + "bias scales must equal input scale times weight scales, with zero zero-points" |
| 404 | + ) |
| 405 | + multipliers, shifts = _linear_multipliers( |
| 406 | + input_params, output_params, weight, scales, bias |
| 407 | + ) |
| 408 | + multiplier_node, shift_node = ( |
| 409 | + add_constant( |
| 410 | + program, f"{node.name}_{name}", tensor, node, InputKind.CONSTANT_TENSOR |
| 411 | + ) |
| 412 | + for name, tensor in (("multiplier", multipliers), ("shift", shifts)) |
| 413 | + ) |
| 414 | + return ( |
| 415 | + inp, |
| 416 | + weight_node, |
| 417 | + bias_node, |
| 418 | + multiplier_node, |
| 419 | + shift_node, |
| 420 | + -input_params.zp, |
| 421 | + output_params.zp, |
| 422 | + ) |
| 423 | + |
| 424 | + |
275 | 425 | class LowerToEsp(ExportedProgramPassBase): |
276 | 426 | """Lower supported fused-quant operators to ESP-NN.""" |
277 | 427 |
|
278 | 428 | def call(self, exported_program): |
279 | 429 | module = exported_program.graph_module |
280 | 430 | graph = module.graph |
| 431 | + linear_nodes = graph.find_nodes( |
| 432 | + op="call_function", target=exir_ops.edge.fused_quant.linear.default |
| 433 | + ) |
| 434 | + constants = ( |
| 435 | + get_propagated_const_tensor_dict(exported_program, None) |
| 436 | + if linear_nodes |
| 437 | + else {} |
| 438 | + ) |
281 | 439 |
|
282 | 440 | modified = False |
283 | 441 | for node in list(graph.nodes): |
| 442 | + if node.target == exir_ops.edge.fused_quant.linear.default: |
| 443 | + try: |
| 444 | + args = _linear_kernel_args(exported_program, node, constants) |
| 445 | + except ValueError: |
| 446 | + continue |
| 447 | + with graph.inserting_before(node): |
| 448 | + replacement = graph.call_function( |
| 449 | + exir_ops.edge.esp_nn.quantized_linear_int8.default, args |
| 450 | + ) |
| 451 | + replacement.meta = node.meta.copy() |
| 452 | + node.replace_all_uses_with(replacement) |
| 453 | + graph.erase_node(node) |
| 454 | + modified = True |
| 455 | + continue |
284 | 456 | if ( |
285 | 457 | node.target |
286 | 458 | == exir_ops.edge.fused_quant.avg_pool2d_channels_last.default |
|
0 commit comments