Skip to content

Commit ea0f88f

Browse files
authored
Add opt-in Espressif operator-library export pipeline (#24309)
### Summary The Espressif exporter currently lowers supported regions through the ESP-NN delegate. Add an opt-in `use_operator_library=True` route to `to_espnn_edge()` that uses `EspPassManager` and the existing `EspNNQuantizer`. This first step retains executable portable operators and Q/DQ, with normal Edge validation, direct-operator reporting, and strict fallback handling. The delegate remains the default for existing callers. Add the frontend `EspTester` and tests for both export entry points, multiple methods, constants, shared float consumers, fallback, and serialization. Multiply lowering (#24310) and runtime execution (#24311) follow in this stack. Authored with AI assistance using Codex. ### Test plan The isolated frontend change passed all 67 Espressif tests, including 11 new frontend tests. After rebasing the complete stack onto main, all 146 Python tests and all 11 optimized ESP32-S3 implementation cases passed. Scoped lint and whitespace checks also passed. ```bash python -m pytest -q backends/espressif/test --disable-warnings ```
1 parent 77b687d commit ea0f88f

5 files changed

Lines changed: 319 additions & 2 deletions

File tree

‎backends/espressif/lowering.py‎

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,78 @@
88

99
from __future__ import annotations
1010

11+
import operator
12+
1113
import executorch.kernels.quantized # noqa: F401 registers quantize/dequantize out variants
1214

1315
from executorch.backends.espressif.partition.espnn_partitioner import EspNNPartitioner
14-
from executorch.exir import EdgeCompileConfig, to_edge_transform_and_lower
16+
from executorch.backends.espressif.passes import EspPassManager
17+
from executorch.exir import (
18+
EdgeCompileConfig,
19+
EdgeProgramManager,
20+
to_edge_transform_and_lower,
21+
)
1522
from executorch.exir.backend.canonical_partitioners.duplicate_dequant_node_pass import (
1623
DuplicateDequantNodePass,
1724
)
25+
from executorch.exir.dialects.edge._ops import EdgeOpOverload
26+
from torch._ops import OpOverload
27+
1828

29+
def _operator_library_report(edge: EdgeProgramManager) -> dict:
30+
esp_nn_ops = 0
31+
cpu_ops = []
32+
for method in sorted(edge.methods):
33+
for node in edge.exported_program(method).graph.nodes:
34+
if node.op != "call_function" or node.target == operator.getitem:
35+
continue
36+
target = node.target
37+
if isinstance(target, (EdgeOpOverload, OpOverload)):
38+
if target._schema.name in {
39+
"quantized_decomposed::quantize_per_tensor",
40+
"quantized_decomposed::dequantize_per_tensor",
41+
"quantized_decomposed::quantize_per_channel",
42+
"quantized_decomposed::dequantize_per_channel",
43+
}:
44+
continue
45+
if target.namespace == "esp_nn":
46+
esp_nn_ops += 1
47+
continue
48+
target = target.name()
49+
cpu_ops.append({"method": method, "name": node.name, "target": str(target)})
50+
return {"esp_nn_ops": esp_nn_ops, "cpu_ops": cpu_ops}
1951

20-
def to_espnn_edge(program, allow_cpu_fallback: bool = True):
52+
53+
def to_espnn_edge(
54+
program, allow_cpu_fallback: bool = True, *, use_operator_library: bool = False
55+
):
2156
"""
2257
Lower a PT2E-quantized torch.export program for the ESP32 ExecuTorch
2358
runtime: int8 regions become ESP-NN delegates, anything else stays in
2459
the program for the portable CPU kernels. Returns (edge_manager, report).
60+
61+
With use_operator_library=True, run EspPassManager without partitioning.
62+
This route currently preserves portable operators and executable Q/DQ.
63+
Its report counts esp_nn_ops and lists remaining cpu_ops by method,
64+
excluding Q/DQ and structural getitem nodes.
2565
"""
66+
if use_operator_library:
67+
edge = to_edge_transform_and_lower(
68+
program,
69+
transform_passes=EspPassManager(),
70+
compile_config=EdgeCompileConfig(),
71+
)
72+
report = _operator_library_report(edge)
73+
if not allow_cpu_fallback and report["cpu_ops"]:
74+
raise RuntimeError(
75+
"The ESP-NN operator library left these operators on the CPU: "
76+
+ ", ".join(
77+
f"{op['method']}.{op['name']} ({op['target']})"
78+
for op in report["cpu_ops"]
79+
)
80+
)
81+
return edge, report
82+
2683
partitioner = EspNNPartitioner(allow_cpu_fallback=allow_cpu_fallback)
2784
edge = to_edge_transform_and_lower(
2885
program,
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from .esp_pass_manager import EspPassManager
8+
9+
__all__ = ["EspPassManager"]
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from executorch.exir.pass_manager import ExportedProgramPassManager
8+
9+
10+
class EspPassManager(ExportedProgramPassManager):
11+
"""Run Espressif operator-library transformations on an Edge program."""
12+
13+
def __init__(self) -> None:
14+
super().__init__(passes=[])
Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import pytest
8+
import torch
9+
from executorch.backends.espressif import to_espnn_edge
10+
from executorch.backends.espressif.passes import EspPassManager
11+
from executorch.backends.espressif.test.tester import EspRunPasses, EspTester
12+
from executorch.exir import to_edge, to_edge_transform_and_lower
13+
from executorch.exir._serialize._program import deserialize_pte_binary
14+
from executorch.exir.dialects._ops import ops as exir_ops
15+
16+
17+
class Multiply(torch.nn.Module):
18+
def forward(self, x, y):
19+
return x * y
20+
21+
22+
class MultiplyAndFloatUsers(torch.nn.Module):
23+
def forward(self, x, y):
24+
product = x * y
25+
return product, torch.sin(x), torch.sin(product)
26+
27+
28+
@pytest.mark.parametrize("entry_point", ["separate", "combined"])
29+
@pytest.mark.parametrize("broadcast", [False, True])
30+
def test_quantized_and_float_operations_survive_export(entry_point, broadcast):
31+
inputs = (
32+
torch.linspace(-1, 1, 6).reshape(2, 3),
33+
torch.linspace(-2, 2, 3 if broadcast else 6).reshape(
34+
(3,) if broadcast else (2, 3)
35+
),
36+
)
37+
tester = EspTester(MultiplyAndFloatUsers(), inputs).quantize().export()
38+
if entry_point == "combined":
39+
tester.to_edge_transform_and_lower()
40+
else:
41+
tester.to_edge().run_passes()
42+
tester.check_node_count(
43+
{
44+
exir_ops.edge.aten.mul.Tensor: 1,
45+
exir_ops.edge.aten.sin.default: 2,
46+
exir_ops.edge.quantized_decomposed.quantize_per_tensor.default: 3,
47+
exir_ops.edge.quantized_decomposed.dequantize_per_tensor.default: 3,
48+
}
49+
).run_method_and_compare_outputs(inputs=inputs, atol=0, rtol=0)
50+
edge = tester.get_artifact()
51+
assert edge.compile_config._check_ir_validity
52+
edge.exported_program().validate()
53+
tester.to_executorch().serialize()
54+
program = deserialize_pte_binary(tester.get_artifact()).program
55+
assert program.execution_plan[0].name == "forward"
56+
assert not program.execution_plan[0].delegates
57+
assert {op.name for op in program.execution_plan[0].operators} >= {
58+
"aten::mul",
59+
"aten::sin",
60+
"quantized_decomposed::quantize_per_tensor",
61+
"quantized_decomposed::dequantize_per_tensor",
62+
}
63+
64+
65+
def test_public_entry_point_keeps_delegate_default():
66+
inputs = (torch.linspace(-1, 1, 6), torch.linspace(-2, 2, 6))
67+
program = EspTester(Multiply(), inputs).quantize().export().get_artifact()
68+
69+
delegated, delegate_report = to_espnn_edge(program, False)
70+
assert delegate_report["partitions"] == 1
71+
assert delegate_report["delegated_ops"] == 1
72+
assert not delegate_report["cpu_ops"]
73+
assert delegated.to_executorch().executorch_program.execution_plan[0].delegates
74+
75+
edge, report = to_espnn_edge(program, use_operator_library=True)
76+
assert report["esp_nn_ops"] == 0
77+
assert [(op["method"], op["target"]) for op in report["cpu_ops"]] == [
78+
("forward", "aten::mul.Tensor")
79+
]
80+
assert edge.compile_config._check_ir_validity
81+
torch.testing.assert_close(
82+
edge.exported_program().module()(*inputs),
83+
program.module()(*inputs),
84+
atol=0,
85+
rtol=0,
86+
)
87+
serialized = deserialize_pte_binary(edge.to_executorch().buffer).program
88+
assert not serialized.execution_plan[0].delegates
89+
90+
91+
def test_operator_library_strict_mode_rejects_remaining_compute():
92+
inputs = (torch.linspace(-1, 1, 6), torch.linspace(-2, 2, 6))
93+
program = EspTester(Multiply(), inputs).quantize().export().get_artifact()
94+
with pytest.raises(RuntimeError, match=r"forward\.\w+ \(aten::mul\.Tensor\)"):
95+
to_espnn_edge(program, False, use_operator_library=True)
96+
97+
98+
def test_operator_library_strict_mode_accepts_no_compute():
99+
program = torch.export.export(torch.nn.Identity(), (torch.ones(3),))
100+
_, report = to_espnn_edge(program, False, use_operator_library=True)
101+
assert report == {"esp_nn_ops": 0, "cpu_ops": []}
102+
103+
104+
def test_operator_library_uses_standard_edge_decompositions():
105+
inputs = (torch.linspace(-1, 1, 6).reshape(2, 3),)
106+
program = (
107+
EspTester(torch.nn.Linear(3, 2), inputs).quantize().export().get_artifact()
108+
)
109+
edge, report = to_espnn_edge(program, use_operator_library=True)
110+
targets = {op["target"] for op in report["cpu_ops"]}
111+
assert "aten::linear" not in targets
112+
assert "aten::addmm" in targets
113+
assert not any(target.startswith("quantized_decomposed::") for target in targets)
114+
torch.testing.assert_close(
115+
edge.exported_program().module()(*inputs), program.module()(*inputs)
116+
)
117+
serialized = deserialize_pte_binary(edge.to_executorch().buffer).program
118+
assert not serialized.execution_plan[0].delegates
119+
120+
121+
@pytest.mark.parametrize("entry_point", ["separate", "combined"])
122+
def test_passes_preserve_methods_and_constants(entry_point):
123+
class Scale(torch.nn.Module):
124+
def __init__(self, scale, offset):
125+
super().__init__()
126+
self.register_buffer("scale", scale)
127+
self.offset = offset
128+
129+
def forward(self, x):
130+
return x * self.scale + self.offset
131+
132+
inputs = (torch.tensor([-1.0, 2.0, 3.0]),)
133+
models = {
134+
"forward": Scale(torch.tensor([2.0, 3.0, 4.0]), torch.tensor([0.5, 1.0, 1.5])),
135+
"other": Scale(torch.tensor([5.0, 6.0, 7.0]), torch.tensor([1.5, 2.0, 2.5])),
136+
}
137+
programs = {
138+
name: torch.export.export(model, inputs) for name, model in models.items()
139+
}
140+
if entry_point == "combined":
141+
transformed = to_edge_transform_and_lower(
142+
programs, transform_passes=EspPassManager(), constant_methods={"version": 1}
143+
)
144+
else:
145+
stage = EspRunPasses()
146+
stage.run(to_edge(programs, constant_methods={"version": 1}))
147+
transformed = stage.artifact
148+
assert transformed.methods == {"forward", "other"}
149+
assert transformed.config_methods == {"version"}
150+
for name, model in models.items():
151+
program = transformed.exported_program(name)
152+
program.validate()
153+
torch.testing.assert_close(
154+
program.module()(*inputs), model(*inputs), atol=0, rtol=0
155+
)
156+
serialized = deserialize_pte_binary(transformed.to_executorch().buffer).program
157+
assert {plan.name for plan in serialized.execution_plan} == {
158+
"forward",
159+
"other",
160+
"version",
161+
}
162+
assert all(not plan.delegates for plan in serialized.execution_plan)
163+
164+
165+
def test_operator_library_reports_all_methods():
166+
inputs = (torch.linspace(-1, 1, 6), torch.linspace(-2, 2, 6))
167+
programs = {
168+
name: EspTester(Multiply(), inputs).quantize().export().get_artifact()
169+
for name in ("forward", "other")
170+
}
171+
edge, report = to_espnn_edge(programs, use_operator_library=True)
172+
assert report["esp_nn_ops"] == 0
173+
assert [(op["method"], op["target"]) for op in report["cpu_ops"]] == [
174+
(name, "aten::mul.Tensor") for name in ("forward", "other")
175+
]
176+
for name, program in programs.items():
177+
torch.testing.assert_close(
178+
edge.exported_program(name).module()(*inputs),
179+
program.module()(*inputs),
180+
atol=0,
181+
rtol=0,
182+
)
183+
serialized = deserialize_pte_binary(edge.to_executorch().buffer).program
184+
assert {plan.name for plan in serialized.execution_plan} == {"forward", "other"}
185+
assert all(not plan.delegates for plan in serialized.execution_plan)

‎backends/espressif/test/tester.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
from executorch.backends.espressif.passes import EspPassManager
8+
from executorch.backends.espressif.quantizer import EspNNQuantizer
9+
from executorch.backends.test.harness import Tester
10+
from executorch.backends.test.harness.stages import (
11+
Quantize,
12+
RunPasses,
13+
StageType,
14+
ToEdgeTransformAndLower,
15+
)
16+
from executorch.exir import (
17+
EdgeCompileConfig,
18+
EdgeProgramManager,
19+
to_edge_transform_and_lower,
20+
)
21+
22+
23+
class EspRunPasses(RunPasses):
24+
def __init__(self):
25+
super().__init__(EspPassManager)
26+
27+
def run(self, artifact: EdgeProgramManager, inputs=None) -> None:
28+
self.edge_or_aten_program = artifact.transform(EspPassManager())
29+
30+
31+
class EspToEdgeTransformAndLower(ToEdgeTransformAndLower):
32+
def __init__(self):
33+
super().__init__(edge_compile_config=EdgeCompileConfig())
34+
35+
def run(self, artifact, inputs=None, generate_etrecord: bool = False) -> None:
36+
self.edge_dialect_program = to_edge_transform_and_lower(
37+
artifact,
38+
compile_config=self.edge_compile_conf,
39+
transform_passes=EspPassManager(),
40+
generate_etrecord=generate_etrecord,
41+
)
42+
43+
44+
class EspTester(Tester):
45+
def __init__(self, module, example_inputs, calibration_samples=None):
46+
stages = self.default_stage_classes()
47+
stages[StageType.QUANTIZE] = lambda: Quantize(
48+
EspNNQuantizer(), calibration_samples=calibration_samples
49+
)
50+
stages[StageType.RUN_PASSES] = EspRunPasses
51+
stages[StageType.TO_EDGE_TRANSFORM_AND_LOWER] = EspToEdgeTransformAndLower
52+
super().__init__(module, example_inputs, stages)

0 commit comments

Comments
 (0)