Skip to content

Commit 724da85

Browse files
Arm backend: Reland tensorized GPT-OSS MoE regression tests (#24297)
### Summary Reland #24149, reverted in #24242 because the tests fail with Transformers 4.57.1. In 4.x, GptOssTopKRouter passes dense routing weights for all experts instead of weights for the selected experts, and builds them with scatter. Gather the selected routing weights when the experts receive dense weights, so the reference check passes on 4.x and 5.x. Xfail the TOSA FP delegation test for Transformers < 5, since aten.scatter.src is not delegated. Authored with assistance from Claude. ### Test plan Ran test_moe.py with Transformers 4.57.1: 16 passed, 2 xfailed (test_tensorized_gpt_oss_moe_tosa_FP). With Transformers 5.0.0rc1, the tensorized MLP matches the reference in FP32 and FP16 and exports without scatter. cc @digantdesai @freddan80 @per @zingo @oscarandersson8218 @mansnils @Sebastian-Larsson @robell @rascani Co-authored-by: Yufeng Shi <yufeng.shi@arm.com>
1 parent 84c701b commit 724da85

2 files changed

Lines changed: 128 additions & 1 deletion

File tree

‎backends/arm/test/modules/test_moe.py‎

Lines changed: 126 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@
44
# LICENSE file in the root directory of this source tree.
55
"""Test fixed-top-k MoE lowering with finite, distinct router scores.
66
7-
Non-finite and equal router scores are covered by the TopK operator tests.
7+
Equal router scores are covered by the TopK operator tests. Non-finite router
8+
scores are outside the supported domain.
89
910
"""
1011

@@ -16,6 +17,9 @@
1617
)
1718
from executorch.examples.models.llama.llama_transformer import MOEFeedForward
1819
from executorch.examples.models.llama.model_args import ModelArgs
20+
from transformers import GptOssConfig
21+
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts, GptOssMLP
22+
from typing_extensions import override
1923

2024
input_t = tuple[torch.Tensor]
2125

@@ -102,3 +106,124 @@ def test_moe_vgf_no_quant(test_data: tuple[torch.Tensor, int, int]) -> None:
102106
quantize=False,
103107
)
104108
pipeline.run()
109+
110+
111+
# GPT-OSS expects [batch, sequence, hidden] inputs.
112+
gpt_oss_test_data = {
113+
"fp32": (three_tokens.unsqueeze(0),),
114+
"fp16": (three_tokens.unsqueeze(0).half(),),
115+
}
116+
117+
118+
class TensorizedGptOssExperts(GptOssExperts):
119+
"""Provide tensorized experts for the native GPT-OSS MLP.
120+
121+
Used with GptOssMLP from Transformers 4.57.1, 5.0.0rc1 and 5.3.0::
122+
123+
GptOssMLP
124+
|-- router: GptOssTopKRouter
125+
`-- experts: TensorizedGptOssExperts
126+
127+
Inherits expert parameters from GptOssExperts and reproduces its clipped
128+
activation. Only forward() is rewritten as gather/BMM/reduction.
129+
Accepts either 2D or 3D expert inputs and returns outputs with the same shape.
130+
Accepts routing weights for the selected experts or, as passed by
131+
Transformers 4.x, dense routing weights for all experts.
132+
133+
"""
134+
135+
@override
136+
def forward(
137+
self, hidden_states: torch.Tensor, router_indices=None, routing_weights=None
138+
) -> torch.Tensor:
139+
tokens, k = router_indices.shape
140+
if routing_weights.shape[-1] != k:
141+
routing_weights = routing_weights.gather(1, router_indices)
142+
dim = self.hidden_size
143+
intermediate = self.intermediate_size
144+
input_shape = hidden_states.shape
145+
hidden_states = hidden_states.reshape(-1, dim)
146+
selected_up = self.gate_up_proj[router_indices]
147+
selected_down = self.down_proj[router_indices]
148+
expanded_x = hidden_states[:, None, :].expand(tokens, k, dim)
149+
gate_up = torch.bmm(
150+
expanded_x.reshape(tokens * k, 1, dim),
151+
selected_up.reshape(tokens * k, dim, 2 * intermediate),
152+
).reshape(tokens, k, 2 * intermediate)
153+
gate_up = gate_up + self.gate_up_proj_bias[router_indices]
154+
gate, up = gate_up[..., ::2], gate_up[..., 1::2]
155+
gate = gate.clamp(max=self.limit)
156+
up = up.clamp(min=-self.limit, max=self.limit)
157+
glu = gate * torch.sigmoid(gate * self.alpha)
158+
hidden = (up + 1) * glu
159+
outputs = torch.bmm(
160+
hidden.reshape(tokens * k, 1, intermediate),
161+
selected_down.reshape(tokens * k, intermediate, dim),
162+
).reshape(tokens, k, dim)
163+
outputs = outputs + self.down_proj_bias[router_indices]
164+
outputs = (outputs * routing_weights.unsqueeze(-1)).sum(dim=1)
165+
return outputs.reshape(input_shape)
166+
167+
168+
def _make_gpt_oss_moe(dtype: torch.dtype) -> tuple[GptOssMLP, GptOssConfig]:
169+
config = GptOssConfig(
170+
hidden_size=8,
171+
intermediate_size=16,
172+
num_local_experts=8,
173+
num_experts_per_tok=4,
174+
experts_implementation="eager",
175+
)
176+
model = GptOssMLP(config)
177+
model.experts = TensorizedGptOssExperts(config)
178+
generator = torch.Generator().manual_seed(2648)
179+
with torch.no_grad():
180+
for parameter in model.experts.parameters():
181+
parameter.uniform_(-0.25, 0.25, generator=generator)
182+
model.router.weight.copy_(torch.eye(model.router.weight.shape[1]))
183+
model.router.bias.copy_(torch.linspace(-0.07, 0.07, model.router.bias.numel()))
184+
# Exercise gate/up clipping and unclipped negative gates.
185+
model.experts.gate_up_proj_bias[:, :8].add_(
186+
torch.tensor([9.0, 0.5, 0.5, 9.0, 0.5, -9.0, -9.0, 0.5])
187+
)
188+
return model.eval().to(dtype), config
189+
190+
191+
@common.parametrize("test_data", gpt_oss_test_data)
192+
@torch.no_grad()
193+
def test_tensorized_gpt_oss_moe_matches_reference(test_data: input_t) -> None:
194+
(x,) = test_data
195+
model, config = _make_gpt_oss_moe(x.dtype)
196+
reference = GptOssMLP(config).eval().to(x.dtype)
197+
reference.load_state_dict(model.state_dict())
198+
torch.testing.assert_close(model(x), reference(x))
199+
200+
201+
@common.parametrize("test_data", gpt_oss_test_data)
202+
@common.xfail_if_dependency_version(
203+
"transformers",
204+
"<5",
205+
module="transformers",
206+
reason=(
207+
"Transformers 4.x GptOssTopKRouter scatters dense router scores, and "
208+
"aten.scatter.src is not delegated."
209+
),
210+
raises=RuntimeError,
211+
)
212+
@torch.no_grad()
213+
def test_tensorized_gpt_oss_moe_tosa_FP(test_data: input_t) -> None:
214+
(x,) = test_data
215+
model, _ = _make_gpt_oss_moe(x.dtype)
216+
pipeline = TosaPipelineFP[input_t](
217+
model,
218+
test_data,
219+
[
220+
"torch.ops.aten.topk.default",
221+
"torch.ops.aten.index.Tensor",
222+
"torch.ops.aten.bmm.default",
223+
"torch.ops.aten.slice.Tensor",
224+
"torch.ops.aten.clamp.default",
225+
],
226+
"executorch_exir_dialects_edge__ops_",
227+
tosa_extensions=["INT"],
228+
)
229+
pipeline.run()

‎backends/arm/test/targets.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,8 @@ def define_arm_tests():
187187
] if test_file == "misc/test_docgen_op_support.py" else []) + ([
188188
"//executorch/examples/models/llama:llama_transformer",
189189
"//executorch/examples/models/llama:transformer_modules",
190+
"fbsource//third-party/pypi/transformers:transformers",
191+
"fbsource//third-party/pypi/typing-extensions:typing-extensions",
190192
] if test_file == "modules/test_moe.py" else []) + ([
191193
"fbsource//third-party/pypi/ethos-u-vela:ethos-u-vela",
192194
"fbsource//third-party/pypi/packaging:packaging",

0 commit comments

Comments
 (0)