|
4 | 4 | # LICENSE file in the root directory of this source tree. |
5 | 5 | """Test fixed-top-k MoE lowering with finite, distinct router scores. |
6 | 6 |
|
7 | | -Equal router scores are covered by the TopK operator tests. Non-finite router |
8 | | -scores are outside the supported domain. |
| 7 | +Non-finite and equal router scores are covered by the TopK operator tests. |
9 | 8 |
|
10 | 9 | """ |
11 | 10 |
|
|
17 | 16 | ) |
18 | 17 | from executorch.examples.models.llama.llama_transformer import MOEFeedForward |
19 | 18 | from executorch.examples.models.llama.model_args import ModelArgs |
20 | | -from transformers import GptOssConfig |
21 | | -from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts, GptOssMLP |
22 | | -from typing_extensions import override |
23 | 19 |
|
24 | 20 | input_t = tuple[torch.Tensor] |
25 | 21 |
|
@@ -106,110 +102,3 @@ def test_moe_vgf_no_quant(test_data: tuple[torch.Tensor, int, int]) -> None: |
106 | 102 | quantize=False, |
107 | 103 | ) |
108 | 104 | pipeline.run() |
109 | | - |
110 | | - |
111 | | -# GPT-OSS expects [batch, sequence, hidden] inputs. |
112 | | -gpt_oss_test_data = { |
113 | | - "fp32": (three_tokens.unsqueeze(0),), |
114 | | - "fp16": (three_tokens.unsqueeze(0).half(),), |
115 | | -} |
116 | | - |
117 | | - |
118 | | -class TensorizedGptOssExperts(GptOssExperts): |
119 | | - """Provide tensorized experts for the native GPT-OSS MLP. |
120 | | -
|
121 | | - Used with GptOssMLP from Transformers 5.0.0rc1 and 5.3.0:: |
122 | | -
|
123 | | - GptOssMLP |
124 | | - |-- router: GptOssTopKRouter |
125 | | - `-- experts: TensorizedGptOssExperts |
126 | | -
|
127 | | - Inherits expert parameters from GptOssExperts and reproduces its clipped |
128 | | - activation. Only forward() is rewritten as gather/BMM/reduction. |
129 | | - Accepts either 2D or 3D expert inputs and returns outputs with the same shape. |
130 | | -
|
131 | | - """ |
132 | | - |
133 | | - @override |
134 | | - def forward( |
135 | | - self, hidden_states: torch.Tensor, router_indices=None, routing_weights=None |
136 | | - ) -> torch.Tensor: |
137 | | - tokens, k = router_indices.shape |
138 | | - dim = self.hidden_size |
139 | | - intermediate = self.intermediate_size |
140 | | - input_shape = hidden_states.shape |
141 | | - hidden_states = hidden_states.reshape(-1, dim) |
142 | | - selected_up = self.gate_up_proj[router_indices] |
143 | | - selected_down = self.down_proj[router_indices] |
144 | | - expanded_x = hidden_states[:, None, :].expand(tokens, k, dim) |
145 | | - gate_up = torch.bmm( |
146 | | - expanded_x.reshape(tokens * k, 1, dim), |
147 | | - selected_up.reshape(tokens * k, dim, 2 * intermediate), |
148 | | - ).reshape(tokens, k, 2 * intermediate) |
149 | | - gate_up = gate_up + self.gate_up_proj_bias[router_indices] |
150 | | - gate, up = gate_up[..., ::2], gate_up[..., 1::2] |
151 | | - gate = gate.clamp(max=self.limit) |
152 | | - up = up.clamp(min=-self.limit, max=self.limit) |
153 | | - glu = gate * torch.sigmoid(gate * self.alpha) |
154 | | - hidden = (up + 1) * glu |
155 | | - outputs = torch.bmm( |
156 | | - hidden.reshape(tokens * k, 1, intermediate), |
157 | | - selected_down.reshape(tokens * k, intermediate, dim), |
158 | | - ).reshape(tokens, k, dim) |
159 | | - outputs = outputs + self.down_proj_bias[router_indices] |
160 | | - outputs = (outputs * routing_weights.unsqueeze(-1)).sum(dim=1) |
161 | | - return outputs.reshape(input_shape) |
162 | | - |
163 | | - |
164 | | -def _make_gpt_oss_moe(dtype: torch.dtype) -> tuple[GptOssMLP, GptOssConfig]: |
165 | | - config = GptOssConfig( |
166 | | - hidden_size=8, |
167 | | - intermediate_size=16, |
168 | | - num_local_experts=8, |
169 | | - num_experts_per_tok=4, |
170 | | - experts_implementation="eager", |
171 | | - ) |
172 | | - model = GptOssMLP(config) |
173 | | - model.experts = TensorizedGptOssExperts(config) |
174 | | - generator = torch.Generator().manual_seed(2648) |
175 | | - with torch.no_grad(): |
176 | | - for parameter in model.experts.parameters(): |
177 | | - parameter.uniform_(-0.25, 0.25, generator=generator) |
178 | | - model.router.weight.copy_(torch.eye(model.router.weight.shape[1])) |
179 | | - model.router.bias.copy_(torch.linspace(-0.07, 0.07, model.router.bias.numel())) |
180 | | - # Exercise gate/up clipping and unclipped negative gates. |
181 | | - model.experts.gate_up_proj_bias[:, :8].add_( |
182 | | - torch.tensor([9.0, 0.5, 0.5, 9.0, 0.5, -9.0, -9.0, 0.5]) |
183 | | - ) |
184 | | - return model.eval().to(dtype), config |
185 | | - |
186 | | - |
187 | | -@common.parametrize("test_data", gpt_oss_test_data) |
188 | | -@torch.no_grad() |
189 | | -def test_tensorized_gpt_oss_moe_matches_reference(test_data: input_t) -> None: |
190 | | - (x,) = test_data |
191 | | - model, config = _make_gpt_oss_moe(x.dtype) |
192 | | - reference = GptOssMLP(config).eval().to(x.dtype) |
193 | | - reference.load_state_dict(model.state_dict()) |
194 | | - torch.testing.assert_close(model(x), reference(x)) |
195 | | - |
196 | | - |
197 | | -@common.parametrize("test_data", gpt_oss_test_data) |
198 | | -@torch.no_grad() |
199 | | -def test_tensorized_gpt_oss_moe_tosa_FP(test_data: input_t) -> None: |
200 | | - (x,) = test_data |
201 | | - model, _ = _make_gpt_oss_moe(x.dtype) |
202 | | - pipeline = TosaPipelineFP[input_t]( |
203 | | - model, |
204 | | - test_data, |
205 | | - [ |
206 | | - "torch.ops.aten.topk.default", |
207 | | - "torch.ops.aten.index.Tensor", |
208 | | - "torch.ops.aten.bmm.default", |
209 | | - "torch.ops.aten.slice.Tensor", |
210 | | - "torch.ops.aten.clamp.default", |
211 | | - ], |
212 | | - "executorch_exir_dialects_edge__ops_", |
213 | | - tosa_extensions=["INT"], |
214 | | - ) |
215 | | - pipeline.run() |
0 commit comments