|
4 | 4 | # LICENSE file in the root directory of this source tree. |
5 | 5 | """Test fixed-top-k MoE lowering with finite, distinct router scores. |
6 | 6 |
|
7 | | -Non-finite and equal router scores are covered by the TopK operator tests. |
| 7 | +Equal router scores are covered by the TopK operator tests. Non-finite router |
| 8 | +scores are outside the supported domain. |
8 | 9 |
|
9 | 10 | """ |
10 | 11 |
|
|
16 | 17 | ) |
17 | 18 | from executorch.examples.models.llama.llama_transformer import MOEFeedForward |
18 | 19 | from executorch.examples.models.llama.model_args import ModelArgs |
| 20 | +from transformers import GptOssConfig |
| 21 | +from transformers.models.gpt_oss.modeling_gpt_oss import GptOssExperts, GptOssMLP |
| 22 | +from typing_extensions import override |
19 | 23 |
|
20 | 24 | input_t = tuple[torch.Tensor] |
21 | 25 |
|
@@ -102,3 +106,124 @@ def test_moe_vgf_no_quant(test_data: tuple[torch.Tensor, int, int]) -> None: |
102 | 106 | quantize=False, |
103 | 107 | ) |
104 | 108 | pipeline.run() |
| 109 | + |
| 110 | + |
| 111 | +# GPT-OSS expects [batch, sequence, hidden] inputs. |
| 112 | +gpt_oss_test_data = { |
| 113 | + "fp32": (three_tokens.unsqueeze(0),), |
| 114 | + "fp16": (three_tokens.unsqueeze(0).half(),), |
| 115 | +} |
| 116 | + |
| 117 | + |
| 118 | +class TensorizedGptOssExperts(GptOssExperts): |
| 119 | + """Provide tensorized experts for the native GPT-OSS MLP. |
| 120 | +
|
| 121 | + Used with GptOssMLP from Transformers 4.57.1, 5.0.0rc1 and 5.3.0:: |
| 122 | +
|
| 123 | + GptOssMLP |
| 124 | + |-- router: GptOssTopKRouter |
| 125 | + `-- experts: TensorizedGptOssExperts |
| 126 | +
|
| 127 | + Inherits expert parameters from GptOssExperts and reproduces its clipped |
| 128 | + activation. Only forward() is rewritten as gather/BMM/reduction. |
| 129 | + Accepts either 2D or 3D expert inputs and returns outputs with the same shape. |
| 130 | + Accepts routing weights for the selected experts or, as passed by |
| 131 | + Transformers 4.x, dense routing weights for all experts. |
| 132 | +
|
| 133 | + """ |
| 134 | + |
| 135 | + @override |
| 136 | + def forward( |
| 137 | + self, hidden_states: torch.Tensor, router_indices=None, routing_weights=None |
| 138 | + ) -> torch.Tensor: |
| 139 | + tokens, k = router_indices.shape |
| 140 | + if routing_weights.shape[-1] != k: |
| 141 | + routing_weights = routing_weights.gather(1, router_indices) |
| 142 | + dim = self.hidden_size |
| 143 | + intermediate = self.intermediate_size |
| 144 | + input_shape = hidden_states.shape |
| 145 | + hidden_states = hidden_states.reshape(-1, dim) |
| 146 | + selected_up = self.gate_up_proj[router_indices] |
| 147 | + selected_down = self.down_proj[router_indices] |
| 148 | + expanded_x = hidden_states[:, None, :].expand(tokens, k, dim) |
| 149 | + gate_up = torch.bmm( |
| 150 | + expanded_x.reshape(tokens * k, 1, dim), |
| 151 | + selected_up.reshape(tokens * k, dim, 2 * intermediate), |
| 152 | + ).reshape(tokens, k, 2 * intermediate) |
| 153 | + gate_up = gate_up + self.gate_up_proj_bias[router_indices] |
| 154 | + gate, up = gate_up[..., ::2], gate_up[..., 1::2] |
| 155 | + gate = gate.clamp(max=self.limit) |
| 156 | + up = up.clamp(min=-self.limit, max=self.limit) |
| 157 | + glu = gate * torch.sigmoid(gate * self.alpha) |
| 158 | + hidden = (up + 1) * glu |
| 159 | + outputs = torch.bmm( |
| 160 | + hidden.reshape(tokens * k, 1, intermediate), |
| 161 | + selected_down.reshape(tokens * k, intermediate, dim), |
| 162 | + ).reshape(tokens, k, dim) |
| 163 | + outputs = outputs + self.down_proj_bias[router_indices] |
| 164 | + outputs = (outputs * routing_weights.unsqueeze(-1)).sum(dim=1) |
| 165 | + return outputs.reshape(input_shape) |
| 166 | + |
| 167 | + |
| 168 | +def _make_gpt_oss_moe(dtype: torch.dtype) -> tuple[GptOssMLP, GptOssConfig]: |
| 169 | + config = GptOssConfig( |
| 170 | + hidden_size=8, |
| 171 | + intermediate_size=16, |
| 172 | + num_local_experts=8, |
| 173 | + num_experts_per_tok=4, |
| 174 | + experts_implementation="eager", |
| 175 | + ) |
| 176 | + model = GptOssMLP(config) |
| 177 | + model.experts = TensorizedGptOssExperts(config) |
| 178 | + generator = torch.Generator().manual_seed(2648) |
| 179 | + with torch.no_grad(): |
| 180 | + for parameter in model.experts.parameters(): |
| 181 | + parameter.uniform_(-0.25, 0.25, generator=generator) |
| 182 | + model.router.weight.copy_(torch.eye(model.router.weight.shape[1])) |
| 183 | + model.router.bias.copy_(torch.linspace(-0.07, 0.07, model.router.bias.numel())) |
| 184 | + # Exercise gate/up clipping and unclipped negative gates. |
| 185 | + model.experts.gate_up_proj_bias[:, :8].add_( |
| 186 | + torch.tensor([9.0, 0.5, 0.5, 9.0, 0.5, -9.0, -9.0, 0.5]) |
| 187 | + ) |
| 188 | + return model.eval().to(dtype), config |
| 189 | + |
| 190 | + |
| 191 | +@common.parametrize("test_data", gpt_oss_test_data) |
| 192 | +@torch.no_grad() |
| 193 | +def test_tensorized_gpt_oss_moe_matches_reference(test_data: input_t) -> None: |
| 194 | + (x,) = test_data |
| 195 | + model, config = _make_gpt_oss_moe(x.dtype) |
| 196 | + reference = GptOssMLP(config).eval().to(x.dtype) |
| 197 | + reference.load_state_dict(model.state_dict()) |
| 198 | + torch.testing.assert_close(model(x), reference(x)) |
| 199 | + |
| 200 | + |
| 201 | +@common.parametrize("test_data", gpt_oss_test_data) |
| 202 | +@common.xfail_if_dependency_version( |
| 203 | + "transformers", |
| 204 | + "<5", |
| 205 | + module="transformers", |
| 206 | + reason=( |
| 207 | + "Transformers 4.x GptOssTopKRouter scatters dense router scores, and " |
| 208 | + "aten.scatter.src is not delegated." |
| 209 | + ), |
| 210 | + raises=RuntimeError, |
| 211 | +) |
| 212 | +@torch.no_grad() |
| 213 | +def test_tensorized_gpt_oss_moe_tosa_FP(test_data: input_t) -> None: |
| 214 | + (x,) = test_data |
| 215 | + model, _ = _make_gpt_oss_moe(x.dtype) |
| 216 | + pipeline = TosaPipelineFP[input_t]( |
| 217 | + model, |
| 218 | + test_data, |
| 219 | + [ |
| 220 | + "torch.ops.aten.topk.default", |
| 221 | + "torch.ops.aten.index.Tensor", |
| 222 | + "torch.ops.aten.bmm.default", |
| 223 | + "torch.ops.aten.slice.Tensor", |
| 224 | + "torch.ops.aten.clamp.default", |
| 225 | + ], |
| 226 | + "executorch_exir_dialects_edge__ops_", |
| 227 | + tosa_extensions=["INT"], |
| 228 | + ) |
| 229 | + pipeline.run() |
0 commit comments