Skip to content

Commit 9294050

Browse files
committed
Update (base update)
[ghstack-poisoned]
2 parents f864eb5 + e9ddcda commit 9294050

8 files changed

Lines changed: 363 additions & 92 deletions

File tree

‎backends/native/runtime/engine/ggml/GgmlLowerElementwise.cpp‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -43,28 +43,26 @@ void GraphBuilder::lower_rms_norm(const Node& node) {
4343
}
4444

4545
void GraphBuilder::lower_layer_norm(const Node& node) {
46-
TensorRef input = tensor_arg(node, "input", 0);
46+
const TensorRef input = tensor_arg(node, "input", 0);
4747
const std::vector<int64_t> normalized =
4848
int_list(require_arg(node, "normalized_shape", 1).arg);
4949
if (input.sizes.empty() || normalized.size() != 1 ||
5050
normalized[0] != input.sizes.back()) {
5151
throw std::runtime_error("ggml: layer_norm only normalizes the last dim");
5252
}
5353
require_only_first_output_used(node);
54-
// ggml's norm kernels only read F32. Computed in F32, as eager does for F16
55-
// and BF16; set() rounds back.
56-
input.tensor = half_as_f32(input.tensor);
54+
// ggml's norm kernels only read F32.
5755
require_f32(node, input.tensor);
5856
const double eps = float_arg(node, "eps", 4, 1e-5, true);
5957
ggml_context* ctx = instance_.context.get();
6058
ggml_tensor* result =
6159
ggml_norm(ctx, contiguous(input.tensor), static_cast<float>(eps));
6260
if (const std::optional<TensorRef> weight =
6361
optional_tensor(node, "weight", 2)) {
64-
result = ggml_mul(ctx, result, half_as_f32(weight->tensor));
62+
result = ggml_mul(ctx, result, weight->tensor);
6563
}
6664
if (const std::optional<TensorRef> bias = optional_tensor(node, "bias", 3)) {
67-
result = ggml_add(ctx, result, half_as_f32(bias->tensor));
65+
result = ggml_add(ctx, result, bias->tensor);
6866
}
6967
set(output_ids(node).at(0), TensorRef{result, input.sizes});
7068
}
@@ -219,13 +217,8 @@ void GraphBuilder::lower_sqrt(const Node& node) {
219217
}
220218

221219
void GraphBuilder::lower_hardtanh(const Node& node) {
222-
TensorRef input = tensor_arg(node, "self", 0);
220+
const TensorRef input = tensor_arg(node, "self", 0);
223221
require_float(node, input.tensor);
224-
// ggml-cpu clamps F32 and F16 only, so BF16 clamps in F32, exactly; set()
225-
// rounds back.
226-
if (input.tensor->type == GGML_TYPE_BF16) {
227-
input.tensor = as_f32(input.tensor);
228-
}
229222
const float min =
230223
static_cast<float>(float_arg(node, "min_val", 1, -1.0, true));
231224
const float max =

‎backends/native/test/runtime/ggml/BUCK‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -131,13 +131,7 @@ _VARIANTS = [
131131
name = "test_ggml_ops" + v["suffix"],
132132
srcs = ["test_ggml_ops.py"],
133133
env = {"GGML_PTN_DEVICE": v["device"]} | v.get("env", {}),
134-
# Unbundled, every case starts its own test process: ~19 s of imports
135-
# for ~3 s of test.
136-
labels = (
137-
ggml_ci_labels(v["suffix"].lstrip("_"), bundle = 40)
138-
if v["device"] == "gpu"
139-
else v.get("labels", [])
140-
),
134+
labels = v.get("labels", []),
141135
remote_execution = v.get("remote_execution"),
142136
resources = {":ggml_ptn_runner" + v["suffix"]: "ggml_ptn_runner"},
143137
deps = [

‎backends/native/test/runtime/ggml/test_ggml_ops.py‎

Lines changed: 6 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -694,12 +694,6 @@ def test_hardtanh(self):
694694
with self.assertRaisesRegex(AssertionError, "floating-point"):
695695
self._check(Lambda(F.relu6), (torch.arange(6).reshape(2, 3) - 2,))
696696

697-
def test_relu6_of_half_inputs(self):
698-
x = torch.randn(2, 3, 4) * 8.0
699-
for dtype in (torch.float16, torch.bfloat16):
700-
with self.subTest(dtype=dtype):
701-
self._check(Lambda(F.relu6), (x.to(dtype),), atol=0, rtol=0)
702-
703697
def test_elu(self):
704698
self._check(Lambda(F.elu), (torch.randn(2, 3, 4) * 3.0,))
705699
# Eager's expm1 is within rtol of these; exp(x) - 1 is 19% off at -1e-7.
@@ -1091,20 +1085,12 @@ def test_layer_norm_without_affine(self):
10911085
torch.nn.LayerNorm(16, elementwise_affine=False), (torch.randn(2, 3, 16),)
10921086
)
10931087

1094-
def test_layer_norm_half(self):
1095-
norm = torch.nn.LayerNorm(16)
1096-
torch.nn.init.uniform_(norm.weight, 0.5, 1.5)
1097-
torch.nn.init.uniform_(norm.bias, -0.5, 0.5)
1098-
x = torch.randn(2, 3, 16)
1099-
for dtype in (torch.float16, torch.bfloat16):
1100-
with self.subTest(dtype=dtype):
1101-
# One rounding of the output; eager normalizes in F32 too.
1102-
self._check(
1103-
copy.deepcopy(norm).to(dtype),
1104-
(x.to(dtype),),
1105-
atol=1e-2 if dtype == torch.bfloat16 else 2e-3,
1106-
rtol=1e-2 if dtype == torch.bfloat16 else 2e-3,
1107-
)
1088+
def test_layer_norm_rejects_half(self):
1089+
with self.assertRaisesRegex(AssertionError, "F32 tensors only"):
1090+
self._check(
1091+
torch.nn.LayerNorm(16).half(),
1092+
(torch.randn(2, 3, 16, dtype=torch.float16),),
1093+
)
11081094

11091095
def test_layer_norm_rejects_used_mean(self):
11101096
def forward(x):
@@ -1981,34 +1967,6 @@ def test_reductions_and_sqrt_of_half_linears(self):
19811967
rtol=1e-2,
19821968
)
19831969

1984-
def test_layer_norm_and_hardtanh_of_half_intermediates(self):
1985-
# A half linear or embedding result is rounded to its dtype, so
1986-
# layer_norm, which computes in F32 only, and hardtanh, which ggml-cpu
1987-
# clamps in F32 and F16 only, must widen it again.
1988-
torch.manual_seed(0)
1989-
norm = torch.nn.LayerNorm(64)
1990-
torch.nn.init.uniform_(norm.weight, 0.5, 1.5)
1991-
torch.nn.init.uniform_(norm.bias, -0.5, 0.5)
1992-
linear = torch.nn.Linear(16, 64)
1993-
x = torch.randn(4, 16)
1994-
cases = {
1995-
"layer_norm of a linear": (linear, norm, x),
1996-
"layer_norm of an embedding": (
1997-
torch.nn.Embedding(10, 64),
1998-
norm,
1999-
torch.tensor([[1, 7, 3]]),
2000-
),
2001-
"hardtanh of a linear": (linear, torch.nn.Hardtanh(-0.5, 0.5), x),
2002-
"relu6 of a linear": (linear, torch.nn.ReLU6(), x * 8),
2003-
}
2004-
for name, (first, second, x) in cases.items():
2005-
for dtype in (torch.float16, torch.bfloat16):
2006-
with self.subTest(case=name, dtype=dtype):
2007-
model = torch.nn.Sequential(first, second)
2008-
half = copy.deepcopy(model).to(dtype).requires_grad_(False)
2009-
inputs = (x.to(dtype) if x.is_floating_point() else x,)
2010-
self._check(half, inputs, atol=1e-2, rtol=1e-2)
2011-
20121970
def test_rope_llama3(self):
20131971
config = LlamaConfig(
20141972
hidden_size=256,

‎examples/llm_server/python/openai_transcript.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,10 @@ def build_prompt_input(
313313
reasoning string match what we returned, and we kept faithful ids for it.
314314
Omitted or null reasoning permits reuse; a string edit invalidates the tail.
315315
``history_messages`` preserves source content before model normalization,
316-
such as image URLs replaced with placeholders for rendering.
316+
such as image URLs replaced with placeholders for rendering, and the
317+
client's tool-result order for fingerprinting. Normalization must keep
318+
non-tool messages at their original indices and finish before this call;
319+
sentinel rendering replaces spliced assistants without their tool calls.
317320
Falls back to text on a sentinel collision or a render that
318321
dropped/duplicated a sentinel."""
319322
stored = self._turns.get(session_id or "")

‎examples/models/muse-glimmer/serving/chat_template.py‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,9 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
"""Muse's role-complete template and canonical OpenAI tool-name translation."""
6+
"""Muse's template, tool-name translation, and tool-result ordering."""
77

8+
from collections import Counter
89
from copy import deepcopy
910
from functools import cached_property
1011
from pathlib import Path
@@ -32,6 +33,54 @@ def resolve_tool_result_names(messages):
3233
return result
3334

3435

36+
def order_repeated_tool_results(messages):
37+
"""Put results with the same rendered name in call order, in place.
38+
39+
The template labels a result only with its function name, so the model can
40+
tell results of calls to the same function apart only by their order. Each
41+
such group is reordered within the positions it already occupies. Results
42+
already in call order, results of other functions, and results that do not
43+
match the preceding consecutive assistant messages' calls keep their
44+
positions. Repeated call IDs and repeated result IDs within a run are
45+
excluded from reordering. Non-tool messages never move.
46+
"""
47+
calls = []
48+
start = 0
49+
while start < len(messages):
50+
if messages[start].role != "tool":
51+
if messages[start].role == "assistant":
52+
calls.extend(messages[start].tool_calls or [])
53+
else:
54+
calls.clear()
55+
start += 1
56+
continue
57+
end = start
58+
while end < len(messages) and messages[end].role == "tool":
59+
end += 1
60+
call_ids = Counter(call.id for call in calls)
61+
call_order = {
62+
call.id: (index, wire_tool_name(call.function.name))
63+
for index, call in enumerate(calls)
64+
if call_ids[call.id] == 1 and call.function.name
65+
}
66+
ids = Counter(message.tool_call_id for message in messages[start:end])
67+
groups = {}
68+
for position in range(start, end):
69+
message = messages[position]
70+
call = call_order.get(message.tool_call_id)
71+
if call is None or not message.name or ids[message.tool_call_id] != 1:
72+
continue
73+
index, name = call
74+
if name == wire_tool_name(message.name):
75+
groups.setdefault(name, []).append((index, position))
76+
for positions in groups.values():
77+
ordered = [messages[position] for _, position in sorted(positions)]
78+
for (_, position), message in zip(positions, ordered):
79+
messages[position] = message
80+
calls.clear()
81+
start = end
82+
83+
3584
class MuseGlimmerChatTemplate(ChatTemplate):
3685
def __init__(self, *args, **kwargs):
3786
kwargs["chat_template"] = Path(__file__).with_suffix(".jinja").read_text()

‎examples/models/muse-glimmer/serving/serve.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from executorch.examples.llm_server.python.worker_client import spawn_worker
2929
from executorch.examples.models.muse_glimmer.serving.chat_template import (
3030
MuseGlimmerChatTemplate as ChatTemplate,
31+
order_repeated_tool_results,
3132
resolve_tool_result_names,
3233
)
3334
from executorch.examples.models.muse_glimmer.serving.framing import (
@@ -355,9 +356,10 @@ async def _create(
355356

356357
history_req = req
357358
req, image_segment = _normalize_image_request(req, self._max_image_bytes)
358-
req = req.model_copy(
359-
update={"messages": resolve_tool_result_names(req.messages)}
360-
)
359+
messages = resolve_tool_result_names(req.messages)
360+
# Keep non-tool indices fixed and reorder before build_prompt_input's sentinel render.
361+
order_repeated_tool_results(messages)
362+
req = req.model_copy(update={"messages": messages})
361363
template_tools = None if req.tool_choice == "none" else req.tools
362364
template_kwargs = dict(req.chat_template_kwargs or {})
363365
template_kwargs.pop("return_reasoning", None)

0 commit comments

Comments
 (0)