Skip to content

Commit f498e0f

Browse files
[ET][native] Verify bundled LLM samples through the runner (#24234)
This PR was created by the merge bot to help merge the original PR into the main branch. ghstack PR number: #24175 by @digantdesai ^ Please use this as the source of truth for the PR details, comments, and reviews ghstack PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/181/base ghstack PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/181/head Merge bot PR base: https://github.com/pytorch/executorch/tree/gh/digantdesai/180/orig Merge bot PR head: https://github.com/pytorch/executorch/tree/gh/digantdesai/181/orig Differential Revision: [D125544389](https://our.internmc.facebook.com/intern/diff/D125544389/) @diff-train-skip-merge Co-authored-by: Digant Desai <digantdesai@meta.com>
1 parent 5ba5a60 commit f498e0f

7 files changed

Lines changed: 463 additions & 2 deletions

File tree

‎backends/native/examples/hf_llm/BUCK‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,21 @@ fbcode_target(
3939
],
4040
)
4141

42+
fbcode_target(
43+
_kind = runtime.python_test,
44+
name = "test_verify_samples",
45+
srcs = ["test_verify_samples.py"],
46+
env = {
47+
"RUN_HF_LLM": "$(location //executorch/backends/native/examples/hf_llm/runtime:run_hf_llm_ggml_cpu)",
48+
},
49+
deps = [
50+
":export_hf_llm_lib",
51+
"//caffe2:torch",
52+
"//executorch/exir/native:native",
53+
"fbsource//third-party/pypi/transformers:transformers",
54+
],
55+
)
56+
4257
fbcode_target(
4358
_kind = runtime.python_binary,
4459
name = "convert_meta_llama",

‎backends/native/examples/hf_llm/runtime/BUCK‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ runtime.cxx_library(
3232
"//executorch/backends/native/examples/runner_utils:runner_utils",
3333
"//executorch/backends/native/runtime:runtime",
3434
"//executorch/backends/native/runtime/deserialize:package",
35+
"//executorch/backends/native/runtime/deserialize:samples",
36+
"//executorch/backends/native/runtime/deserialize:zip_reader",
3537
"//executorch/backends/native/runtime/graph:scalar_type",
3638
"//executorch/examples/models/llama/tokenizer:tiktoken",
3739
],

‎backends/native/examples/hf_llm/runtime/LlmRunner.cpp‎

Lines changed: 174 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
// LICENSE file in the root directory of this source tree.
66

77
#include <algorithm>
8+
#include <array>
89
#include <chrono>
910
#include <cstdint>
1011
#include <cstdio>
12+
#include <cstring>
1113
#include <exception>
1214
#include <functional>
1315
#include <limits>
@@ -24,6 +26,8 @@
2426
#include <executorch/backends/native/examples/runner_utils/runner_utils.h>
2527
#include <executorch/backends/native/runtime/Program.h>
2628
#include <executorch/backends/native/runtime/deserialize/Package.h>
29+
#include <executorch/backends/native/runtime/deserialize/Samples.h>
30+
#include <executorch/backends/native/runtime/deserialize/ZipReader.h>
2731
#include <executorch/backends/native/runtime/engine/Engine.h>
2832
#include <executorch/backends/native/runtime/graph/ScalarType.h>
2933
#include <executorch/examples/models/llama/tokenizer/llama_tiktoken.h>
@@ -54,6 +58,12 @@ DEFINE_bool(
5458
warmup_prefill,
5559
false,
5660
"Replay the identical prompt once before timing prefill (benchmark only).");
61+
DEFINE_string(
62+
verify_samples,
63+
"",
64+
"Replay this sample set, whose calls must all run --method, instead of "
65+
"--prompt, comparing each call's logits and forcing the sample's next "
66+
"token; exits 1 on mismatch.");
5767
namespace {
5868

5969
using Clock = std::chrono::steady_clock;
@@ -86,6 +96,31 @@ size_t kv_cache_length(const ptn::Method& method) {
8696
return length;
8797
}
8898

99+
// The values of an [n] or [1, n] Int or Long sample tensor.
100+
std::vector<int64_t> integer_values(const ptn::SampleTensor& tensor) {
101+
int64_t n = -1;
102+
if (tensor.sizes.size() == 1) {
103+
n = tensor.sizes[0];
104+
} else if (tensor.sizes.size() == 2 && tensor.sizes[0] == 1) {
105+
n = tensor.sizes[1];
106+
}
107+
if ((tensor.dtype != ptn::kLong && tensor.dtype != ptn::kInt) || n < 0) {
108+
throw std::runtime_error(
109+
"sample token and position inputs must be [n] or [1, n] Int or Long");
110+
}
111+
std::vector<int64_t> values(static_cast<size_t>(n));
112+
for (size_t i = 0; i < values.size(); ++i) {
113+
if (tensor.dtype == ptn::kLong) {
114+
std::memcpy(&values[i], tensor.bytes.data() + 8 * i, 8);
115+
} else {
116+
int32_t value = 0;
117+
std::memcpy(&value, tensor.bytes.data() + 4 * i, 4);
118+
values[i] = value;
119+
}
120+
}
121+
return values;
122+
}
123+
89124
// Chooses the token after a logits output whose last dimension is the
90125
// vocabulary.
91126
using NextToken = std::function<
@@ -317,11 +352,11 @@ int run_llm(const std::function<std::unique_ptr<EngineHost>()>& make_host) {
317352
method.graph.output_ids.size(), user_outputs);
318353

319354
const std::unique_ptr<ptn::EngineHost> host = make_host();
320-
const std::unique_ptr<ptn::EngineContext> context =
355+
std::unique_ptr<ptn::EngineContext> context =
321356
host->create_context(program, package);
322357
const Clock::time_point load_end = Clock::now();
323358
const Clock::time_point compile_start = Clock::now();
324-
const std::unique_ptr<ptn::EngineExecutable> executable =
359+
std::unique_ptr<ptn::EngineExecutable> executable =
325360
context->compile(method_name);
326361
const Clock::time_point compile_end = Clock::now();
327362

@@ -375,6 +410,143 @@ int run_llm(const std::function<std::unique_ptr<EngineHost>()>& make_host) {
375410
"logits must be Float or Half, got " +
376411
std::string(ptn::scalar_type_name(output_dtype)));
377412
}
413+
if (!FLAGS_verify_samples.empty()) {
414+
if (FLAGS_warmup_prefill || !FLAGS_dump_first_logits.empty()) {
415+
throw std::invalid_argument(
416+
"verify_samples cannot be combined with warmup_prefill or "
417+
"dump_first_logits");
418+
}
419+
const ptn::SampleSet set = ptn::read_sample_set(
420+
ptn::ZipReader::open(FLAGS_package), FLAGS_verify_samples);
421+
size_t failed_calls = 0;
422+
for (size_t i = 0; i < set.samples.size(); ++i) {
423+
const std::vector<ptn::SampleCall>& calls = set.samples[i].calls;
424+
// Mutable state lives in the context, so a fresh one is the program's
425+
// initial state.
426+
if (i > 0) {
427+
executable.reset();
428+
context.reset();
429+
context = host->create_context(program, package);
430+
executable = context->compile(method_name);
431+
}
432+
std::vector<uint64_t> prompt;
433+
std::vector<uint64_t> forced;
434+
for (size_t j = 0; j < calls.size(); ++j) {
435+
if (calls[j].method != method_name) {
436+
throw std::runtime_error(
437+
"sample " + std::to_string(i) + " call " + std::to_string(j) +
438+
" runs method " + calls[j].method +
439+
", but the runner replays only " + method_name);
440+
}
441+
if (calls[j].inputs.size() != 2 || !calls[j].outputs ||
442+
calls[j].outputs->size() <= logits_output ||
443+
(*calls[j].outputs)[logits_output].dtype != ptn::kFloat ||
444+
(*calls[j].outputs)[logits_output].sizes.empty() ||
445+
calls[j].inputs[0].sizes.size() != 2 ||
446+
calls[j].inputs[1].sizes.size() != 1) {
447+
throw std::runtime_error(
448+
"sample calls need [1, n] tokens, [n] positions and Float logits");
449+
}
450+
const std::vector<int64_t> ids = integer_values(calls[j].inputs[0]);
451+
const int64_t vocabulary_size =
452+
(*calls[j].outputs)[logits_output].sizes.back();
453+
if (std::any_of(ids.begin(), ids.end(), [&](int64_t id) {
454+
return id < 0 || id >= vocabulary_size;
455+
})) {
456+
throw std::runtime_error(
457+
"sample " + std::to_string(i) + " call " + std::to_string(j) +
458+
" has a token outside the vocabulary");
459+
}
460+
// The runner sends what generate() would: the prompt from 0, then one
461+
// token per call.
462+
const size_t count = j == 0 ? ids.size() : 1;
463+
const int64_t start =
464+
static_cast<int64_t>(j == 0 ? 0 : prompt.size() + j - 1);
465+
std::vector<int64_t> positions(per_token_positions ? count : 1);
466+
std::iota(positions.begin(), positions.end(), start);
467+
if (ids.empty() || (j > 0 && ids.size() != 1) ||
468+
integer_values(calls[j].inputs[1]) != positions) {
469+
throw std::runtime_error(
470+
"sample " + std::to_string(i) + " call " + std::to_string(j) +
471+
" is not a prefill or the next decode step");
472+
}
473+
if (ids.size() > static_cast<size_t>(token_shape[1])) {
474+
throw std::runtime_error(
475+
"sample " + std::to_string(i) +
476+
" prefill exceeds the method's " +
477+
std::to_string(token_shape[1]) + " token input");
478+
}
479+
if (j == 0) {
480+
prompt.assign(ids.begin(), ids.end());
481+
} else {
482+
forced.push_back(static_cast<uint64_t>(ids[0]));
483+
}
484+
}
485+
// generate() runs one call per generated token.
486+
if (ptn::generation_limit(
487+
prompt.size(),
488+
calls.size(),
489+
static_cast<size_t>(FLAGS_max_seq_length),
490+
kv_cache_length(method),
491+
static_cast<size_t>(token_shape[1])) < calls.size()) {
492+
throw std::runtime_error(
493+
"sample " + std::to_string(i) + " does not fit the method");
494+
}
495+
size_t call = 0;
496+
std::string report;
497+
const NextToken check_then_force = [&](const std::vector<float>& logits,
498+
size_t vocabulary_size) {
499+
const ptn::SampleTensor& want = (*calls[call].outputs)[logits_output];
500+
if (want.bytes.size() != logits.size() * sizeof(float)) {
501+
throw std::runtime_error("sample logits have a different size");
502+
}
503+
std::vector<float> want_logits(logits.size());
504+
std::memcpy(want_logits.data(), want.bytes.data(), want.bytes.size());
505+
const ptn::Tolerance tol = calls[call].tol.value_or(ptn::Tolerance{});
506+
const ptn::Closeness result = ptn::compare_close(
507+
logits.data(),
508+
want_logits.data(),
509+
logits.size(),
510+
tol.atol,
511+
tol.rtol);
512+
failed_calls += result.close ? 0 : 1;
513+
std::array<char, 160> line{};
514+
std::snprintf(
515+
line.data(),
516+
line.size(),
517+
"sample %zu call %zu: max_abs_error=%.6g nan=%zu (atol %g, rtol %g) "
518+
"%s\n",
519+
i,
520+
call,
521+
result.max_abs_error,
522+
result.nan_count,
523+
tol.atol,
524+
tol.rtol,
525+
result.close ? "ok" : "FAIL");
526+
report += line.data();
527+
++call;
528+
return call < calls.size()
529+
? forced[call - 1]
530+
: ptn::greedy_argmax_last_row(logits, vocabulary_size);
531+
};
532+
generate(
533+
*executable,
534+
logits_output,
535+
per_token_positions,
536+
*tokenizer,
537+
prompt,
538+
calls.size(),
539+
{},
540+
check_then_force);
541+
std::printf("%s", report.c_str());
542+
}
543+
std::printf(
544+
"verify: %s, %zu calls failed\n",
545+
FLAGS_verify_samples.c_str(),
546+
failed_calls);
547+
return failed_calls == 0 ? 0 : 1;
548+
}
549+
378550
const size_t limit = ptn::generation_limit(
379551
prompt_tokens.size(),
380552
static_cast<size_t>(FLAGS_max_new_tokens),

‎backends/native/examples/hf_llm/runtime/LlmRunnerUtils.cpp‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,4 +175,30 @@ size_t generation_limit(
175175
return std::min(max_new_tokens, max_total_tokens - prompt_length);
176176
}
177177

178+
// cppcheck-suppress unusedFunction
179+
Closeness compare_close(
180+
const float* got,
181+
const float* want,
182+
size_t n,
183+
double atol,
184+
double rtol) {
185+
Closeness result;
186+
for (size_t i = 0; i < n; ++i) {
187+
if (got[i] == want[i]) {
188+
continue;
189+
}
190+
const double error =
191+
std::abs(static_cast<double>(got[i]) - static_cast<double>(want[i]));
192+
if (std::isnan(error)) {
193+
++result.nan_count;
194+
result.close = false;
195+
continue;
196+
}
197+
result.close &= std::isfinite(error) &&
198+
error <= atol + rtol * std::abs(static_cast<double>(want[i]));
199+
result.max_abs_error = std::max(result.max_abs_error, error);
200+
}
201+
return result;
202+
}
203+
178204
} // namespace ptn

‎backends/native/examples/hf_llm/runtime/LlmRunnerUtils.h‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,19 @@ size_t generation_limit(
4242
size_t cache_length,
4343
size_t token_upper_bound);
4444

45+
struct Closeness {
46+
double max_abs_error = 0.0;
47+
size_t nan_count = 0;
48+
bool close = true;
49+
};
50+
// Largest non-NaN |got - want|, the number of elements whose difference is NaN,
51+
// and whether every element is within atol + rtol * |want|. Equal values,
52+
// infinities included, are close; a NaN never is.
53+
Closeness compare_close(
54+
const float* got,
55+
const float* want,
56+
size_t n,
57+
double atol,
58+
double rtol);
59+
4560
} // namespace ptn

‎backends/native/examples/hf_llm/runtime/LlmRunnerUtilsTest.cpp‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,48 @@ TEST(LlmRunnerUtilsTest, TopLogitIndicesRankNanFirst) {
140140
(std::vector<size_t>{2, 104, 103, 102, 101}));
141141
EXPECT_EQ(top_logit_indices(row.data(), 2, 5), (std::vector<size_t>{0, 1}));
142142
}
143+
144+
TEST(LlmRunnerUtilsTest, CompareCloseAppliesAbsoluteAndRelativeBounds) {
145+
const std::vector<float> want = {1.0f, -100.0f, 0.0f};
146+
const std::vector<float> got = {1.5f, -101.0f, 0.0f};
147+
148+
const Closeness loose = compare_close(got.data(), want.data(), 3, 0.5, 0.01);
149+
EXPECT_TRUE(loose.close);
150+
EXPECT_DOUBLE_EQ(loose.max_abs_error, 1.0);
151+
EXPECT_FALSE(compare_close(got.data(), want.data(), 3, 0.4, 0.01).close);
152+
EXPECT_FALSE(compare_close(got.data(), want.data(), 3, 0.5, 0.004).close);
153+
}
154+
155+
TEST(LlmRunnerUtilsTest, CompareCloseScalesRtolByWant) {
156+
const std::vector<float> small = {100.0f};
157+
const std::vector<float> large = {102.0f};
158+
159+
EXPECT_FALSE(compare_close(large.data(), small.data(), 1, 0.0, 0.0199).close);
160+
EXPECT_TRUE(compare_close(small.data(), large.data(), 1, 0.0, 0.0199).close);
161+
}
162+
163+
TEST(LlmRunnerUtilsTest, CompareCloseRejectsNonFiniteMismatches) {
164+
const float nan = std::numeric_limits<float>::quiet_NaN();
165+
const float inf = std::numeric_limits<float>::infinity();
166+
const std::vector<float> want = {0.0f, inf, 2.0f};
167+
168+
const Closeness result = compare_close(
169+
std::vector<float>{nan, inf, 2.5f}.data(), want.data(), 3, 1.0, 1.0);
170+
EXPECT_FALSE(result.close);
171+
EXPECT_DOUBLE_EQ(result.max_abs_error, 0.5);
172+
EXPECT_EQ(result.nan_count, 1);
173+
const Closeness both_nan = compare_close(
174+
std::vector<float>{nan}.data(), std::vector<float>{nan}.data(), 1, 1, 1);
175+
EXPECT_FALSE(both_nan.close);
176+
EXPECT_EQ(both_nan.nan_count, 1);
177+
EXPECT_TRUE(compare_close(want.data() + 1, want.data() + 1, 2, 0, 0).close);
178+
EXPECT_FALSE(
179+
compare_close(std::vector<float>{-inf}.data(), want.data() + 1, 1, 1, 1)
180+
.close);
181+
EXPECT_FALSE(
182+
compare_close(std::vector<float>{0.0f}.data(), want.data() + 1, 1, 1, 1)
183+
.close);
184+
}
143185
// cppcheck-suppress-end syntaxError
144186

145187
} // namespace

0 commit comments

Comments
 (0)