|
5 | 5 | // LICENSE file in the root directory of this source tree. |
6 | 6 |
|
7 | 7 | #include <algorithm> |
| 8 | +#include <array> |
8 | 9 | #include <chrono> |
9 | 10 | #include <cstdint> |
10 | 11 | #include <cstdio> |
| 12 | +#include <cstring> |
11 | 13 | #include <exception> |
12 | 14 | #include <functional> |
13 | 15 | #include <limits> |
|
24 | 26 | #include <executorch/backends/native/examples/runner_utils/runner_utils.h> |
25 | 27 | #include <executorch/backends/native/runtime/Program.h> |
26 | 28 | #include <executorch/backends/native/runtime/deserialize/Package.h> |
| 29 | +#include <executorch/backends/native/runtime/deserialize/Samples.h> |
| 30 | +#include <executorch/backends/native/runtime/deserialize/ZipReader.h> |
27 | 31 | #include <executorch/backends/native/runtime/engine/Engine.h> |
28 | 32 | #include <executorch/backends/native/runtime/graph/ScalarType.h> |
29 | 33 | #include <executorch/examples/models/llama/tokenizer/llama_tiktoken.h> |
@@ -54,6 +58,12 @@ DEFINE_bool( |
54 | 58 | warmup_prefill, |
55 | 59 | false, |
56 | 60 | "Replay the identical prompt once before timing prefill (benchmark only)."); |
| 61 | +DEFINE_string( |
| 62 | + verify_samples, |
| 63 | + "", |
| 64 | + "Replay this sample set, whose calls must all run --method, instead of " |
| 65 | + "--prompt, comparing each call's logits and forcing the sample's next " |
| 66 | + "token; exits 1 on mismatch."); |
57 | 67 | namespace { |
58 | 68 |
|
59 | 69 | using Clock = std::chrono::steady_clock; |
@@ -86,6 +96,31 @@ size_t kv_cache_length(const ptn::Method& method) { |
86 | 96 | return length; |
87 | 97 | } |
88 | 98 |
|
| 99 | +// The values of an [n] or [1, n] Int or Long sample tensor. |
| 100 | +std::vector<int64_t> integer_values(const ptn::SampleTensor& tensor) { |
| 101 | + int64_t n = -1; |
| 102 | + if (tensor.sizes.size() == 1) { |
| 103 | + n = tensor.sizes[0]; |
| 104 | + } else if (tensor.sizes.size() == 2 && tensor.sizes[0] == 1) { |
| 105 | + n = tensor.sizes[1]; |
| 106 | + } |
| 107 | + if ((tensor.dtype != ptn::kLong && tensor.dtype != ptn::kInt) || n < 0) { |
| 108 | + throw std::runtime_error( |
| 109 | + "sample token and position inputs must be [n] or [1, n] Int or Long"); |
| 110 | + } |
| 111 | + std::vector<int64_t> values(static_cast<size_t>(n)); |
| 112 | + for (size_t i = 0; i < values.size(); ++i) { |
| 113 | + if (tensor.dtype == ptn::kLong) { |
| 114 | + std::memcpy(&values[i], tensor.bytes.data() + 8 * i, 8); |
| 115 | + } else { |
| 116 | + int32_t value = 0; |
| 117 | + std::memcpy(&value, tensor.bytes.data() + 4 * i, 4); |
| 118 | + values[i] = value; |
| 119 | + } |
| 120 | + } |
| 121 | + return values; |
| 122 | +} |
| 123 | + |
89 | 124 | // Chooses the token after a logits output whose last dimension is the |
90 | 125 | // vocabulary. |
91 | 126 | using NextToken = std::function< |
@@ -317,11 +352,11 @@ int run_llm(const std::function<std::unique_ptr<EngineHost>()>& make_host) { |
317 | 352 | method.graph.output_ids.size(), user_outputs); |
318 | 353 |
|
319 | 354 | const std::unique_ptr<ptn::EngineHost> host = make_host(); |
320 | | - const std::unique_ptr<ptn::EngineContext> context = |
| 355 | + std::unique_ptr<ptn::EngineContext> context = |
321 | 356 | host->create_context(program, package); |
322 | 357 | const Clock::time_point load_end = Clock::now(); |
323 | 358 | const Clock::time_point compile_start = Clock::now(); |
324 | | - const std::unique_ptr<ptn::EngineExecutable> executable = |
| 359 | + std::unique_ptr<ptn::EngineExecutable> executable = |
325 | 360 | context->compile(method_name); |
326 | 361 | const Clock::time_point compile_end = Clock::now(); |
327 | 362 |
|
@@ -375,6 +410,143 @@ int run_llm(const std::function<std::unique_ptr<EngineHost>()>& make_host) { |
375 | 410 | "logits must be Float or Half, got " + |
376 | 411 | std::string(ptn::scalar_type_name(output_dtype))); |
377 | 412 | } |
| 413 | + if (!FLAGS_verify_samples.empty()) { |
| 414 | + if (FLAGS_warmup_prefill || !FLAGS_dump_first_logits.empty()) { |
| 415 | + throw std::invalid_argument( |
| 416 | + "verify_samples cannot be combined with warmup_prefill or " |
| 417 | + "dump_first_logits"); |
| 418 | + } |
| 419 | + const ptn::SampleSet set = ptn::read_sample_set( |
| 420 | + ptn::ZipReader::open(FLAGS_package), FLAGS_verify_samples); |
| 421 | + size_t failed_calls = 0; |
| 422 | + for (size_t i = 0; i < set.samples.size(); ++i) { |
| 423 | + const std::vector<ptn::SampleCall>& calls = set.samples[i].calls; |
| 424 | + // Mutable state lives in the context, so a fresh one is the program's |
| 425 | + // initial state. |
| 426 | + if (i > 0) { |
| 427 | + executable.reset(); |
| 428 | + context.reset(); |
| 429 | + context = host->create_context(program, package); |
| 430 | + executable = context->compile(method_name); |
| 431 | + } |
| 432 | + std::vector<uint64_t> prompt; |
| 433 | + std::vector<uint64_t> forced; |
| 434 | + for (size_t j = 0; j < calls.size(); ++j) { |
| 435 | + if (calls[j].method != method_name) { |
| 436 | + throw std::runtime_error( |
| 437 | + "sample " + std::to_string(i) + " call " + std::to_string(j) + |
| 438 | + " runs method " + calls[j].method + |
| 439 | + ", but the runner replays only " + method_name); |
| 440 | + } |
| 441 | + if (calls[j].inputs.size() != 2 || !calls[j].outputs || |
| 442 | + calls[j].outputs->size() <= logits_output || |
| 443 | + (*calls[j].outputs)[logits_output].dtype != ptn::kFloat || |
| 444 | + (*calls[j].outputs)[logits_output].sizes.empty() || |
| 445 | + calls[j].inputs[0].sizes.size() != 2 || |
| 446 | + calls[j].inputs[1].sizes.size() != 1) { |
| 447 | + throw std::runtime_error( |
| 448 | + "sample calls need [1, n] tokens, [n] positions and Float logits"); |
| 449 | + } |
| 450 | + const std::vector<int64_t> ids = integer_values(calls[j].inputs[0]); |
| 451 | + const int64_t vocabulary_size = |
| 452 | + (*calls[j].outputs)[logits_output].sizes.back(); |
| 453 | + if (std::any_of(ids.begin(), ids.end(), [&](int64_t id) { |
| 454 | + return id < 0 || id >= vocabulary_size; |
| 455 | + })) { |
| 456 | + throw std::runtime_error( |
| 457 | + "sample " + std::to_string(i) + " call " + std::to_string(j) + |
| 458 | + " has a token outside the vocabulary"); |
| 459 | + } |
| 460 | + // The runner sends what generate() would: the prompt from 0, then one |
| 461 | + // token per call. |
| 462 | + const size_t count = j == 0 ? ids.size() : 1; |
| 463 | + const int64_t start = |
| 464 | + static_cast<int64_t>(j == 0 ? 0 : prompt.size() + j - 1); |
| 465 | + std::vector<int64_t> positions(per_token_positions ? count : 1); |
| 466 | + std::iota(positions.begin(), positions.end(), start); |
| 467 | + if (ids.empty() || (j > 0 && ids.size() != 1) || |
| 468 | + integer_values(calls[j].inputs[1]) != positions) { |
| 469 | + throw std::runtime_error( |
| 470 | + "sample " + std::to_string(i) + " call " + std::to_string(j) + |
| 471 | + " is not a prefill or the next decode step"); |
| 472 | + } |
| 473 | + if (ids.size() > static_cast<size_t>(token_shape[1])) { |
| 474 | + throw std::runtime_error( |
| 475 | + "sample " + std::to_string(i) + |
| 476 | + " prefill exceeds the method's " + |
| 477 | + std::to_string(token_shape[1]) + " token input"); |
| 478 | + } |
| 479 | + if (j == 0) { |
| 480 | + prompt.assign(ids.begin(), ids.end()); |
| 481 | + } else { |
| 482 | + forced.push_back(static_cast<uint64_t>(ids[0])); |
| 483 | + } |
| 484 | + } |
| 485 | + // generate() runs one call per generated token. |
| 486 | + if (ptn::generation_limit( |
| 487 | + prompt.size(), |
| 488 | + calls.size(), |
| 489 | + static_cast<size_t>(FLAGS_max_seq_length), |
| 490 | + kv_cache_length(method), |
| 491 | + static_cast<size_t>(token_shape[1])) < calls.size()) { |
| 492 | + throw std::runtime_error( |
| 493 | + "sample " + std::to_string(i) + " does not fit the method"); |
| 494 | + } |
| 495 | + size_t call = 0; |
| 496 | + std::string report; |
| 497 | + const NextToken check_then_force = [&](const std::vector<float>& logits, |
| 498 | + size_t vocabulary_size) { |
| 499 | + const ptn::SampleTensor& want = (*calls[call].outputs)[logits_output]; |
| 500 | + if (want.bytes.size() != logits.size() * sizeof(float)) { |
| 501 | + throw std::runtime_error("sample logits have a different size"); |
| 502 | + } |
| 503 | + std::vector<float> want_logits(logits.size()); |
| 504 | + std::memcpy(want_logits.data(), want.bytes.data(), want.bytes.size()); |
| 505 | + const ptn::Tolerance tol = calls[call].tol.value_or(ptn::Tolerance{}); |
| 506 | + const ptn::Closeness result = ptn::compare_close( |
| 507 | + logits.data(), |
| 508 | + want_logits.data(), |
| 509 | + logits.size(), |
| 510 | + tol.atol, |
| 511 | + tol.rtol); |
| 512 | + failed_calls += result.close ? 0 : 1; |
| 513 | + std::array<char, 160> line{}; |
| 514 | + std::snprintf( |
| 515 | + line.data(), |
| 516 | + line.size(), |
| 517 | + "sample %zu call %zu: max_abs_error=%.6g nan=%zu (atol %g, rtol %g) " |
| 518 | + "%s\n", |
| 519 | + i, |
| 520 | + call, |
| 521 | + result.max_abs_error, |
| 522 | + result.nan_count, |
| 523 | + tol.atol, |
| 524 | + tol.rtol, |
| 525 | + result.close ? "ok" : "FAIL"); |
| 526 | + report += line.data(); |
| 527 | + ++call; |
| 528 | + return call < calls.size() |
| 529 | + ? forced[call - 1] |
| 530 | + : ptn::greedy_argmax_last_row(logits, vocabulary_size); |
| 531 | + }; |
| 532 | + generate( |
| 533 | + *executable, |
| 534 | + logits_output, |
| 535 | + per_token_positions, |
| 536 | + *tokenizer, |
| 537 | + prompt, |
| 538 | + calls.size(), |
| 539 | + {}, |
| 540 | + check_then_force); |
| 541 | + std::printf("%s", report.c_str()); |
| 542 | + } |
| 543 | + std::printf( |
| 544 | + "verify: %s, %zu calls failed\n", |
| 545 | + FLAGS_verify_samples.c_str(), |
| 546 | + failed_calls); |
| 547 | + return failed_calls == 0 ? 0 : 1; |
| 548 | + } |
| 549 | + |
378 | 550 | const size_t limit = ptn::generation_limit( |
379 | 551 | prompt_tokens.size(), |
380 | 552 | static_cast<size_t>(FLAGS_max_new_tokens), |
|
0 commit comments