Skip to content

Commit c87270e

Browse files
committed
LLM runner: validate and drive hf_static v1 cache buffers
Use authoritative positions, float-cache version and dtype checks, exact buffer geometry and selected methods. Bind optional HF state outputs to existing buffers, preserve input aliases and register each concrete IOManager from its own implementation. Float-only fixtures cover reset, padding, malformed metadata and near-context tails.
1 parent 3a344c3 commit c87270e

10 files changed

Lines changed: 1666 additions & 9 deletions

File tree

‎extension/llm/runner/io_manager/hf_static_io_manager.cpp‎

Lines changed: 538 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
* All rights reserved.
4+
*
5+
* This source code is licensed under the BSD-style license found in the
6+
* LICENSE file in the root directory of this source tree.
7+
*/
8+
9+
#pragma once
10+
11+
#include <executorch/extension/llm/runner/io_manager/io_manager.h>
12+
#include <executorch/extension/tensor/tensor.h>
13+
14+
#include <map>
15+
16+
namespace executorch {
17+
namespace extension {
18+
namespace llm {
19+
20+
/**
21+
* @brief Drives Hugging Face's static-cache ABI ("hf_static").
22+
*
23+
* Each method is HF's decode graph at a static query length N. Inputs:
24+
* input_ids [1, N], attention_mask(s) [1, 1, N, L] (bool), position_ids
25+
* [1, N], then per layer keys and values [1, H, L, D] and cumulative_length.
26+
* Outputs: the caches and counters updated in place, logits [1, N, V], and
27+
* optionally the duplicate HF user cache state.
28+
* A program may carry `prefill` (N tokens per step, for the prompt) and
29+
* `decode` (one token), or a single `forward`.
30+
*
31+
* Version 1 supports float32 and float16 caches. See the versioned contract
32+
* in docs/source/llm-hf-static-io-contract.md for metadata and padding rules.
33+
*
34+
* Sliding-window layers may hold window-sized caches (HF's): get_max_seq_len
35+
* defines the context, and a window's mask also marks a prompt chunk's padding
36+
* rows (all false), so the program shifts the window by the real rows only.
37+
*
38+
* This IOManager owns the caches and counters, one set shared by every method
39+
* (from the I/O buffer allocator when there is one, so a delegate reads them in
40+
* place), and builds each step's tokens, masks and positions as HF's generator
41+
* does. The program writes each step's rows and counters into those buffers.
42+
*/
43+
class ET_EXPERIMENTAL HfStaticIOManager : public IOManager {
44+
public:
45+
static constexpr const char* kContract = "hf_static";
46+
static constexpr int64_t kContractVersion = 1;
47+
48+
explicit HfStaticIOManager(ET_MODULE_NAMESPACE::Module& module);
49+
~HfStaticIOManager() override;
50+
51+
ET_NODISCARD runtime::Error load(
52+
const std::string& prefill_method,
53+
const std::string& decode_method) override;
54+
55+
ET_NODISCARD runtime::Error reset(
56+
const std::string& prefill_method,
57+
const std::string& decode_method) override;
58+
59+
runtime::Result<std::vector<runtime::EValue>> prepare_prefill(
60+
const TensorPtr& input,
61+
const TensorPtr& start_pos,
62+
const std::string& prefill_method) override;
63+
64+
runtime::Result<std::vector<runtime::EValue>> prepare_decode(
65+
const TensorPtr& input,
66+
const TensorPtr& start_pos,
67+
const std::string& decode_method) override;
68+
69+
ET_NODISCARD runtime::Error update_prefill(
70+
const std::vector<runtime::EValue>& model_outputs,
71+
const std::string& prefill_method) override;
72+
73+
ET_NODISCARD runtime::Error update_decode(
74+
const std::vector<runtime::EValue>& model_outputs,
75+
const std::string& decode_method) override;
76+
77+
runtime::Result<executorch::aten::Tensor> logits(
78+
const std::vector<runtime::EValue>& model_outputs,
79+
const std::string& method) override;
80+
81+
std::pair<std::string, std::string> method_names(
82+
const std::unordered_set<std::string>& available) const override;
83+
84+
int64_t max_tokens_per_step(const std::string& method) const override;
85+
int64_t max_tokens_per_step(const std::string& method, int64_t start_pos)
86+
const override;
87+
88+
private:
89+
struct Buffer {
90+
void* data = nullptr;
91+
size_t nbytes = 0;
92+
TensorPtr tensor;
93+
};
94+
// A method's per-step inputs; the caches are shared.
95+
struct Method {
96+
int64_t query_len = 1;
97+
size_t num_inputs = 0;
98+
// Input index -> buffer, for input_ids, the masks and position_ids.
99+
std::map<size_t, Buffer> step_inputs;
100+
size_t input_ids = 0;
101+
size_t position_ids = 0;
102+
std::vector<size_t> masks;
103+
size_t first_cache = 0;
104+
// The logits, when the program leaves them to the runtime.
105+
Buffer logits;
106+
// The last step's real (unpadded) token count.
107+
int64_t valid = 0;
108+
};
109+
110+
runtime::Error load_metadata();
111+
runtime::Error load_method(const std::string& method);
112+
runtime::Result<std::vector<runtime::EValue>> prepare(
113+
const TensorPtr& input,
114+
const TensorPtr& start_pos,
115+
const std::string& method);
116+
void allocate(
117+
const std::vector<std::pair<Buffer*, runtime::TensorInfo>>& buffers);
118+
119+
ET_MODULE_NAMESPACE::Module& module_;
120+
std::unique_ptr<IOBufferAllocator> allocator_;
121+
struct Block {
122+
void* data;
123+
bool from_allocator;
124+
};
125+
std::vector<Block> blocks_;
126+
std::map<std::string, Method> methods_;
127+
// Per layer keys, values, cumulative length; shared by every method.
128+
std::vector<Buffer> caches_;
129+
bool metadata_loaded_ = false;
130+
executorch::aten::ScalarType cache_dtype_ =
131+
executorch::aten::ScalarType::Float;
132+
int32_t num_layers_ = 0;
133+
int64_t context_len_ = 0;
134+
std::vector<float> logits_f_;
135+
TensorPtr logits_f_tensor_;
136+
};
137+
138+
void register_hf_static_io_manager();
139+
140+
} // namespace llm
141+
} // namespace extension
142+
} // namespace executorch

‎extension/llm/runner/io_manager/targets.bzl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,14 @@ def define_common_targets():
1717
],
1818
visibility = ["PUBLIC"],
1919
)
20+
21+
runtime.cxx_library(
22+
name = "hf_static_io_manager" + aten_suffix,
23+
exported_headers = ["hf_static_io_manager.h"],
24+
srcs = ["hf_static_io_manager.cpp"],
25+
exported_deps = [
26+
":io_manager" + aten_suffix,
27+
"//executorch/extension/llm/runner:constants",
28+
],
29+
visibility = ["PUBLIC"],
30+
)

‎extension/llm/runner/llm_runner_helper.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <executorch/extension/llm/runner/image_prefiller.h>
1212
#include <executorch/extension/llm/runner/llm_runner_helper.h>
1313

14+
#include <executorch/extension/llm/runner/io_manager/hf_static_io_manager.h>
1415
#include <executorch/extension/llm/runner/multimodal_decoder_runner.h>
1516
#include <executorch/extension/llm/runner/multimodal_prefiller.h>
1617
#include <executorch/extension/llm/runner/multimodal_runner.h>
@@ -34,7 +35,9 @@ namespace executorch::extension::llm {
3435
using ::executorch::extension::Module;
3536
using ::executorch::runtime::Error;
3637

37-
void register_builtin_io_managers() {}
38+
void register_builtin_io_managers() {
39+
register_hf_static_io_manager();
40+
}
3841

3942
std::unique_ptr<tokenizers::Tokenizer> load_tokenizer(
4043
const std::string& tokenizer_path,

‎extension/llm/runner/targets.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ def define_common_targets():
162162
":text_prefiller" + aten_suffix,
163163
":text_token_generator" + aten_suffix,
164164
"//executorch/extension/llm/runner/io_manager:io_manager" + aten_suffix,
165+
"//executorch/extension/llm/runner/io_manager:hf_static_io_manager" + aten_suffix,
165166
"//executorch/extension/memory_allocator:cpu_caching_allocator",
166167
"//pytorch/tokenizers:hf_tokenizer",
167168
"//pytorch/tokenizers:llama2c_tokenizer",

‎extension/llm/runner/test/BUCK‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,3 +45,50 @@ fbcode_target(_kind = runtime.cxx_test,
4545
"NO_KVCACHE": "$(location fbcode//executorch/test/models:exported_programs[ModuleNoKVCache.pte])",
4646
}
4747
)
48+
49+
fbcode_target(_kind = runtime.python_library,
50+
name = "export_hf_static_program_lib",
51+
srcs = ["export_hf_static_program.py"],
52+
base_module = "executorch.extension.llm.runner.test",
53+
deps = [
54+
"//caffe2:torch",
55+
"//executorch/exir:lib",
56+
],
57+
)
58+
59+
fbcode_target(_kind = runtime.python_binary,
60+
name = "export_hf_static_program",
61+
main_module = "executorch.extension.llm.runner.test.export_hf_static_program",
62+
deps = [":export_hf_static_program_lib"],
63+
)
64+
65+
# A tiny program with Hugging Face's static-cache ABI.
66+
fbcode_target(_kind = runtime.genrule,
67+
name = "hf_static_program",
68+
cmd = "$(exe :export_hf_static_program) --outdir $OUT",
69+
outs = {
70+
"HfStaticTiny.pte": ["HfStaticTiny.pte"],
71+
"HfStaticTinyHalf.pte": ["HfStaticTinyHalf.pte"],
72+
"HfStaticTinyWithState.pte": ["HfStaticTinyWithState.pte"],
73+
},
74+
default_outs = ["."],
75+
)
76+
77+
fbcode_target(_kind = runtime.cxx_test,
78+
name = "test_hf_static_io_manager",
79+
srcs = ["test_hf_static_io_manager.cpp"],
80+
deps = [
81+
"//executorch/extension/llm/runner:runner_lib",
82+
"//executorch/extension/llm/runner/io_manager:hf_static_io_manager",
83+
"//executorch/extension/llm/runner/io_manager:io_manager",
84+
"//executorch/extension/module:module",
85+
"//executorch/extension/tensor:tensor",
86+
"//executorch/kernels/portable:generated_lib",
87+
],
88+
env = {
89+
"ET_HF_STATIC_FIXTURES_DIR": "$(location :hf_static_program)",
90+
"ET_HF_STATIC_TINY_PATH": "$(location :hf_static_program[HfStaticTiny.pte])",
91+
"ET_HF_STATIC_TINY_HALF_PATH": "$(location :hf_static_program[HfStaticTinyHalf.pte])",
92+
"ET_HF_STATIC_TINY_STATE_PATH": "$(location :hf_static_program[HfStaticTinyWithState.pte])",
93+
},
94+
)

‎extension/llm/runner/test/CMakeLists.txt‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)
1919

2020
set(_test_srcs
2121
test_generation_config.cpp
22+
test_hf_static_io_manager.cpp
2223
test_io_manager.cpp
2324
test_model_metadata.cpp
2425
test_text_llm_runner.cpp
@@ -59,30 +60,45 @@ endif()
5960

6061
add_custom_command(
6162
OUTPUT ${_metadata_pte_files}
62-
# Run against the installed executorch (which stages exir/_serialize/*.fbs at
63-
# install time). Do NOT prepend the source tree to PYTHONPATH: it shadows the
64-
# installed package with a source copy that lacks the staged .fbs, breaking
65-
# to_executorch() under a non-editable install.
6663
COMMAND
67-
${_export_prefix} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/export_model_metadata.py
64+
${_export_prefix} ${PYTHON_EXECUTABLE}
65+
${CMAKE_CURRENT_SOURCE_DIR}/export_model_metadata.py
6866
--outdir ${CMAKE_CURRENT_BINARY_DIR}
6967
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
7068
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/export_model_metadata.py
7169
${EXECUTORCH_ROOT}/extension/llm/export/model_metadata.py
7270
)
71+
add_custom_command(
72+
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/hf_static_fixtures.stamp"
73+
COMMAND
74+
${_export_prefix} ${PYTHON_EXECUTABLE}
75+
${CMAKE_CURRENT_SOURCE_DIR}/export_hf_static_program.py --outdir
76+
${CMAKE_CURRENT_BINARY_DIR}
77+
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
78+
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/export_hf_static_program.py
79+
)
7380
add_custom_target(
74-
generated_model_metadata_test_files DEPENDS ${_metadata_pte_files}
81+
generated_model_metadata_test_files
82+
DEPENDS ${_metadata_pte_files}
83+
"${CMAKE_CURRENT_BINARY_DIR}/hf_static_fixtures.stamp"
7584
)
7685

7786
et_cxx_test(
78-
test_runner SOURCES ${_test_srcs} EXTRA_LIBS executorch extension_llm_runner
87+
test_runner
88+
SOURCES
89+
${_test_srcs}
90+
EXTRA_LIBS
91+
executorch
92+
extension_llm_runner
93+
portable_kernels
94+
portable_ops_lib
7995
)
8096
add_dependencies(test_runner generated_model_metadata_test_files)
8197
set_property(
8298
TEST test_runner
8399
PROPERTY
84100
ENVIRONMENT
85-
"ET_MODEL_METADATA_FULL_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_full.pte;ET_MODEL_METADATA_LAST_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_last.pte;ET_MODEL_METADATA_SELECTED_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_selected.pte;ET_MODEL_METADATA_INVALID_CONTEXT_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_context.pte;ET_MODEL_METADATA_INVALID_PREFILL_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_prefill.pte;ET_MODEL_METADATA_INVALID_VOCAB_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_vocab.pte;ET_MODEL_METADATA_MISSING_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_missing.pte;ET_MODEL_METADATA_GEOMETRY_WRONG_TYPE_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_wrong_type.pte;ET_MODEL_METADATA_GEOMETRY_MISMATCHED_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_mismatched.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_HEADS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_heads.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_DIMS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_dims.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_WINDOWS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_windows.pte;ET_MODEL_METADATA_GEOMETRY_EMPTY_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_empty.pte"
101+
"ET_MODEL_METADATA_FULL_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_full.pte;ET_MODEL_METADATA_LAST_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_last.pte;ET_MODEL_METADATA_SELECTED_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_selected.pte;ET_MODEL_METADATA_INVALID_CONTEXT_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_context.pte;ET_MODEL_METADATA_INVALID_PREFILL_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_prefill.pte;ET_MODEL_METADATA_INVALID_VOCAB_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_invalid_vocab.pte;ET_MODEL_METADATA_MISSING_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_missing.pte;ET_MODEL_METADATA_GEOMETRY_WRONG_TYPE_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_wrong_type.pte;ET_MODEL_METADATA_GEOMETRY_MISMATCHED_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_mismatched.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_HEADS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_heads.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_DIMS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_dims.pte;ET_MODEL_METADATA_GEOMETRY_INVALID_WINDOWS_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_invalid_windows.pte;ET_MODEL_METADATA_GEOMETRY_EMPTY_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModelMetadata_geometry_empty.pte;ET_HF_STATIC_TINY_PATH=${CMAKE_CURRENT_BINARY_DIR}/HfStaticTiny.pte;ET_HF_STATIC_TINY_HALF_PATH=${CMAKE_CURRENT_BINARY_DIR}/HfStaticTinyHalf.pte;ET_HF_STATIC_TINY_STATE_PATH=${CMAKE_CURRENT_BINARY_DIR}/HfStaticTinyWithState.pte;ET_HF_STATIC_FIXTURES_DIR=${CMAKE_CURRENT_BINARY_DIR}"
86102
)
87103

88104
# Override sanitizer to this issue:

0 commit comments

Comments
 (0)