Skip to content

Commit 88e6748

Browse files
authored
[executorch][native-ggml] Share the ggml runners' engine flags
Stack from [ghstack](https://github.com/ezyang/ghstack/tree/0.15.0) (oldest at bottom): * #25381 * #25380 * #25379 * #25378 * #25377 * #25376 * #25375 * #25374 * #25373 * #25372 * #25371 * __->__ #25370 * #25369 * #25368 run_hf_llm_ggml and ggml_ptn_runner take their engine flags (--device, --device_index, --cpu_threads, --max_cached_graphs, --print_graph_ops) from one GgmlEngineFlags library, and the README gains a table of the engine options that change numerics. Both runners then accept the same flags with the same meaning and help, and a later option is added and documented once. Authored with assistance from Claude and Codex. Differential Revision: D133480460 Pull Request resolved: #25370
1 parent 9294050 commit 88e6748

10 files changed

Lines changed: 112 additions & 72 deletions

File tree

‎backends/native/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,15 @@ ExecuTorch with the desired GGML backend (for example, `GGML_CUDA=ON`,
268268
`GGML_HIP=ON`, or `GGML_METAL=ON`) and set `ptn::GgmlEngineOptions::device` to
269269
`ptn::GgmlDevice::Gpu`. CPU support is required even when selecting a GPU.
270270

271+
The ggml runners (`run_hf_llm_ggml`, `ggml_ptn_runner`) share their engine
272+
flags (`runtime/engine/ggml/GgmlEngineFlags.h`). Engine options that change
273+
numerics default to llama.cpp's behaviour; the testing ones end in
274+
`_for_testing` and turn on eager's exact behaviour where ggml's differs.
275+
276+
| Option (flag) | Default, as in llama.cpp | Set | Ops | Backends |
277+
| --- | --- | --- | --- | --- |
278+
| `allow_activation_quant_for_weight_only` | quantizes the activations of matmuls with weight-only quantized weights in their block format | false (debugging): those weights load dequantized to F32, about 4-12x their packed size by type | `linear` | all |
279+
271280
The CMake installation includes the GGML libraries and package configuration.
272281
It also copies the submodule's upstream MIT license, including its copyright
273282
notice, to `share/licenses/executorch/ggml/LICENSE` under the installation

‎backends/native/examples/hf_llm/runtime/BUCK‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ fbcode_target(
6363
srcs = ["run_hf_llm_ggml.cpp"],
6464
deps = [
6565
":llm_runner",
66-
"//executorch/backends/native/runtime/engine:ggml_engine_cpu",
66+
"//executorch/backends/native/runtime/engine:ggml_engine_flags_cpu",
6767
],
6868
external_deps = ["gflags"],
6969
visibility = ["PUBLIC"],
@@ -80,7 +80,7 @@ fbcode_target(
8080
# library beside the binary. Buck does not stage the CUDA driver stub's
8181
# library; add its directory to LD_LIBRARY_PATH when running.
8282
"fbsource//third-party/llama.cpp:ggml_base[prefer-shared]",
83-
"//executorch/backends/native/runtime/engine:ggml_engine_cuda",
83+
"//executorch/backends/native/runtime/engine:ggml_engine_flags_cuda",
8484
],
8585
external_deps = [
8686
"gflags",
@@ -98,7 +98,7 @@ fbcode_target(
9898
srcs = ["run_hf_llm_ggml.cpp"],
9999
deps = [
100100
":llm_runner",
101-
"//executorch/backends/native/runtime/engine:ggml_engine_hip",
101+
"//executorch/backends/native/runtime/engine:ggml_engine_flags_hip",
102102
],
103103
external_deps = ["gflags"],
104104
labels = ggml_ci_labels("hip"),
@@ -114,7 +114,7 @@ non_fbcode_target(
114114
srcs = ["run_hf_llm_ggml.cpp"],
115115
deps = [
116116
":llm_runner",
117-
"//executorch/backends/native/runtime/engine:ggml_engine_metal",
117+
"//executorch/backends/native/runtime/engine:ggml_engine_flags_metal",
118118
],
119119
external_deps = ["gflags"],
120120
labels = ggml_ci_labels("metal"),

‎backends/native/examples/hf_llm/runtime/CMakeLists.txt‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,11 @@ if(EXECUTORCH_BUILD_NATIVE_VULKAN)
5151
list(APPEND _llm_runners run_hf_llm_vulkan)
5252
endif()
5353
if(EXECUTORCH_BUILD_NATIVE_GGML)
54-
add_executable(run_hf_llm_ggml run_hf_llm_ggml.cpp)
54+
add_executable(
55+
run_hf_llm_ggml
56+
run_hf_llm_ggml.cpp
57+
${EXECUTORCH_ROOT}/backends/native/runtime/engine/ggml/GgmlEngineFlags.cpp
58+
)
5559
target_link_libraries(run_hf_llm_ggml PRIVATE native_ggml_engine)
5660
list(APPEND _llm_runners run_hf_llm_ggml)
5761
endif()

‎backends/native/examples/hf_llm/runtime/run_hf_llm_ggml.cpp‎

Lines changed: 5 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,25 +4,11 @@
44
// This source code is licensed under the BSD-style license found in the
55
// LICENSE file in the root directory of this source tree.
66

7-
#include <stdexcept>
8-
97
#include <executorch/backends/native/examples/hf_llm/runtime/LlmRunner.h>
108
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngine.h>
9+
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngineFlags.h>
1110
#include <gflags/gflags.h>
1211

13-
DEFINE_string(
14-
device,
15-
"cpu",
16-
"ggml device: cpu, or gpu for the GPU backend this binary links.");
17-
DEFINE_int32(device_index, 0, "GPU device index.");
18-
DEFINE_int32(
19-
cpu_threads,
20-
0,
21-
"ggml CPU threads; zero keeps the backend default.");
22-
DEFINE_int32(
23-
max_cached_graphs,
24-
1,
25-
"Graphs kept per executable by input shapes; 0 rebuilds every call.");
2612
DEFINE_bool(
2713
allow_activation_quant_for_weight_only,
2814
false,
@@ -34,25 +20,10 @@ int main(int argc, char** argv) {
3420
"Generate text from an LLM .ptn with the native ggml engine.");
3521
gflags::ParseCommandLineFlags(&argc, &argv, true);
3622
ptn::register_ggml_cache_builders();
23+
ptn::GgmlEngineOptions options = ptn::ggml_engine_options_from_flags();
24+
options.allow_activation_quant_for_weight_only =
25+
FLAGS_allow_activation_quant_for_weight_only;
3726
return ptn::run_llm(
38-
[] {
39-
ptn::GgmlEngineOptions options;
40-
if (FLAGS_device == "gpu") {
41-
options.device = ptn::GgmlDevice::Gpu;
42-
} else if (FLAGS_device != "cpu") {
43-
throw std::invalid_argument("--device must be cpu or gpu");
44-
}
45-
options.device_index = FLAGS_device_index;
46-
options.cpu_threads = FLAGS_cpu_threads;
47-
if (FLAGS_max_cached_graphs < 0) {
48-
throw std::invalid_argument(
49-
"--max_cached_graphs must be non-negative");
50-
}
51-
options.max_cached_graphs =
52-
static_cast<size_t>(FLAGS_max_cached_graphs);
53-
options.allow_activation_quant_for_weight_only =
54-
FLAGS_allow_activation_quant_for_weight_only;
55-
return ptn::GgmlEngineHost::create(options);
56-
},
27+
[options] { return ptn::GgmlEngineHost::create(options); },
5728
ptn::kGgmlCacheBackendId);
5829
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngineFlags.h>
8+
9+
#include <stdexcept>
10+
11+
#include <gflags/gflags.h>
12+
13+
DEFINE_string(
14+
device,
15+
"cpu",
16+
"ggml device: cpu, or gpu for the GPU backend this binary links.");
17+
DEFINE_int32(device_index, 0, "GPU device index.");
18+
DEFINE_int32(cpu_threads, 0, "ggml CPU threads; 0 keeps ggml's default of 4.");
19+
DEFINE_int32(
20+
max_cached_graphs,
21+
1,
22+
"Graphs kept per executable by input shapes; 0 rebuilds every call.");
23+
DEFINE_bool(
24+
print_graph_ops,
25+
false,
26+
"Print each built graph's node count per ggml op to stderr.");
27+
28+
namespace ptn {
29+
30+
GgmlEngineOptions ggml_engine_options_from_flags() {
31+
GgmlEngineOptions options;
32+
if (FLAGS_device == "gpu") {
33+
options.device = GgmlDevice::Gpu;
34+
} else if (FLAGS_device != "cpu") {
35+
throw std::invalid_argument("--device must be cpu or gpu");
36+
}
37+
options.device_index = FLAGS_device_index;
38+
options.cpu_threads = FLAGS_cpu_threads;
39+
if (FLAGS_max_cached_graphs < 0) {
40+
throw std::invalid_argument("--max_cached_graphs must be non-negative");
41+
}
42+
options.max_cached_graphs = static_cast<size_t>(FLAGS_max_cached_graphs);
43+
options.print_graph_ops = FLAGS_print_graph_ops;
44+
return options;
45+
}
46+
47+
} // namespace ptn
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Copyright (c) Meta Platforms, Inc. and affiliates.
2+
// All rights reserved.
3+
//
4+
// This source code is licensed under the BSD-style license found in the
5+
// LICENSE file in the root directory of this source tree.
6+
7+
#pragma once
8+
9+
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngine.h>
10+
11+
namespace ptn {
12+
13+
// The GgmlEngineOptions that the command-line flags every ggml runner shares
14+
// select, so that each runner accepts them with the same meaning and help.
15+
GgmlEngineOptions ggml_engine_options_from_flags();
16+
17+
} // namespace ptn

‎backends/native/runtime/engine/targets.bzl‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,3 +96,16 @@ def define_common_targets():
9696
visibility = ["//executorch/backends/native/..."],
9797
**compat
9898
)
99+
100+
# The command-line flags the ggml runners share.
101+
runtime.cxx_library(
102+
name = "ggml_engine_flags_" + variant,
103+
srcs = ["ggml/GgmlEngineFlags.cpp"],
104+
exported_headers = ["ggml/GgmlEngineFlags.h"],
105+
exported_deps = [":ggml_engine_" + variant],
106+
external_deps = ["gflags"],
107+
labels = ggml_ci_labels(variant),
108+
platforms = None if variant == "cpu" else [CXX],
109+
visibility = ["//executorch/backends/native/..."],
110+
**compat
111+
)

‎backends/native/test/CMakeLists.txt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,9 +153,14 @@ if(EXECUTORCH_BUILD_NATIVE_GGML)
153153
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
154154
)
155155

156-
add_executable(native_ggml_sample_test runtime/ggml/ggml_ptn_runner.cpp)
156+
add_executable(
157+
native_ggml_sample_test
158+
runtime/ggml/ggml_ptn_runner.cpp
159+
${EXECUTORCH_ROOT}/backends/native/runtime/engine/ggml/GgmlEngineFlags.cpp
160+
)
157161
target_link_libraries(
158162
native_ggml_sample_test PRIVATE native_sample_test_main native_ggml_engine
163+
${_native_test_gflags}
159164
)
160165
add_dependencies(native_ggml_sample_test native_op_samples)
161166
add_test(NAME native_ggml_sample_test

‎backends/native/test/runtime/ggml/BUCK‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,10 @@ _VARIANTS = [
7373
link_whole = True,
7474
visibility = ["//executorch/backends/native/test/..."],
7575
exported_deps = ["//executorch/backends/native/test/runtime:engine_host"],
76-
deps = ["//executorch/backends/native/runtime/engine:" + v["engine"]] +
76+
deps = [
77+
"//executorch/backends/native/runtime/engine:" +
78+
v["engine"].replace("ggml_engine_", "ggml_engine_flags_"),
79+
] +
7780
v.get("deps", []),
7881
external_deps = ["gflags"] + v.get("external_deps", []),
7982
labels = v.get("labels", []),

‎backends/native/test/runtime/ggml/ggml_ptn_runner.cpp‎

Lines changed: 2 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,44 +6,15 @@
66

77
// The ggml engine host for PTN test drivers.
88

9-
#include <stdexcept>
10-
119
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngine.h>
10+
#include <executorch/backends/native/runtime/engine/ggml/GgmlEngineFlags.h>
1211
#include <executorch/backends/native/test/runtime/PtnRunner.h>
13-
#include <gflags/gflags.h>
14-
15-
DEFINE_string(
16-
device,
17-
"cpu",
18-
"ggml device: cpu, or gpu for the GPU backend this binary links.");
19-
DEFINE_int32(device_index, 0, "GPU device index.");
20-
DEFINE_int32(cpu_threads, 0, "ggml CPU threads; 0 keeps ggml's default.");
21-
DEFINE_int32(
22-
max_cached_graphs,
23-
1,
24-
"Graphs kept per executable by input shapes; 0 rebuilds every call.");
25-
DEFINE_bool(
26-
print_graph_ops,
27-
false,
28-
"Print each built graph's node count per ggml op to stderr.");
2912

3013
namespace ptn::testing {
3114

3215
// cppcheck-suppress unusedFunction
3316
std::unique_ptr<EngineHost> make_engine_host(const HostOptions& host_options) {
34-
GgmlEngineOptions options;
35-
if (FLAGS_device == "gpu") {
36-
options.device = GgmlDevice::Gpu;
37-
} else if (FLAGS_device != "cpu") {
38-
throw std::invalid_argument("--device must be cpu or gpu");
39-
}
40-
options.device_index = FLAGS_device_index;
41-
options.cpu_threads = FLAGS_cpu_threads;
42-
if (FLAGS_max_cached_graphs < 0) {
43-
throw std::invalid_argument("--max_cached_graphs must be non-negative");
44-
}
45-
options.max_cached_graphs = static_cast<size_t>(FLAGS_max_cached_graphs);
46-
options.print_graph_ops = FLAGS_print_graph_ops;
17+
GgmlEngineOptions options = ggml_engine_options_from_flags();
4718
options.allow_activation_quant_for_weight_only =
4819
host_options.allow_activation_quant_for_weight_only;
4920
return GgmlEngineHost::create(options);

0 commit comments

Comments
 (0)