Skip to content

Commit 1d55f9c

Browse files
authored
Add a Native CPU engine backed by CPUPlan (#23935)
Summary: Add `CPUEngineHost` and `CPUEngineContext` using the existing CPU backend's provider selection, memory planning, and execution. Consume standard Native preprocessing output by materializing supported views and functionalizing supported in-place activations during engine compilation. The engine integrates with `Model`/`Session`, shares package constants across methods, writes inputs once into dedicated buffers outside the activation arena so they persist across repeated execution, and converts host dtypes with checked integer narrowing. Support static, unquantized graphs with contiguous constants and public I/O; reject unsupported aliasing and mutation semantics. `current_execution_context` moves from `CPUBackend.cpp` into `cpu_runtime` so `CPUBackend` and the engine read the threadpool the same way, and the unused `ExecutionContext::avx2_fma` flag, which no provider reads, is removed. `tensor_bytes` no longer checks dtypes; `CPUBackend` keeps its own check for the FP32, integer, and boolean dtypes it supports. Alias handling lives in the shared `functionalize_aliases` pass (`runtime/graph/utils/Aliases.h`), which also hosts the `alias_root`/`has_mutated_alias_use` helpers that Vulkan's `MaterializeViewCopies` now uses instead of its local copies. A mutation is functionalized only when every live alias reaches the mutated value through `view`s alone with the same shape, so the decision does not depend on activation dim-order hints. `as_strided` accepts operands that index their root's storage in order, including the output of a functionalized mutation. The pass has direct graph-level unit tests in `aliases_test`. Add CMake libraries and installed targets for the CPU runtime, providers, and engine. `EXECUTORCH_BUILD_NATIVE_CPU` requires XNNPACK and optimized kernels. Register optimized kernels wherever available and portable kernels only for operators absent from the optimized list. Shared builds resolve CPU lookups through `executorch_shared`. Include the CPU engine integration suite in CTest. Authored with OpenAI Codex and Claude Code. Reviewed By: digantdesai Differential Revision: D125540986 Pull Request resolved: #23935
1 parent 6a2784e commit 1d55f9c

31 files changed

Lines changed: 2848 additions & 86 deletions

‎backends/cpu/CMakeLists.txt‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
add_library(
8+
cpu_runtime runtime/KernelProvider.cpp runtime/CPUPlan.cpp
9+
runtime/op_helpers/Convolution.cpp
10+
)
11+
target_link_libraries(
12+
cpu_runtime
13+
PUBLIC native_runtime
14+
PRIVATE extension_threadpool
15+
)
16+
if(EXECUTORCH_BUILD_SHARED)
17+
target_link_libraries(cpu_runtime PUBLIC executorch_shared)
18+
else()
19+
target_link_libraries(cpu_runtime PUBLIC executorch_core)
20+
endif()
21+
22+
add_library(cpu_reference runtime/providers/reference/ReferenceProvider.cpp)
23+
target_link_libraries(cpu_reference PUBLIC cpu_runtime)
24+
25+
add_library(cpu_xnnpack runtime/providers/xnnpack/XNNPACKProvider.cpp)
26+
target_link_libraries(
27+
cpu_xnnpack
28+
PUBLIC cpu_runtime
29+
PRIVATE extension_threadpool
30+
)
31+
if(EXECUTORCH_BUILD_SHARED)
32+
target_link_libraries(cpu_xnnpack PRIVATE xnnpack_shared)
33+
else()
34+
target_link_libraries(cpu_xnnpack PRIVATE XNNPACK)
35+
endif()
36+
37+
foreach(target cpu_runtime cpu_reference cpu_xnnpack)
38+
add_library(executorch::${target} ALIAS ${target})
39+
target_compile_features(${target} PUBLIC cxx_std_20)
40+
target_compile_options(
41+
${target} PRIVATE $<$<CXX_COMPILER_ID:MSVC>:/EHsc>
42+
$<$<NOT:$<CXX_COMPILER_ID:MSVC>>:-fexceptions>
43+
)
44+
endforeach()
45+
46+
install(
47+
TARGETS cpu_runtime cpu_reference cpu_xnnpack
48+
EXPORT ExecuTorchTargets
49+
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
50+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
51+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
52+
)
53+
install(
54+
DIRECTORY runtime/
55+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/executorch/backends/cpu/runtime
56+
FILES_MATCHING
57+
PATTERN "*.h"
58+
PATTERN "test" EXCLUDE
59+
)

‎backends/cpu/runtime/CPUBackend.cpp‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66

77
#include <executorch/backends/cpu/runtime/CPUPlan.h>
88
#include <executorch/backends/native/runtime/Program.h>
9-
#include <executorch/extension/threadpool/threadpool.h>
109

1110
#include <algorithm>
1211
#include <chrono>
@@ -32,20 +31,6 @@ std::optional<RuntimeConfigurationFactory>& runtime_configuration_factory() {
3231
return factory;
3332
}
3433

35-
ExecutionContext execution_context(runtime::EventTracer* tracer = nullptr) {
36-
ExecutionContext context;
37-
context.threadpool = extension::threadpool::get_pthreadpool();
38-
context.threads = context.threadpool
39-
? extension::threadpool::get_threadpool()->get_thread_count()
40-
: 1;
41-
#if defined(__x86_64__) || defined(__i386__)
42-
context.avx2_fma =
43-
__builtin_cpu_supports("avx2") && __builtin_cpu_supports("fma");
44-
#endif
45-
context.event_tracer = tracer;
46-
return context;
47-
}
48-
4934
class CPUExecution {
5035
public:
5136
explicit CPUExecution(ptn::Program program) : program_(std::move(program)) {}
@@ -78,6 +63,19 @@ class CPUExecution {
7863
NotSupported,
7964
"CPU static delegate requires unaliased values: %s",
8065
value.name.c_str());
66+
if (value.is_tensor()) {
67+
const auto dtype = value.tensor_meta().dtype;
68+
ET_CHECK_OR_RETURN_ERROR(
69+
dtype == ptn::ScalarType::Float || dtype == ptn::ScalarType::Byte ||
70+
dtype == ptn::ScalarType::Char ||
71+
dtype == ptn::ScalarType::Short ||
72+
dtype == ptn::ScalarType::Int ||
73+
dtype == ptn::ScalarType::Long ||
74+
dtype == ptn::ScalarType::Bool,
75+
NotSupported,
76+
"CPU static delegate requires a supported dtype: %s",
77+
value.name.c_str());
78+
}
8179
auto bytes = tensor_bytes(value);
8280
if (!bytes.ok()) {
8381
return bytes.error();
@@ -95,7 +93,7 @@ class CPUExecution {
9593
return error;
9694
}
9795
plan_ = std::make_unique<CPUPlan>(
98-
graph, buffers_, std::move(configuration), execution_context());
96+
graph, buffers_, std::move(configuration), current_execution_context());
9997
error = plan_->select();
10098
if (error != Error::Ok) {
10199
return error;
@@ -219,7 +217,7 @@ class CPUExecution {
219217
}
220218
}
221219
const auto error =
222-
plan_->execute(execution_context(context.event_tracer()));
220+
plan_->execute(current_execution_context(context.event_tracer()));
223221
if (error != Error::Ok) {
224222
return error;
225223
}

‎backends/cpu/runtime/CPUPlan.cpp‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,6 @@ Error CPUPlan::execute(const ExecutionContext& context) {
145145
ET_CHECK_OR_RETURN_ERROR(prepared_, InvalidState, "CPU plan is not prepared");
146146
ET_CHECK_OR_RETURN_ERROR(
147147
context.threads == execution_.threads &&
148-
context.avx2_fma == execution_.avx2_fma &&
149148
context.threadpool == execution_.threadpool,
150149
NotSupported,
151150
"CPU execution properties changed; reload the static method");
@@ -609,9 +608,8 @@ std::string CPUPlan::describe(double preparation_ms) const {
609608
"\"xnn_workspace_shared\":true,\"xnn_packed_cache\":\"per_plan\","
610609
"\"delegate_io\":\"copy_for_unknown_external_tail_capacity\","
611610
"\"selection_context\":{\"threads\":" +
612-
std::to_string(execution_.threads) +
613-
",\"avx2_fma\":" + (execution_.avx2_fma ? "true" : "false") +
614-
"},\"candidates\":[" + decisions_ + "]}";
611+
std::to_string(execution_.threads) + "},\"candidates\":[" + decisions_ +
612+
"]}";
615613
return report;
616614
}
617615
} // namespace executorch::backends::cpu

‎backends/cpu/runtime/KernelProvider.cpp‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,24 @@
66

77
#include <executorch/backends/cpu/runtime/KernelProvider.h>
88
#include <executorch/backends/native/runtime/graph/ScalarType.h>
9+
#include <executorch/extension/threadpool/threadpool.h>
910

1011
#include <algorithm>
1112
#include <cstring>
1213

1314
namespace executorch::backends::cpu {
1415
using namespace executorch::runtime;
1516

17+
ExecutionContext current_execution_context(EventTracer* tracer) {
18+
ExecutionContext context;
19+
context.threadpool = extension::threadpool::get_pthreadpool();
20+
context.threads = context.threadpool
21+
? extension::threadpool::get_threadpool()->get_thread_count()
22+
: 1;
23+
context.event_tracer = tracer;
24+
return context;
25+
}
26+
1627
bool BufferRequirements::valid() const {
1728
return alignment >= kBufferAlignment && !(alignment & (alignment - 1));
1829
}
@@ -91,16 +102,9 @@ Result<size_t> tensor_bytes(const ptn::Value& value) {
91102
value.name.c_str());
92103
const auto& meta = value.tensor_meta();
93104
ET_CHECK_OR_RETURN_ERROR(
94-
(meta.dtype == ptn::ScalarType::Float ||
95-
meta.dtype == ptn::ScalarType::Byte ||
96-
meta.dtype == ptn::ScalarType::Char ||
97-
meta.dtype == ptn::ScalarType::Short ||
98-
meta.dtype == ptn::ScalarType::Int ||
99-
meta.dtype == ptn::ScalarType::Long ||
100-
meta.dtype == ptn::ScalarType::Bool) &&
101-
meta.is_contiguous() && meta.sizes.size() <= 16,
105+
meta.is_contiguous() && meta.sizes.size() <= 16,
102106
NotSupported,
103-
"CPU static delegate requires supported dtype and contiguous rank <= 16: %s",
107+
"CPU static delegate requires contiguous rank <= 16: %s",
104108
value.name.c_str());
105109
size_t elements = 1;
106110
for (auto dimension = meta.sizes.rbegin(); dimension != meta.sizes.rend();
@@ -114,11 +118,11 @@ Result<size_t> tensor_bytes(const ptn::Value& value) {
114118
value.name.c_str());
115119
elements *= size;
116120
}
117-
const size_t element_size = ptn::element_size(meta.dtype);
121+
const auto element_size = ptn::element_size(meta.dtype);
118122
ET_CHECK_OR_RETURN_ERROR(
119123
elements <= SIZE_MAX / element_size,
120124
NotSupported,
121-
"CPU tensor byte size overflows: %s",
125+
"CPU tensor byte size overflows size_t: %s",
122126
value.name.c_str());
123127
return elements * element_size;
124128
}

‎backends/cpu/runtime/KernelProvider.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,11 +64,15 @@ runtime::Result<Buffer> allocate_buffer(
6464
/// Threadpool lifetime covers preparation and every invocation of the plan.
6565
struct ExecutionContext {
6666
size_t threads = 1;
67-
bool avx2_fma = false;
6867
pthreadpool* threadpool = nullptr;
6968
runtime::EventTracer* event_tracer = nullptr;
7069
};
7170

71+
/// Reads the process thread pool. Call it for every execution: the thread
72+
/// pool can be reset between executions.
73+
ExecutionContext current_execution_context(
74+
runtime::EventTracer* tracer = nullptr);
75+
7276
struct Support {
7377
bool supported;
7478
std::string_view reason;
@@ -135,7 +139,7 @@ class KernelImplementation {
135139
return false;
136140
}
137141
/// Pure metadata query, before compile can read constants or allocate
138-
/// storage. The default declares contiguous tensor storage for every region
142+
/// storage. The default declares contiguous storage for every region
139143
/// boundary.
140144
virtual runtime::Result<StorageRequirements> requirements(
141145
const KernelRegion& region,

‎backends/cpu/runtime/test/ProviderTest.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ class ProviderTest : public ::testing::Test {
6666
std::vector<EValue> values;
6767
std::vector<Buffer> buffers;
6868
size_t private_bytes = 0;
69-
ExecutionContext execution_context{1, true};
69+
ExecutionContext execution_context;
7070
};
7171

7272
// Cppcheck cannot expand GTest fixture macros without build headers.

‎backends/cpu/targets.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ def define_common_targets():
1919
"//executorch/backends/native/runtime:runtime",
2020
"//executorch/runtime/backend:interface",
2121
],
22+
deps = ["//executorch/extension/threadpool:threadpool"],
2223
visibility = ["PUBLIC"],
2324
)
2425

2526
runtime.cxx_library(
2627
name = "cpu_engine",
2728
srcs = ["runtime/CPUBackend.cpp"],
2829
exported_deps = [":cpu_runtime"],
29-
deps = ["//executorch/extension/threadpool:threadpool"],
3030
visibility = ["PUBLIC"],
3131
# @lint-ignore BUCKLINT: Backend registration runs at static initialization.
3232
link_whole = True,

‎backends/native/CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ endif()
1818

1919
add_subdirectory(runtime)
2020

21+
if(EXECUTORCH_BUILD_NATIVE_CPU)
22+
add_subdirectory(runtime/cpu)
23+
endif()
24+
2125
if(EXECUTORCH_BUILD_NATIVE_VULKAN)
2226
add_subdirectory(runtime/vulkan)
2327
endif()
@@ -32,6 +36,7 @@ install(
3236
FILES_MATCHING
3337
PATTERN "*.h"
3438
PATTERN "test" EXCLUDE
39+
PATTERN "cpu" EXCLUDE
3540
PATTERN "vulkan" EXCLUDE
3641
PATTERN "ggml" EXCLUDE
3742
)

‎backends/native/runtime/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ add_library(
3838
graph/Scalar.cpp
3939
graph/TensorMeta.cpp
4040
graph/Value.cpp
41+
graph/utils/Aliases.cpp
4142
graph/utils/GraphUtils.cpp
4243
)
4344

‎backends/native/runtime/cpu/BUCK‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target", "non_fbcode_target")
2+
load(":targets.bzl", "define_common_targets")
3+
4+
oncall("executorch")
5+
6+
non_fbcode_target(_kind = define_common_targets)
7+
fbcode_target(_kind = define_common_targets)

0 commit comments

Comments
 (0)