Skip to content

Commit 149bba5

Browse files
authored
Release packed-only constants after CPU backend preparation (#24235)
Summary: Let providers mark immutable constants as `preparation_only` and release their original loader buffers after successful plan preparation. Retain originals required by runtime consumers or graph outputs. Enable this contract for XNNPACK linear weights and biases, and report `released_constant_bytes`. Authored with AI assistance using Codex. Reviewed By: digantdesai Differential Revision: D126926759 Pull Request resolved: #24235
1 parent 670d0bd commit 149bba5

7 files changed

Lines changed: 222 additions & 4 deletions

File tree

‎backends/cpu/runtime/CPUBackend.cpp‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include <string>
2020
#include <string_view>
2121
#include <unordered_set>
22+
#include <utility>
2223

2324
namespace executorch::backends::cpu {
2425
namespace {
@@ -140,12 +141,19 @@ class CPUExecution {
140141
copied_constant_bytes_ += buffer.readable_bytes;
141142
}
142143
buffers_[id] = buffer;
143-
constants_.push_back(std::move(data.get()));
144+
constants_.emplace_back(id, std::move(data.get()));
144145
}
145146
error = plan_->prepare(allocator);
146147
if (error != Error::Ok) {
147148
return error;
148149
}
150+
for (auto& [id, constant] : constants_) {
151+
if (plan_->can_release_constant(id)) {
152+
released_constant_bytes_ += constant.size();
153+
buffers_[id] = {};
154+
constant.Free();
155+
}
156+
}
149157
const double milliseconds = std::chrono::duration<double, std::milli>(
150158
std::chrono::steady_clock::now() - start)
151159
.count();
@@ -233,7 +241,8 @@ class CPUExecution {
233241
report.pop_back();
234242
report +=
235243
",\"copied_constant_bytes\":" + std::to_string(copied_constant_bytes_) +
236-
"}";
244+
",\"released_constant_bytes\":" +
245+
std::to_string(released_constant_bytes_) + "}";
237246
constexpr size_t chunk_bytes = 80;
238247
constexpr std::string_view hex = "0123456789abcdef";
239248
const size_t chunks = (report.size() + chunk_bytes - 1) / chunk_bytes;
@@ -252,12 +261,13 @@ class CPUExecution {
252261

253262
ptn::Program program_;
254263
const ptn::Method* method_ = nullptr;
255-
std::vector<FreeableBuffer> constants_;
264+
std::vector<std::pair<ptn::ValueId, FreeableBuffer>> constants_;
256265
std::vector<size_t> bytes_;
257266
std::vector<Buffer> buffers_;
258267
std::vector<ptn::ValueId> input_ids_;
259268
std::unique_ptr<CPUPlan> plan_;
260269
size_t copied_constant_bytes_ = 0;
270+
size_t released_constant_bytes_ = 0;
261271
std::mutex mutex_;
262272
};
263273

‎backends/cpu/runtime/CPUPlan.cpp‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,32 @@ Error CPUPlan::prepare(MemoryAllocator& allocator) {
115115
return Error::Ok;
116116
}
117117

118+
bool CPUPlan::can_release_constant(ptn::ValueId id) const {
119+
if (!prepared_ || id >= graph_.values.size() ||
120+
std::find(graph_.output_ids.begin(), graph_.output_ids.end(), id) !=
121+
graph_.output_ids.end()) {
122+
return false;
123+
}
124+
const auto role = graph_.value(id).role;
125+
if (role != ptn::ValueRole::Parameter &&
126+
role != ptn::ValueRole::ConstantTensor &&
127+
role != ptn::ValueRole::Buffer) {
128+
return false;
129+
}
130+
bool consumed = false;
131+
for (const auto& step : steps_) {
132+
for (const auto& value : step.storage.values) {
133+
if (value.id == id) {
134+
if (!value.preparation_only) {
135+
return false;
136+
}
137+
consumed = true;
138+
}
139+
}
140+
}
141+
return consumed;
142+
}
143+
118144
Error CPUPlan::execute(const ExecutionContext& context) {
119145
ET_CHECK_OR_RETURN_ERROR(prepared_, InvalidState, "CPU plan is not prepared");
120146
ET_CHECK_OR_RETURN_ERROR(
@@ -125,6 +151,9 @@ Error CPUPlan::execute(const ExecutionContext& context) {
125151
"CPU execution properties changed; reload the static method");
126152
for (auto& step : steps_) {
127153
for (const auto& value : step.storage.values) {
154+
if (value.preparation_only) {
155+
continue;
156+
}
128157
ET_CHECK_OR_RETURN_ERROR(
129158
buffers_.at(value.id).accepts(requirements_.at(value.id)),
130159
InvalidArgument,
@@ -393,6 +422,15 @@ Error CPUPlan::select() {
393422
InvalidArgument,
394423
"Invalid CPU storage requirement");
395424
requirements_[value.id].merge(value.buffer);
425+
const auto role = graph.value(value.id).role;
426+
ET_CHECK_OR_RETURN_ERROR(
427+
!value.preparation_only ||
428+
(value.buffer.writable_bytes == 0 &&
429+
(role == ptn::ValueRole::Parameter ||
430+
role == ptn::ValueRole::ConstantTensor ||
431+
role == ptn::ValueRole::Buffer)),
432+
InvalidArgument,
433+
"Only immutable constants can have preparation-only storage");
396434
}
397435
for (auto id : step.region.outputs) {
398436
auto bytes = tensor_bytes(graph.value(id));

‎backends/cpu/runtime/CPUPlan.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ class CPUPlan {
4141
runtime::Error prepare(runtime::MemoryAllocator& allocator);
4242
runtime::Error execute(const ExecutionContext& context);
4343
std::string describe(double preparation_ms) const;
44+
/// True only after preparation, when no selected region or graph output
45+
/// needs the original constant. The caller owns and releases its storage.
46+
bool can_release_constant(ptn::ValueId id) const;
4447

4548
const std::vector<PlanStep>& steps() const {
4649
return steps_;

‎backends/cpu/runtime/KernelProvider.h‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ runtime::Result<BufferRequirements> tensor_requirements(
4040
enum class StorageOwner { Borrowed, ConstantCopy, Activation, Scratch };
4141

4242
/// Capacities are measured from data. Readable bytes are initialized; writes
43-
/// require their own declared extent. The owner retains storage through run.
43+
/// require their own declared extent. The owner retains storage for its
44+
/// declared lifetime.
4445
struct Buffer {
4546
void* data = nullptr;
4647
size_t readable_bytes = 0;
@@ -86,6 +87,9 @@ struct KernelRegion {
8687
struct ValueRequirement {
8788
ptn::ValueId id = ptn::kInvalid;
8889
BufferRequirements buffer;
90+
/// Immutable constant needed only through successful plan preparation.
91+
/// bind/run must not read it; other regions may still require the original.
92+
bool preparation_only = false;
8993
};
9094

9195
/// Scratch is invocation-local and reusable immediately after run returns.

‎backends/cpu/runtime/providers/xnnpack/XNNPACKProvider.cpp‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,41 @@ class XNNImplementation final : public KernelImplementation {
944944
op->supports(node, graph);
945945
}
946946

947+
Result<StorageRequirements> requirements(
948+
const KernelRegion& region,
949+
const ptn::Graph& graph,
950+
const ExecutionContext& context) const override {
951+
auto storage = KernelImplementation::requirements(region, graph, context);
952+
if (!storage.ok()) {
953+
return storage.error();
954+
}
955+
for (auto& value : storage->values) {
956+
if (!constant(graph.value(value.id))) {
957+
continue;
958+
}
959+
bool packed = false;
960+
bool retained = false;
961+
for (auto id : region.nodes) {
962+
const auto& node = graph.node(id);
963+
const auto inputs = node.input_value_ids();
964+
if (std::find(inputs.begin(), inputs.end(), value.id) == inputs.end()) {
965+
continue;
966+
}
967+
// A constant can also feed an unpacked operation in this region.
968+
const bool packed_use =
969+
node.target == "torch.ops.aten.linear.default" &&
970+
arg(node, 0).as_tensor().id != value.id &&
971+
(arg(node, 1).as_tensor().id == value.id ||
972+
(arg(node, 2).kind() == ptn::ArgKind::Tensor &&
973+
arg(node, 2).as_tensor().id == value.id));
974+
packed |= packed_use;
975+
retained |= !packed_use;
976+
}
977+
value.preparation_only = packed && !retained;
978+
}
979+
return storage;
980+
}
981+
947982
Result<std::unique_ptr<Executable>> compile(
948983
const KernelRegion& region,
949984
PreparationContext& context) override {

‎backends/cpu/runtime/test/CPUPlanTest.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,80 @@ TEST_F(CPUPlanTest, LastDimensionSoftmaxUsesXNNPACKForDynamicInput) {
273273
}
274274
}
275275

276+
TEST_F(CPUPlanTest, RetainsConstantsWithRuntimeConsumersOrGraphOutputs) {
277+
for (const std::string use : {"add", "gelu", "output"}) {
278+
SCOPED_TRACE(use);
279+
graph = {};
280+
buffers.clear();
281+
for (const auto* name : {"input", "weight", "bias", "linear", "out"}) {
282+
graph.values.emplace_back(
283+
name, ptn::ScalarType::Float, std::vector<int64_t>{1, 3});
284+
}
285+
graph.values[0].role = ptn::ValueRole::UserInput;
286+
graph.values[1] = ptn::Value("weight", ptn::ScalarType::Float, {3, 3});
287+
graph.values[1].role = ptn::ValueRole::Parameter;
288+
graph.values[2] = ptn::Value("bias", ptn::ScalarType::Float, {3});
289+
graph.values[2].role = ptn::ValueRole::ConstantTensor;
290+
graph.input_ids = {0};
291+
ptn::Node linear;
292+
linear.name = "linear";
293+
linear.target = "torch.ops.aten.linear.default";
294+
linear.inputs = {
295+
{"input", ptn::TensorArg{0}},
296+
{"weight", ptn::TensorArg{1}},
297+
{"bias", ptn::TensorArg{2}}};
298+
linear.outputs = {{ptn::OutputValueKind::Tensor, 3}};
299+
graph.nodes.push_back(std::move(linear));
300+
if (use == "add") {
301+
add("out", 3, 2, 4);
302+
finish({4});
303+
} else if (use == "gelu") {
304+
graph.values[4] = ptn::Value("out", ptn::ScalarType::Float, {3});
305+
gelu("out", 2, 4);
306+
finish({3, 4});
307+
} else {
308+
finish({3, 1, 2});
309+
}
310+
for (auto id : {1, 2}) {
311+
auto constant =
312+
allocate_buffer(allocator, tensor_bytes(graph.value(id)).get());
313+
ASSERT_TRUE(constant.ok());
314+
buffers[id] = constant.get();
315+
std::fill_n(
316+
static_cast<float*>(buffers[id].data),
317+
id == 1 ? 9 : 3,
318+
id == 1 ? 2.0f : 3.0f);
319+
}
320+
CPUPlan plan(graph, buffers, configuration, execution);
321+
ASSERT_EQ(plan.select(), Error::Ok);
322+
ASSERT_EQ(plan.prepare(allocator), Error::Ok);
323+
EXPECT_EQ(plan.steps().size(), use == "gelu" ? 2 : 1);
324+
EXPECT_FALSE(plan.can_release_constant(2));
325+
EXPECT_EQ(plan.can_release_constant(1), use != "output");
326+
if (plan.can_release_constant(1)) {
327+
std::memset(buffers[1].data, 0, buffers[1].readable_bytes);
328+
buffers[1] = {};
329+
}
330+
for (float input : {1.0f, -1.0f}) {
331+
std::fill_n(static_cast<float*>(buffers[0].data), 3, input);
332+
ASSERT_EQ(plan.execute(execution), Error::Ok);
333+
const auto* result =
334+
static_cast<const float*>(buffers[use == "add" ? 4 : 3].data);
335+
for (size_t index = 0; index < 3; ++index) {
336+
EXPECT_FLOAT_EQ(result[index], input * 6 + (use == "add" ? 6 : 3));
337+
if (use == "gelu") {
338+
EXPECT_NEAR(
339+
static_cast<const float*>(buffers[4].data)[index], gelu(3), 1e-5);
340+
}
341+
}
342+
}
343+
buffers[2].readable_bytes = 0;
344+
if (use != "output") {
345+
EXPECT_EQ(plan.execute(execution), Error::InvalidArgument);
346+
}
347+
}
348+
}
349+
276350
TEST_F(
277351
CPUPlanTest,
278352
DisconnectedBranchesRemainSeparateAndRejectNonTopologicalSchedule) {

‎backends/cpu/test/test_runtime.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66

77
from __future__ import annotations
88

9+
import json
10+
import os
11+
import re
12+
import tempfile
913
import unittest
1014
from contextlib import contextmanager
1115

@@ -17,6 +21,28 @@
1721
)
1822

1923

24+
def _run_capturing_stderr(fn):
25+
with tempfile.TemporaryFile() as capture:
26+
saved = os.dup(2)
27+
os.dup2(capture.fileno(), 2)
28+
try:
29+
result = fn()
30+
finally:
31+
os.dup2(saved, 2)
32+
os.close(saved)
33+
capture.seek(0)
34+
return result, capture.read().decode()
35+
36+
37+
def _plan_report(log: str) -> dict:
38+
chunks = []
39+
for index, _, encoded in re.findall(r"CPU_PLAN_CHUNK (\d+) (\d+) ([0-9a-f]+)", log):
40+
if index == "0":
41+
chunks = []
42+
chunks.append(encoded)
43+
return json.loads(bytes.fromhex("".join(chunks)))
44+
45+
2046
class CPURuntimeTest(unittest.TestCase):
2147
@contextmanager
2248
def _lower(self, model, inputs):
@@ -176,3 +202,31 @@ def forward(self, x):
176202
torch.testing.assert_close(
177203
runtime.forward(inputs)[0], model(*inputs), rtol=1e-5, atol=1e-5
178204
)
205+
206+
def test_plan_report_counts_released_constants(self):
207+
class AddParameter(torch.nn.Module):
208+
def __init__(self):
209+
super().__init__()
210+
self.p = torch.nn.Parameter(torch.randn(16))
211+
212+
def forward(self, x):
213+
return x + self.p
214+
215+
linear = torch.nn.Linear(16, 8)
216+
# Named constants carry a 64-byte readable tail.
217+
packed_bytes = sum(
218+
t.numel() * t.element_size() + 64 for t in (linear.weight, linear.bias)
219+
)
220+
inputs = (torch.randn(2, 16),)
221+
for model, released in ((linear, packed_bytes), (AddParameter(), 0)):
222+
with self.subTest(model=type(model).__name__):
223+
model.eval()
224+
225+
def run(model=model):
226+
with self._lower(model, inputs) as runtime:
227+
return runtime.forward(inputs)[0]
228+
229+
actual, log = _run_capturing_stderr(run)
230+
report = _plan_report(log)
231+
self.assertEqual(report["released_constant_bytes"], released)
232+
torch.testing.assert_close(actual, model(*inputs), rtol=1e-5, atol=1e-5)

0 commit comments

Comments
 (0)