Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 27 additions & 16 deletions extension/flat_tensor/flat_tensor_data_map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,15 @@ ET_NODISCARD Result<const char*> FlatTensorDataMap::get_key(

/* static */ Result<FlatTensorDataMap> FlatTensorDataMap::load(
DataLoader* loader) {
Result<uint64_t> actual_size = loader->source_size();
if (!actual_size.ok()) {
return actual_size.error();
}
ET_CHECK_OR_RETURN_ERROR(
actual_size.get() >= FlatTensorHeader::kNumHeadBytes,
InvalidExternalData,
"FlatTensor data of %" PRIu64 " bytes is shorter than its header",
actual_size.get());
// Check header.
Result<FreeableBuffer> header = loader->load_at_offset(
/*offset=*/0,
Expand All @@ -289,10 +298,6 @@ ET_NODISCARD Result<const char*> FlatTensorDataMap::get_key(
if (!expected_size.ok()) {
return expected_size.error();
}
Result<uint64_t> actual_size = loader->source_size();
if (!actual_size.ok()) {
return actual_size.error();
}
ET_CHECK_OR_RETURN_ERROR(
expected_size.get() <= actual_size.get(),
InvalidExternalData,
Expand All @@ -310,6 +315,13 @@ ET_NODISCARD Result<const char*> FlatTensorDataMap::get_key(
" overflows uint64_t; malformed PTD file.",
fh->flatbuffer_offset,
fh->flatbuffer_size);
ET_CHECK_OR_RETURN_ERROR(
flat_tensor_data_size <= actual_size.get(),
InvalidExternalData,
"FlatTensor flatbuffer ends at %" PRIu64 ", past the %" PRIu64
" bytes of data",
flat_tensor_data_size,
actual_size.get());
// Load flatbuffer data as a segment.
Result<FreeableBuffer> flat_tensor_data = loader->load_at_offset(
/*offset=*/0,
Expand All @@ -320,6 +332,17 @@ ET_NODISCARD Result<const char*> FlatTensorDataMap::get_key(
return flat_tensor_data.error();
}

// Minimum size: root offset + file identifier, i.e. the flatbuffer header
// before the FlatTensor header begins. Checked before the identifier is
// read, and it keeps the root offset bound below from underflowing.
constexpr size_t kMinBufferSize = FlatTensorHeader::kHeaderOffset;
ET_CHECK_OR_RETURN_ERROR(
flat_tensor_data->size() >= kMinBufferSize,
InvalidExternalData,
"FlatTensor data size %zu is too small (minimum %zu)",
flat_tensor_data->size(),
kMinBufferSize);

// Make sure magic matches.
if (!flat_tensor_flatbuffer::FlatTensorBufferHasIdentifier(
flat_tensor_data->data())) {
Expand All @@ -346,18 +369,6 @@ ET_NODISCARD Result<const char*> FlatTensorDataMap::get_key(
// walks that table's vtable. Nothing here runs full flatbuffer verification,
// so a corrupt offset would otherwise let version()/named_data()/segments()
// dereference memory outside the buffer.
//
// Minimum size: root offset + file identifier, i.e. the flatbuffer header
// before the FlatTensor header begins. The identifier check above already
// implies at least this many bytes, but check explicitly so the bound below
// cannot underflow.
constexpr size_t kMinBufferSize = FlatTensorHeader::kHeaderOffset;
ET_CHECK_OR_RETURN_ERROR(
flat_tensor_data->size() >= kMinBufferSize,
InvalidExternalData,
"FlatTensor data size %zu is too small (minimum %zu)",
flat_tensor_data->size(),
kMinBufferSize);
uint32_t root_offset =
flatbuffers::ReadScalar<flatbuffers::uoffset_t>(flat_tensor_data->data());
// The root table is at buf + root_offset. It must not point into the header
Expand Down
49 changes: 49 additions & 0 deletions extension/flat_tensor/test/flat_tensor_data_map_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
#include <executorch/runtime/platform/runtime.h>

#include <array>
#include <cstdio>
#include <fstream>
#include <limits>

#include <gtest/gtest.h>
Expand Down Expand Up @@ -179,6 +181,13 @@ TEST_F(FlatTensorDataMapTest, LoadAndCheckSize) {
ASSERT_EQ(truncated_program.error(), Error::InvalidExternalData);
}

TEST_F(FlatTensorDataMapTest, DataShorterThanHeaderIsInvalidExternalData) {
const uint8_t data[8] = {};
BufferDataLoader loader(data, sizeof(data));
EXPECT_EQ(
FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData);
}

namespace {

constexpr size_t kAlignment = 16;
Expand Down Expand Up @@ -697,3 +706,43 @@ TEST_F(FlatTensorDataMapTest, RejectsOverflowingFlatbufferExtent) {
EXPECT_EQ(
FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData);
}

TEST_F(FlatTensorDataMapTest, RejectsFlatbufferPastTheEndOfTheData) {
std::vector<uint8_t> data =
CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion);

const uint64_t flatbuffer_size = data.size();
std::memcpy(
data.data() + kFlatbufferSizeFieldOffset,
&flatbuffer_size,
sizeof(flatbuffer_size));

alignas(std::max_align_t) std::array<uint8_t, 1024> aligned_buffer{};
ASSERT_LE(data.size(), aligned_buffer.size());
std::memcpy(aligned_buffer.data(), data.data(), data.size());
BufferDataLoader loader(aligned_buffer.data(), data.size());

EXPECT_EQ(
FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData);
}

TEST_F(FlatTensorDataMapTest, RejectsFlatbufferShorterThanItsIdentifier) {
std::vector<uint8_t> data =
CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion);
const uint64_t zero = 0;
std::memcpy(data.data() + kFlatbufferOffsetFieldOffset, &zero, sizeof(zero));
std::memcpy(data.data() + kFlatbufferSizeFieldOffset, &zero, sizeof(zero));

// FileDataLoader returns no memory for an empty load.
const std::string path =
::testing::TempDir() + "/flatbuffer_shorter_than_identifier.ptd";
std::ofstream(path, std::ios::binary)
.write(reinterpret_cast<const char*>(data.data()), data.size());
Result<FileDataLoader> loader = FileDataLoader::from(path.c_str());
ASSERT_EQ(loader.error(), Error::Ok);

EXPECT_EQ(
FlatTensorDataMap::load(&loader.get()).error(),
Error::InvalidExternalData);
std::remove(path.c_str());
}
4 changes: 2 additions & 2 deletions extension/module/test/bundled_module_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,11 @@ TEST_F(BundledModuleTest, TestNonBPFile) {

auto outputs = bundled_module->execute("forward", /*testset_idx=*/0);
EXPECT_EQ(bundled_module->Module::is_loaded(), false);
EXPECT_EQ(outputs.error(), Error::InvalidArgument);
EXPECT_EQ(outputs.error(), Error::InvalidProgram);

auto status =
bundled_module->verify_method_outputs("forward", /*testset_idx=*/0);
EXPECT_EQ(status, Error::InvalidArgument);
EXPECT_EQ(status, Error::InvalidProgram);
}

TEST_F(BundledModuleTest, TestExecuteInvalidMethod) {
Expand Down
14 changes: 9 additions & 5 deletions extension/named_data_map/merged_data_map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,21 @@ namespace executorch::extension {
namespace ET_MERGED_DATA_MAP_NAMESPACE {
/*static*/ Result<MergedDataMap> MergedDataMap::load(
Span<const NamedDataMap*> named_data_maps) {
// A map with no keys, such as a .ptd file with no tensors, is valid and
// adds nothing.
bool any_data_map = false;
std::vector<const NamedDataMap*> valid_data_maps;
for (auto i : c10::irange(named_data_maps.size())) {
if (named_data_maps[i] != nullptr &&
named_data_maps[i]->get_num_keys().get() > 0) {
if (named_data_maps[i] == nullptr) {
continue;
}
any_data_map = true;
if (named_data_maps[i]->get_num_keys().get() > 0) {
valid_data_maps.push_back(named_data_maps[i]);
}
}
ET_CHECK_OR_RETURN_ERROR(
!valid_data_maps.empty(),
InvalidArgument,
"No non-empty named data maps provided to merge");
any_data_map, InvalidArgument, "No named data maps provided to merge");

// Check for duplicate keys.
KeyToMapIndex key_to_map_index;
Expand Down
37 changes: 37 additions & 0 deletions extension/named_data_map/test/merged_data_map_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,43 @@ TEST_F(MergedDataMapTest, LoadNullDataMap) {
EXPECT_EQ(merged_map.error(), Error::InvalidArgument);
}

namespace {
// A data map with no keys, like a .ptd file with no tensors.
class EmptyDataMap final : public NamedDataMap {
public:
Result<const TensorLayout> get_tensor_layout(
std::string_view /*key*/) const override {
return Error::NotFound;
}
Result<executorch::runtime::FreeableBuffer> get_data(
std::string_view /*key*/) const override {
return Error::NotFound;
}
Error load_data_into(
std::string_view /*key*/,
void* /*buffer*/,
size_t /*size*/) const override {
return Error::NotFound;
}
Result<uint32_t> get_num_keys() const override {
return 0;
}
Result<const char*> get_key(uint32_t /*index*/) const override {
return Error::InvalidArgument;
}
};
} // namespace

TEST_F(MergedDataMapTest, LoadEmptyDataMap) {
EmptyDataMap empty;
std::vector<const NamedDataMap*> ndms = {&empty};
Result<MergedDataMap> merged_map =
MergedDataMap::load(Span<const NamedDataMap*>(ndms.data(), ndms.size()));
ASSERT_EQ(merged_map.error(), Error::Ok);
EXPECT_EQ(merged_map->get_num_keys().get(), 0);
EXPECT_EQ(merged_map->get_tensor_layout("a").error(), Error::NotFound);
}

TEST_F(MergedDataMapTest, LoadSingleDataMap) {
std::vector<const NamedDataMap*> ndms = {data_maps_["addmul"].get(), nullptr};
Result<MergedDataMap> merged_map =
Expand Down
7 changes: 5 additions & 2 deletions runtime/executor/method.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -415,8 +415,6 @@ Result<size_t> Method::get_num_external_constants() {
}

Error Method::parse_external_constants(const NamedDataMap* external_data_map) {
ET_CHECK_OR_RETURN_ERROR(
external_data_map != nullptr, InvalidState, "external_data_map is null");
auto flatbuffer_values = serialization_plan_->values();
size_t n_value = flatbuffer_values->size();

Expand Down Expand Up @@ -457,6 +455,11 @@ Error Method::parse_external_constants(const NamedDataMap* external_data_map) {
nullptr) {
continue;
}
ET_CHECK_OR_RETURN_ERROR(
external_data_map != nullptr,
NotFound,
"Tensor %s is in a .ptd file, but no .ptd data was provided",
key);
Result<const TensorLayout> tensor_layout =
external_data_map->get_tensor_layout(key);
if (!tensor_layout.ok()) {
Expand Down
29 changes: 24 additions & 5 deletions runtime/executor/program.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,17 @@ Result<executorch_flatbuffer::ExecutionPlan*> get_execution_plan(
size_t program_size = 0;
size_t segment_base_offset = 0;
size_t segment_data_size = 0;
// Some loaders cannot report their size; programs whose header does not
// need it still load from them.
const Result<size_t> data_size = loader->size();
{
EXECUTORCH_SCOPE_PROF("Program::check_header");
ET_CHECK_OR_RETURN_ERROR(
!data_size.ok() || data_size.get() >= ExtendedHeader::kNumHeadBytes,
InvalidProgram,
"Program data is %zu bytes, shorter than its %zu-byte header",
data_size.get(),
ExtendedHeader::kNumHeadBytes);
Result<FreeableBuffer> header = loader->load(
/*offset=*/0,
ExtendedHeader::kNumHeadBytes,
Expand All @@ -91,6 +100,14 @@ Result<executorch_flatbuffer::ExecutionPlan*> get_execution_plan(
Result<ExtendedHeader> eh =
ExtendedHeader::Parse(header->data(), header->size());
if (eh.ok()) {
// Compared before program_size narrows it on 32-bit platforms.
ET_CHECK_OR_RETURN_ERROR(
!data_size.ok() || eh->program_size <= data_size.get(),
InvalidProgram,
"Program size %" PRIu64
" from the header is larger than the %zu bytes of data",
eh->program_size,
data_size.get());
// The header has the program size.
program_size = eh->program_size;
segment_base_offset = eh->segment_base_offset;
Expand All @@ -110,7 +127,10 @@ Result<executorch_flatbuffer::ExecutionPlan*> get_execution_plan(
size_t expected = segment_base_offset == 0
? program_size
: segment_base_offset + segment_data_size;
size_t actual = loader->size().get();
if (!data_size.ok()) {
return data_size.error();
}
size_t actual = data_size.get();
ET_CHECK_OR_RETURN_ERROR(
expected <= actual,
InvalidProgram,
Expand All @@ -121,11 +141,10 @@ Result<executorch_flatbuffer::ExecutionPlan*> get_execution_plan(
} else if (eh.error() == Error::NotFound) {
// No header; the program consumes the whole file, and there are no
// segments.
auto result = loader->size();
if (!result.ok()) {
return result.error();
if (!data_size.ok()) {
return data_size.error();
}
program_size = result.get();
program_size = data_size.get();
} else {
ET_LOG(Error, "Extended header may be corrupt");
return eh.error();
Expand Down
4 changes: 2 additions & 2 deletions runtime/executor/tensor_parser_exec_aten.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -291,8 +291,8 @@ ET_NODISCARD Result<void*> getTensorDataPtr(
// Look up tensor in named data map.
ET_CHECK_OR_RETURN_ERROR(
named_data_map != nullptr,
InvalidExternalData,
"Cannot retrieve external tensor with fqn: %s. The named_data_map is null; most likely no external .ptd file was provided.",
NotFound,
"Tensor %s is in a .ptd file, but no .ptd data was provided",
fqn);
Result<const TensorLayout> tensor_layout_res =
named_data_map->get_tensor_layout(fqn);
Expand Down
7 changes: 7 additions & 0 deletions runtime/executor/test/method_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,13 @@ TEST_F(MethodTest, ProgramDataSeparationTest) {
ASSERT_EQ(err, Error::Ok);
}

TEST_F(MethodTest, MissingExternalDataIsNotFound) {
ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes);
Result<Method> method =
programs_["add_mul_program"]->load_method("forward", &mmm.get());
EXPECT_EQ(method.error(), Error::NotFound);
}

TEST_F(MethodTest, MethodGetAttributeTest) {
ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes);
Result<Method> method =
Expand Down
54 changes: 54 additions & 0 deletions runtime/executor/test/program_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,60 @@ TEST_F(ProgramTest, ShortDataHeader) {
Program::HeaderStatus::ShortData);
}

TEST_F(ProgramTest, LoadShortDataIsInvalidProgram) {
const uint8_t data[8] = {};
BufferDataLoader loader(data, sizeof(data));
EXPECT_EQ(Program::load(&loader).error(), Error::InvalidProgram);
}

namespace {
// Reads the whole program and gives it a header from before segment_data_size
// existed, which a loader of unknown size can still load.
std::vector<uint8_t> read_with_legacy_header(FileDataLoader& loader) {
const size_t size = loader.size().get();
Result<FreeableBuffer> bytes = loader.load(
0, size, DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program));
EXPECT_EQ(bytes.error(), Error::Ok);
const auto* begin = static_cast<const uint8_t*>(bytes->data());
std::vector<uint8_t> data(begin, begin + size);
EXPECT_EQ(std::memcmp(data.data() + 8, "eh00", 4), 0);
const uint64_t no_segment_data_size = 0;
// segment_data_size follows program_size and segment_base_offset.
std::memcpy(data.data() + 32, &no_segment_data_size, sizeof(uint64_t));
return data;
}

class UnknownSizeLoader final : public DataLoader {
public:
UnknownSizeLoader(const void* data, size_t size) : loader_(data, size) {}
Result<FreeableBuffer> load(
size_t offset,
size_t size,
const SegmentInfo& segment_info) const override {
return loader_.load(offset, size, segment_info);
}
Result<size_t> size() const override {
return Error::NotSupported;
}

private:
BufferDataLoader loader_;
};
} // namespace

TEST_F(ProgramTest, LoadLegacyHeaderWithoutDataSize) {
std::vector<uint8_t> data = read_with_legacy_header(*multi_loader_);
UnknownSizeLoader loader(data.data(), data.size());
EXPECT_EQ(Program::load(&loader).error(), Error::Ok);
}

TEST_F(ProgramTest, LoadProgramShorterThanItsHeaderSaysIsInvalidProgram) {
std::vector<uint8_t> data = read_with_legacy_header(*multi_loader_);
// Keep the header, but less than the program size it states.
BufferDataLoader loader(data.data(), Program::kMinHeadBytes + 16);
EXPECT_EQ(Program::load(&loader).error(), Error::InvalidProgram);
}

TEST_F(ProgramTest, IncompatibleHeader) {
// Make a local copy of the header.
size_t data_len = Program::kMinHeadBytes;
Expand Down
Loading
Loading