diff --git a/extension/flat_tensor/flat_tensor_data_map.cpp b/extension/flat_tensor/flat_tensor_data_map.cpp index 6cdabc6f401..d00ef23861b 100644 --- a/extension/flat_tensor/flat_tensor_data_map.cpp +++ b/extension/flat_tensor/flat_tensor_data_map.cpp @@ -266,6 +266,15 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( /* static */ Result FlatTensorDataMap::load( DataLoader* loader) { + Result actual_size = loader->source_size(); + if (!actual_size.ok()) { + return actual_size.error(); + } + ET_CHECK_OR_RETURN_ERROR( + actual_size.get() >= FlatTensorHeader::kNumHeadBytes, + InvalidExternalData, + "FlatTensor data of %" PRIu64 " bytes is shorter than its header", + actual_size.get()); // Check header. Result header = loader->load_at_offset( /*offset=*/0, @@ -289,10 +298,6 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( if (!expected_size.ok()) { return expected_size.error(); } - Result actual_size = loader->source_size(); - if (!actual_size.ok()) { - return actual_size.error(); - } ET_CHECK_OR_RETURN_ERROR( expected_size.get() <= actual_size.get(), InvalidExternalData, @@ -310,6 +315,13 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( " overflows uint64_t; malformed PTD file.", fh->flatbuffer_offset, fh->flatbuffer_size); + ET_CHECK_OR_RETURN_ERROR( + flat_tensor_data_size <= actual_size.get(), + InvalidExternalData, + "FlatTensor flatbuffer ends at %" PRIu64 ", past the %" PRIu64 + " bytes of data", + flat_tensor_data_size, + actual_size.get()); // Load flatbuffer data as a segment. Result flat_tensor_data = loader->load_at_offset( /*offset=*/0, @@ -320,6 +332,17 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( return flat_tensor_data.error(); } + // Minimum size: root offset + file identifier, i.e. the flatbuffer header + // before the FlatTensor header begins. Checked before the identifier is + // read, and it keeps the root offset bound below from underflowing. + constexpr size_t kMinBufferSize = FlatTensorHeader::kHeaderOffset; + ET_CHECK_OR_RETURN_ERROR( + flat_tensor_data->size() >= kMinBufferSize, + InvalidExternalData, + "FlatTensor data size %zu is too small (minimum %zu)", + flat_tensor_data->size(), + kMinBufferSize); + // Make sure magic matches. if (!flat_tensor_flatbuffer::FlatTensorBufferHasIdentifier( flat_tensor_data->data())) { @@ -346,18 +369,6 @@ ET_NODISCARD Result FlatTensorDataMap::get_key( // walks that table's vtable. Nothing here runs full flatbuffer verification, // so a corrupt offset would otherwise let version()/named_data()/segments() // dereference memory outside the buffer. - // - // Minimum size: root offset + file identifier, i.e. the flatbuffer header - // before the FlatTensor header begins. The identifier check above already - // implies at least this many bytes, but check explicitly so the bound below - // cannot underflow. - constexpr size_t kMinBufferSize = FlatTensorHeader::kHeaderOffset; - ET_CHECK_OR_RETURN_ERROR( - flat_tensor_data->size() >= kMinBufferSize, - InvalidExternalData, - "FlatTensor data size %zu is too small (minimum %zu)", - flat_tensor_data->size(), - kMinBufferSize); uint32_t root_offset = flatbuffers::ReadScalar(flat_tensor_data->data()); // The root table is at buf + root_offset. It must not point into the header diff --git a/extension/flat_tensor/test/flat_tensor_data_map_test.cpp b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp index d628594b8eb..08551478767 100644 --- a/extension/flat_tensor/test/flat_tensor_data_map_test.cpp +++ b/extension/flat_tensor/test/flat_tensor_data_map_test.cpp @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include @@ -179,6 +181,13 @@ TEST_F(FlatTensorDataMapTest, LoadAndCheckSize) { ASSERT_EQ(truncated_program.error(), Error::InvalidExternalData); } +TEST_F(FlatTensorDataMapTest, DataShorterThanHeaderIsInvalidExternalData) { + const uint8_t data[8] = {}; + BufferDataLoader loader(data, sizeof(data)); + EXPECT_EQ( + FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData); +} + namespace { constexpr size_t kAlignment = 16; @@ -697,3 +706,43 @@ TEST_F(FlatTensorDataMapTest, RejectsOverflowingFlatbufferExtent) { EXPECT_EQ( FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData); } + +TEST_F(FlatTensorDataMapTest, RejectsFlatbufferPastTheEndOfTheData) { + std::vector data = + CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion); + + const uint64_t flatbuffer_size = data.size(); + std::memcpy( + data.data() + kFlatbufferSizeFieldOffset, + &flatbuffer_size, + sizeof(flatbuffer_size)); + + alignas(std::max_align_t) std::array aligned_buffer{}; + ASSERT_LE(data.size(), aligned_buffer.size()); + std::memcpy(aligned_buffer.data(), data.data(), data.size()); + BufferDataLoader loader(aligned_buffer.data(), data.size()); + + EXPECT_EQ( + FlatTensorDataMap::load(&loader).error(), Error::InvalidExternalData); +} + +TEST_F(FlatTensorDataMapTest, RejectsFlatbufferShorterThanItsIdentifier) { + std::vector data = + CreateDataWithVersion(FlatTensorDataMap::kMaxSupportedSchemaVersion); + const uint64_t zero = 0; + std::memcpy(data.data() + kFlatbufferOffsetFieldOffset, &zero, sizeof(zero)); + std::memcpy(data.data() + kFlatbufferSizeFieldOffset, &zero, sizeof(zero)); + + // FileDataLoader returns no memory for an empty load. + const std::string path = + ::testing::TempDir() + "/flatbuffer_shorter_than_identifier.ptd"; + std::ofstream(path, std::ios::binary) + .write(reinterpret_cast(data.data()), data.size()); + Result loader = FileDataLoader::from(path.c_str()); + ASSERT_EQ(loader.error(), Error::Ok); + + EXPECT_EQ( + FlatTensorDataMap::load(&loader.get()).error(), + Error::InvalidExternalData); + std::remove(path.c_str()); +} diff --git a/extension/module/test/bundled_module_test.cpp b/extension/module/test/bundled_module_test.cpp index 0009c15d69e..98e02647746 100644 --- a/extension/module/test/bundled_module_test.cpp +++ b/extension/module/test/bundled_module_test.cpp @@ -55,11 +55,11 @@ TEST_F(BundledModuleTest, TestNonBPFile) { auto outputs = bundled_module->execute("forward", /*testset_idx=*/0); EXPECT_EQ(bundled_module->Module::is_loaded(), false); - EXPECT_EQ(outputs.error(), Error::InvalidArgument); + EXPECT_EQ(outputs.error(), Error::InvalidProgram); auto status = bundled_module->verify_method_outputs("forward", /*testset_idx=*/0); - EXPECT_EQ(status, Error::InvalidArgument); + EXPECT_EQ(status, Error::InvalidProgram); } TEST_F(BundledModuleTest, TestExecuteInvalidMethod) { diff --git a/extension/named_data_map/merged_data_map.cpp b/extension/named_data_map/merged_data_map.cpp index 6778d0beffe..047051ed1d2 100644 --- a/extension/named_data_map/merged_data_map.cpp +++ b/extension/named_data_map/merged_data_map.cpp @@ -24,17 +24,21 @@ namespace executorch::extension { namespace ET_MERGED_DATA_MAP_NAMESPACE { /*static*/ Result MergedDataMap::load( Span named_data_maps) { + // A map with no keys, such as a .ptd file with no tensors, is valid and + // adds nothing. + bool any_data_map = false; std::vector valid_data_maps; for (auto i : c10::irange(named_data_maps.size())) { - if (named_data_maps[i] != nullptr && - named_data_maps[i]->get_num_keys().get() > 0) { + if (named_data_maps[i] == nullptr) { + continue; + } + any_data_map = true; + if (named_data_maps[i]->get_num_keys().get() > 0) { valid_data_maps.push_back(named_data_maps[i]); } } ET_CHECK_OR_RETURN_ERROR( - !valid_data_maps.empty(), - InvalidArgument, - "No non-empty named data maps provided to merge"); + any_data_map, InvalidArgument, "No named data maps provided to merge"); // Check for duplicate keys. KeyToMapIndex key_to_map_index; diff --git a/extension/named_data_map/test/merged_data_map_test.cpp b/extension/named_data_map/test/merged_data_map_test.cpp index b69a488b8dc..01b68eb6ad6 100644 --- a/extension/named_data_map/test/merged_data_map_test.cpp +++ b/extension/named_data_map/test/merged_data_map_test.cpp @@ -132,6 +132,43 @@ TEST_F(MergedDataMapTest, LoadNullDataMap) { EXPECT_EQ(merged_map.error(), Error::InvalidArgument); } +namespace { +// A data map with no keys, like a .ptd file with no tensors. +class EmptyDataMap final : public NamedDataMap { + public: + Result get_tensor_layout( + std::string_view /*key*/) const override { + return Error::NotFound; + } + Result get_data( + std::string_view /*key*/) const override { + return Error::NotFound; + } + Error load_data_into( + std::string_view /*key*/, + void* /*buffer*/, + size_t /*size*/) const override { + return Error::NotFound; + } + Result get_num_keys() const override { + return 0; + } + Result get_key(uint32_t /*index*/) const override { + return Error::InvalidArgument; + } +}; +} // namespace + +TEST_F(MergedDataMapTest, LoadEmptyDataMap) { + EmptyDataMap empty; + std::vector ndms = {&empty}; + Result merged_map = + MergedDataMap::load(Span(ndms.data(), ndms.size())); + ASSERT_EQ(merged_map.error(), Error::Ok); + EXPECT_EQ(merged_map->get_num_keys().get(), 0); + EXPECT_EQ(merged_map->get_tensor_layout("a").error(), Error::NotFound); +} + TEST_F(MergedDataMapTest, LoadSingleDataMap) { std::vector ndms = {data_maps_["addmul"].get(), nullptr}; Result merged_map = diff --git a/runtime/executor/method.cpp b/runtime/executor/method.cpp index e512081d6ff..f74a3d73ea4 100644 --- a/runtime/executor/method.cpp +++ b/runtime/executor/method.cpp @@ -415,8 +415,6 @@ Result Method::get_num_external_constants() { } Error Method::parse_external_constants(const NamedDataMap* external_data_map) { - ET_CHECK_OR_RETURN_ERROR( - external_data_map != nullptr, InvalidState, "external_data_map is null"); auto flatbuffer_values = serialization_plan_->values(); size_t n_value = flatbuffer_values->size(); @@ -457,6 +455,11 @@ Error Method::parse_external_constants(const NamedDataMap* external_data_map) { nullptr) { continue; } + ET_CHECK_OR_RETURN_ERROR( + external_data_map != nullptr, + NotFound, + "Tensor %s is in a .ptd file, but no .ptd data was provided", + key); Result tensor_layout = external_data_map->get_tensor_layout(key); if (!tensor_layout.ok()) { diff --git a/runtime/executor/program.cpp b/runtime/executor/program.cpp index 61b8ab73526..d8453f19abf 100644 --- a/runtime/executor/program.cpp +++ b/runtime/executor/program.cpp @@ -79,8 +79,17 @@ Result get_execution_plan( size_t program_size = 0; size_t segment_base_offset = 0; size_t segment_data_size = 0; + // Some loaders cannot report their size; programs whose header does not + // need it still load from them. + const Result data_size = loader->size(); { EXECUTORCH_SCOPE_PROF("Program::check_header"); + ET_CHECK_OR_RETURN_ERROR( + !data_size.ok() || data_size.get() >= ExtendedHeader::kNumHeadBytes, + InvalidProgram, + "Program data is %zu bytes, shorter than its %zu-byte header", + data_size.get(), + ExtendedHeader::kNumHeadBytes); Result header = loader->load( /*offset=*/0, ExtendedHeader::kNumHeadBytes, @@ -91,6 +100,14 @@ Result get_execution_plan( Result eh = ExtendedHeader::Parse(header->data(), header->size()); if (eh.ok()) { + // Compared before program_size narrows it on 32-bit platforms. + ET_CHECK_OR_RETURN_ERROR( + !data_size.ok() || eh->program_size <= data_size.get(), + InvalidProgram, + "Program size %" PRIu64 + " from the header is larger than the %zu bytes of data", + eh->program_size, + data_size.get()); // The header has the program size. program_size = eh->program_size; segment_base_offset = eh->segment_base_offset; @@ -110,7 +127,10 @@ Result get_execution_plan( size_t expected = segment_base_offset == 0 ? program_size : segment_base_offset + segment_data_size; - size_t actual = loader->size().get(); + if (!data_size.ok()) { + return data_size.error(); + } + size_t actual = data_size.get(); ET_CHECK_OR_RETURN_ERROR( expected <= actual, InvalidProgram, @@ -121,11 +141,10 @@ Result get_execution_plan( } else if (eh.error() == Error::NotFound) { // No header; the program consumes the whole file, and there are no // segments. - auto result = loader->size(); - if (!result.ok()) { - return result.error(); + if (!data_size.ok()) { + return data_size.error(); } - program_size = result.get(); + program_size = data_size.get(); } else { ET_LOG(Error, "Extended header may be corrupt"); return eh.error(); diff --git a/runtime/executor/tensor_parser_exec_aten.cpp b/runtime/executor/tensor_parser_exec_aten.cpp index 6636feaabc9..18f28e2af46 100644 --- a/runtime/executor/tensor_parser_exec_aten.cpp +++ b/runtime/executor/tensor_parser_exec_aten.cpp @@ -291,8 +291,8 @@ ET_NODISCARD Result getTensorDataPtr( // Look up tensor in named data map. ET_CHECK_OR_RETURN_ERROR( named_data_map != nullptr, - InvalidExternalData, - "Cannot retrieve external tensor with fqn: %s. The named_data_map is null; most likely no external .ptd file was provided.", + NotFound, + "Tensor %s is in a .ptd file, but no .ptd data was provided", fqn); Result tensor_layout_res = named_data_map->get_tensor_layout(fqn); diff --git a/runtime/executor/test/method_test.cpp b/runtime/executor/test/method_test.cpp index 3922accf794..9e5eb44847b 100644 --- a/runtime/executor/test/method_test.cpp +++ b/runtime/executor/test/method_test.cpp @@ -757,6 +757,13 @@ TEST_F(MethodTest, ProgramDataSeparationTest) { ASSERT_EQ(err, Error::Ok); } +TEST_F(MethodTest, MissingExternalDataIsNotFound) { + ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); + Result method = + programs_["add_mul_program"]->load_method("forward", &mmm.get()); + EXPECT_EQ(method.error(), Error::NotFound); +} + TEST_F(MethodTest, MethodGetAttributeTest) { ManagedMemoryManager mmm(kDefaultNonConstMemBytes, kDefaultRuntimeMemBytes); Result method = diff --git a/runtime/executor/test/program_test.cpp b/runtime/executor/test/program_test.cpp index 8d349e53387..88d7d355341 100644 --- a/runtime/executor/test/program_test.cpp +++ b/runtime/executor/test/program_test.cpp @@ -625,6 +625,60 @@ TEST_F(ProgramTest, ShortDataHeader) { Program::HeaderStatus::ShortData); } +TEST_F(ProgramTest, LoadShortDataIsInvalidProgram) { + const uint8_t data[8] = {}; + BufferDataLoader loader(data, sizeof(data)); + EXPECT_EQ(Program::load(&loader).error(), Error::InvalidProgram); +} + +namespace { +// Reads the whole program and gives it a header from before segment_data_size +// existed, which a loader of unknown size can still load. +std::vector read_with_legacy_header(FileDataLoader& loader) { + const size_t size = loader.size().get(); + Result bytes = loader.load( + 0, size, DataLoader::SegmentInfo(DataLoader::SegmentInfo::Type::Program)); + EXPECT_EQ(bytes.error(), Error::Ok); + const auto* begin = static_cast(bytes->data()); + std::vector data(begin, begin + size); + EXPECT_EQ(std::memcmp(data.data() + 8, "eh00", 4), 0); + const uint64_t no_segment_data_size = 0; + // segment_data_size follows program_size and segment_base_offset. + std::memcpy(data.data() + 32, &no_segment_data_size, sizeof(uint64_t)); + return data; +} + +class UnknownSizeLoader final : public DataLoader { + public: + UnknownSizeLoader(const void* data, size_t size) : loader_(data, size) {} + Result load( + size_t offset, + size_t size, + const SegmentInfo& segment_info) const override { + return loader_.load(offset, size, segment_info); + } + Result size() const override { + return Error::NotSupported; + } + + private: + BufferDataLoader loader_; +}; +} // namespace + +TEST_F(ProgramTest, LoadLegacyHeaderWithoutDataSize) { + std::vector data = read_with_legacy_header(*multi_loader_); + UnknownSizeLoader loader(data.data(), data.size()); + EXPECT_EQ(Program::load(&loader).error(), Error::Ok); +} + +TEST_F(ProgramTest, LoadProgramShorterThanItsHeaderSaysIsInvalidProgram) { + std::vector data = read_with_legacy_header(*multi_loader_); + // Keep the header, but less than the program size it states. + BufferDataLoader loader(data.data(), Program::kMinHeadBytes + 16); + EXPECT_EQ(Program::load(&loader).error(), Error::InvalidProgram); +} + TEST_F(ProgramTest, IncompatibleHeader) { // Make a local copy of the header. size_t data_len = Program::kMinHeadBytes; diff --git a/runtime/executor/test/tensor_parser_test.cpp b/runtime/executor/test/tensor_parser_test.cpp index 055cd379d17..00efecca5d2 100644 --- a/runtime/executor/test/tensor_parser_test.cpp +++ b/runtime/executor/test/tensor_parser_test.cpp @@ -27,6 +27,7 @@ using executorch::runtime::Program; using executorch::runtime::Result; using executorch::runtime::Span; using executorch::runtime::TensorLayout; +using executorch::runtime::deserialization::getTensorDataPtr; using executorch::runtime::deserialization::parseListOptionalType; using executorch::runtime::deserialization::parseTensor; using executorch::runtime::deserialization::parseTensorList; @@ -349,3 +350,45 @@ TEST_F(TensorParserTest, ParseListOptionalTypeRejectsWrongType) { auto result = parseListOptionalType(indices, values, 2, &mmm.get()); EXPECT_EQ(result.error(), Error::InvalidType); } + +TEST(GetTensorDataPtrTest, MutableExternalTensorWithoutDataIsNotFound) { + flatbuffers::FlatBufferBuilder builder; + std::vector sizes = {1}; + std::vector dim_order = {0}; + auto sizes_offset = builder.CreateVector(sizes); + auto dim_order_offset = builder.CreateVector(dim_order); + // Allocation details make the tensor mutable, so its data is loaded from + // the .ptd data into planned memory. + auto allocation_info = executorch_flatbuffer::CreateAllocationDetails( + builder, /*memory_id=*/1, /*memory_offset_low=*/0); + auto extra_tensor_info = executorch_flatbuffer::CreateExtraTensorInfo( + builder, + /*mutable_data_segments_idx=*/0, + builder.CreateString("state"), + executorch_flatbuffer::TensorDataLocation::EXTERNAL); + auto tensor_offset = executorch_flatbuffer::CreateTensor( + builder, + executorch_flatbuffer::ScalarType::FLOAT, + /*storage_offset=*/0, + sizes_offset, + dim_order_offset, + /*requires_grad=*/false, + /*data_buffer_idx=*/0, + allocation_info, + /*layout=*/0, + executorch_flatbuffer::TensorShapeDynamism::STATIC, + extra_tensor_info); + builder.Finish(tensor_offset); + const auto* s_tensor = flatbuffers::GetRoot( + builder.GetBufferPointer()); + + ManagedMemoryManager mmm( + /*planned_memory_bytes=*/64, /*method_allocator_bytes=*/64); + Result data = getTensorDataPtr( + s_tensor, + /*program=*/nullptr, + /*nbytes=*/sizeof(float), + mmm.get().planned_memory(), + /*named_data_map=*/nullptr); + EXPECT_EQ(data.error(), Error::NotFound); +}