diff --git a/extension/module/bundled_module.cpp b/extension/module/bundled_module.cpp index 50dbfe69237..6d667f8338a 100644 --- a/extension/module/bundled_module.cpp +++ b/extension/module/bundled_module.cpp @@ -12,17 +12,21 @@ #include #include #include +#include namespace executorch { namespace extension { namespace { +const flatbuffers::Vector* embedded_program( + const void* bundled_program_ptr) { + return bundled_program_flatbuffer::GetBundledProgram(bundled_program_ptr) + ->program(); +} + std::unique_ptr program_data_loader( const void* bundled_program_ptr) { - auto bundled_program = - bundled_program_flatbuffer::GetBundledProgram(bundled_program_ptr); - // the program inside the bundled program - auto program = bundled_program->program(); + auto program = embedded_program(bundled_program_ptr); return std::make_unique(program->data(), program->size()); } } // namespace @@ -35,11 +39,26 @@ BundledModule::BundledModule( std::unique_ptr temp_allocator, std::unique_ptr event_tracer, std::unique_ptr data_map_loader) - : Module( + : BundledModule( + bundled_program_ptr, program_data_loader(bundled_program_ptr), std::move(memory_allocator), std::move(temp_allocator), std::move(event_tracer), + std::move(data_map_loader)) {} + +BundledModule::BundledModule( + const void* bundled_program_ptr, + std::unique_ptr program_loader, + std::unique_ptr memory_allocator, + std::unique_ptr temp_allocator, + std::unique_ptr event_tracer, + std::unique_ptr data_map_loader) + : Module( + std::move(program_loader), + std::move(memory_allocator), + std::move(temp_allocator), + std::move(event_tracer), std::move(data_map_loader)), bundled_program_ptr_(bundled_program_ptr) {} @@ -60,22 +79,27 @@ runtime::Result> BundledModule::from_file( } size_t file_size = file_size_result.get(); - auto file_data = std::make_unique(file_size); + std::shared_ptr file_data(new uint8_t[file_size]); auto buffer_result = data_loader_result->load_into(0, file_size, {}, file_data.get()); if (buffer_result != runtime::Error::Ok) { return buffer_result; } - // Pass ownership of the data to BundledModule - auto bm = std::make_unique( - file_data.release(), + // The loader shares the file data, so a Program it loads can outlive this + // module. + auto program = embedded_program(file_data.get()); + auto program_loader = std::make_unique( + std::shared_ptr(file_data, const_cast(program->data())), + program->size()); + std::unique_ptr bm(new BundledModule( + file_data.get(), + std::move(program_loader), std::move(memory_allocator), std::move(temp_allocator), std::move(event_tracer), - std::move(data_map_loader)); - - bm->is_loaded_from_file_ = true; + std::move(data_map_loader))); + bm->file_data_ = std::move(file_data); return bm; } diff --git a/extension/module/bundled_module.h b/extension/module/bundled_module.h index c75d83bcbad..bad04581fec 100644 --- a/extension/module/bundled_module.h +++ b/extension/module/bundled_module.h @@ -56,12 +56,6 @@ class BundledModule : public Module { // Disallow copying BundledModule(BundledModule&&) = delete; BundledModule& operator=(BundledModule&&) = delete; - // Default destructor - ~BundledModule() { - if (is_loaded_from_file_) { - delete[] static_cast(bundled_program_ptr_); - } - } /** * Constructs an instance by loading a bundled program from a file with @@ -125,8 +119,16 @@ class BundledModule : public Module { double atol = 1e-8); private: + BundledModule( + const void* bundled_program_ptr, + std::unique_ptr program_loader, + std::unique_ptr memory_allocator, + std::unique_ptr temp_allocator, + std::unique_ptr event_tracer, + std::unique_ptr data_map_loader); + const void* bundled_program_ptr_; - bool is_loaded_from_file_ = false; + std::shared_ptr file_data_; }; } // namespace ET_BUNDLED_MODULE_NAMESPACE diff --git a/extension/module/module.cpp b/extension/module/module.cpp index 6a6fd143949..2f3e63819a9 100644 --- a/extension/module/module.cpp +++ b/extension/module/module.cpp @@ -333,45 +333,7 @@ runtime::Error Module::load_internal( ptn_ = std::move(*ptn_load_result); return runtime::Error::Ok; } - if (data_files_.size() > 0) { - for (const auto& data_file : data_files_) { - auto data_map_loader_result = make_data_loader(data_file, load_mode_); - if (!data_map_loader_result.ok()) { - return data_map_loader_result.error(); - } - data_map_loaders_.push_back(std::move(*data_map_loader_result)); - } - } - - if (data_map_loaders_.size() > 0) { - for (auto i = 0; i < data_map_loaders_.size(); ++i) { - auto res_flat_tensor = - FlatTensorDataMap::load(data_map_loaders_[i].get()); - if (!res_flat_tensor.ok()) { - return res_flat_tensor.error(); - } - named_data_maps_.push_back( - std::make_unique< - std::remove_reference_t>( - std::move(*res_flat_tensor))); - } - - // Extract raw pointers from unique_ptrs to pass to MergedDataMap::load() - std::vector raw_data_maps; - raw_data_maps.reserve(named_data_maps_.size()); - for (const auto& data_map : named_data_maps_) { - raw_data_maps.push_back(data_map.get()); - } - auto res_merged = MergedDataMap::load(runtime::Span( - raw_data_maps.data(), raw_data_maps.size())); - if (!res_merged.ok()) { - return res_merged.error(); - } - merged_data_map_ = - std::make_unique>( - std::move(*res_merged)); - } - + ET_CHECK_OK_OR_RETURN_ERROR(load_data_maps()); auto res_program = Program::load(data_loader_.get(), verification); if (!res_program.ok()) { return res_program.error(); @@ -379,9 +341,65 @@ runtime::Error Module::load_internal( auto program = std::make_unique>( std::move(*res_program)); + // The program reads from its loader, so it keeps the loader alive. The + // loader goes with the program, not with the last weak pointer to it. + std::shared_ptr data_loader = std::move(data_loader_); program_ = std::shared_ptr( - program.release(), [](Program* pointer) { delete pointer; }); + program.release(), + [data_loader = std::move(data_loader)](Program* pointer) mutable { + delete pointer; + data_loader.reset(); + }); + } else if (program_) { + // A Module made from a shared program still loads its own data maps. + ET_CHECK_OK_OR_RETURN_ERROR(load_data_maps()); + } + return runtime::Error::Ok; +} + +runtime::Error Module::load_data_maps() { + if (merged_data_map_ || (data_files_.empty() && data_map_loaders_.empty())) { + return runtime::Error::Ok; + } + // Open every file before keeping any, so a failed open can be retried. + std::vector> file_loaders; + file_loaders.reserve(data_files_.size()); + for (const auto& data_file : data_files_) { + auto data_map_loader_result = make_data_loader(data_file, load_mode_); + if (!data_map_loader_result.ok()) { + return data_map_loader_result.error(); + } + file_loaders.push_back(std::move(*data_map_loader_result)); + } + for (auto& file_loader : file_loaders) { + data_map_loaders_.push_back(std::move(file_loader)); + } + data_files_.clear(); + named_data_maps_.clear(); + for (const auto& data_map_loader : data_map_loaders_) { + auto res_flat_tensor = FlatTensorDataMap::load(data_map_loader.get()); + if (!res_flat_tensor.ok()) { + return res_flat_tensor.error(); + } + named_data_maps_.push_back( + std::make_unique>( + std::move(*res_flat_tensor))); + } + + // Extract raw pointers from unique_ptrs to pass to MergedDataMap::load() + std::vector raw_data_maps; + raw_data_maps.reserve(named_data_maps_.size()); + for (const auto& data_map : named_data_maps_) { + raw_data_maps.push_back(data_map.get()); + } + auto res_merged = MergedDataMap::load(runtime::Span( + raw_data_maps.data(), raw_data_maps.size())); + if (!res_merged.ok()) { + return res_merged.error(); } + merged_data_map_ = + std::make_unique>( + std::move(*res_merged)); return runtime::Error::Ok; } diff --git a/extension/module/module.h b/extension/module/module.h index df66df0c871..3a9077f0959 100644 --- a/extension/module/module.h +++ b/extension/module/module.h @@ -780,6 +780,8 @@ class Module { LoadBackendOptionsMap backend_options_map_; bool share_memory_arenas_; + ET_NODISCARD runtime::Error load_data_maps(); + ET_NODISCARD runtime::Error load_internal( const Program::Verification verification, bool has_backend_options = false); diff --git a/extension/module/targets.bzl b/extension/module/targets.bzl index 363c55b7c96..54154725f55 100644 --- a/extension/module/targets.bzl +++ b/extension/module/targets.bzl @@ -56,6 +56,7 @@ def define_common_targets(): deps = [ "//executorch/extension/data_loader:buffer_data_loader", "//executorch/extension/data_loader:file_data_loader", + "//executorch/extension/data_loader:shared_ptr_data_loader", "//executorch/devtools/bundled_program:runtime" + aten_suffix, "//executorch/devtools/bundled_program/schema:bundled_program_schema_fbs", ], diff --git a/extension/module/test/CMakeLists.txt b/extension/module/test/CMakeLists.txt index 40d5fb285ba..38609014288 100644 --- a/extension/module/test/CMakeLists.txt +++ b/extension/module/test/CMakeLists.txt @@ -75,3 +75,25 @@ add_dependencies(extension_module_test generated_module_test_files) set_property(TEST extension_module_test PROPERTY ENVIRONMENT ${test_env}) set_property(TEST extension_module_test PROPERTY ENVIRONMENT "${test_env}") + +if(TARGET bundled_program) + et_cxx_test( + extension_bundled_module_test + SOURCES + bundled_module_test.cpp + ${EXECUTORCH_ROOT}/extension/module/bundled_module.cpp + EXTRA_LIBS + bundled_program + extension_data_loader + extension_module_static + extension_tensor + portable_kernels + portable_ops_lib + ) + add_dependencies(extension_bundled_module_test generated_module_test_files) + set_property( + TEST extension_bundled_module_test + PROPERTY ENVIRONMENT "RESOURCES_PATH=${CMAKE_CURRENT_SOURCE_DIR}/resources" + "ET_MODULE_PTE_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte" + ) +endif() diff --git a/extension/module/test/bundled_module_test.cpp b/extension/module/test/bundled_module_test.cpp index 0009c15d69e..84ec5ae6ba5 100644 --- a/extension/module/test/bundled_module_test.cpp +++ b/extension/module/test/bundled_module_test.cpp @@ -108,3 +108,17 @@ TEST_F(BundledModuleTest, TestVerifyInvalidIdx) { bundled_module->verify_method_outputs("forward", /*testset_idx=*/10000); EXPECT_EQ(status, Error::InvalidArgument); } + +TEST_F(BundledModuleTest, TestProgramOutlivesModuleFromFile) { + std::shared_ptr program; + { + auto bundled_module = BundledModule::from_file(bpte_path_.c_str()); + ASSERT_EQ(bundled_module.error(), Error::Ok); + ASSERT_EQ(bundled_module.get()->load(), Error::Ok); + program = bundled_module.get()->program(); + } + ASSERT_NE(program, nullptr); + auto method_name = program->get_method_name(0); + ASSERT_EQ(method_name.error(), Error::Ok); + EXPECT_STREQ(method_name.get(), "forward"); +} diff --git a/extension/module/test/module_test.cpp b/extension/module/test/module_test.cpp index 7a29375cf65..7d962010b56 100644 --- a/extension/module/test/module_test.cpp +++ b/extension/module/test/module_test.cpp @@ -9,7 +9,9 @@ #include #include +#include #include +#include #include #include @@ -424,6 +426,50 @@ TEST_F(ModuleTest, TestProgramPersistenceAndReuseAfterModuleDestruction) { EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } +namespace { +class TrackedLoader final : public DataLoader { + public: + TrackedLoader(std::unique_ptr loader, bool& destroyed) + : loader_(std::move(loader)), destroyed_(destroyed) {} + ~TrackedLoader() override { + destroyed_ = true; + } + Result load( + size_t offset, + size_t size, + const SegmentInfo& segment_info) const override { + return loader_->load(offset, size, segment_info); + } + Result size() const override { + return loader_->size(); + } + + private: + std::unique_ptr loader_; + bool& destroyed_; +}; +} // namespace + +TEST_F(ModuleTest, TestProgramKeepsItsLoaderAlive) { + auto file = FileDataLoader::from(model_path_.c_str()); + ASSERT_EQ(file.error(), Error::Ok); + bool destroyed = false; + std::shared_ptr program; + { + Module owner(std::make_unique( + std::make_unique(std::move(*file)), destroyed)); + ASSERT_EQ(owner.load(), Error::Ok); + program = owner.program(); + } + // Programs load delegate and data segments through the loader on demand. + EXPECT_FALSE(destroyed); + // A weak pointer keeps the control block, but not the program or its loader. + std::weak_ptr weak_program = program; + program.reset(); + EXPECT_TRUE(weak_program.expired()); + EXPECT_TRUE(destroyed); +} + TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) { std::shared_ptr program; { @@ -580,6 +626,56 @@ TEST_F(ModuleTest, TestPTD) { ASSERT_EQ(module.forward(tensor).error(), Error::Ok); } +TEST_F(ModuleTest, TestSharedProgramLoadsItsDataMap) { + Module owner(add_mul_path_); + ASSERT_EQ(owner.load(), Error::Ok); + auto loader = FileDataLoader::from(add_mul_data_path_.c_str()); + ASSERT_EQ(loader.error(), Error::Ok); + + Module module( + owner.program(), + nullptr, + nullptr, + nullptr, + std::make_unique(std::move(*loader))); + ASSERT_EQ(module.load_method("forward"), Error::Ok); + auto tensor = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 2.f}); + EXPECT_EQ(module.forward(tensor).error(), Error::Ok); +} + +TEST_F(ModuleTest, TestLoadRetriesAfterAMissingDataFile) { + const std::string late_path = + ::testing::TempDir() + "/module_test_late_data_file.ptd"; + std::remove(late_path.c_str()); + Module module( + add_mul_path_, std::vector{add_mul_data_path_, late_path}); + EXPECT_NE(module.load(), Error::Ok); + + { + std::ifstream source(linear_data_path_, std::ios::binary); + std::ofstream destination(late_path, std::ios::binary); + destination << source.rdbuf(); + } + // The first file must not be added twice by the second attempt. + EXPECT_EQ(module.load(), Error::Ok); + std::remove(late_path.c_str()); +} + +TEST_F(ModuleTest, TestLoadRetriesAfterAFailedProgramLoad) { + const std::string bad_program_path = + ::testing::TempDir() + "/module_test_bad_program.pte"; + { + std::ofstream bad_program(bad_program_path, std::ios::binary); + bad_program << std::string(128, 'x'); + } + Module module(bad_program_path, add_mul_data_path_); + const Error first_error = module.load(); + EXPECT_NE(first_error, Error::Ok); + // A retry reports the program error again, not a repeated .ptd key. + EXPECT_EQ(module.load(), first_error); + std::remove(bad_program_path.c_str()); +} + TEST_F(ModuleTest, TestPTD_Multiple) { std::vector data_files = {add_mul_data_path_, linear_data_path_};