Skip to content

Commit 2e7a4b9

Browse files
committed
Do not add .ptd files twice when Module::load is retried
Module::load opens the .ptd files it was given and adds each one to its list of data map loaders as soon as it is open. When opening a later file fails, or loading one of the data maps fails, the earlier entries stay. A retried load() then adds them a second time, and merging the data maps fails with a duplicate key error, even once the missing file exists. load() now opens every file before it keeps any, forgets the file names once the files are kept, and rebuilds its list of data maps on each attempt. So a retry adds each file once. Successful first loads are unchanged. This builds on the change that moves the data map code into one helper.
1 parent c653379 commit 2e7a4b9

2 files changed

Lines changed: 27 additions & 1 deletion

File tree

‎extension/module/module.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -361,13 +361,21 @@ runtime::Error Module::load_data_maps() {
361361
if (merged_data_map_ || (data_files_.empty() && data_map_loaders_.empty())) {
362362
return runtime::Error::Ok;
363363
}
364+
// Open every file before keeping any, so a failed open can be retried.
365+
std::vector<std::unique_ptr<runtime::DataLoader>> file_loaders;
366+
file_loaders.reserve(data_files_.size());
364367
for (const auto& data_file : data_files_) {
365368
auto data_map_loader_result = make_data_loader(data_file, load_mode_);
366369
if (!data_map_loader_result.ok()) {
367370
return data_map_loader_result.error();
368371
}
369-
data_map_loaders_.push_back(std::move(*data_map_loader_result));
372+
file_loaders.push_back(std::move(*data_map_loader_result));
370373
}
374+
for (auto& file_loader : file_loaders) {
375+
data_map_loaders_.push_back(std::move(file_loader));
376+
}
377+
data_files_.clear();
378+
named_data_maps_.clear();
371379
for (const auto& data_map_loader : data_map_loaders_) {
372380
auto res_flat_tensor = FlatTensorDataMap::load(data_map_loader.get());
373381
if (!res_flat_tensor.ok()) {

‎extension/module/test/module_test.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -643,6 +643,24 @@ TEST_F(ModuleTest, TestSharedProgramLoadsItsDataMap) {
643643
EXPECT_EQ(module.forward(tensor).error(), Error::Ok);
644644
}
645645

646+
TEST_F(ModuleTest, TestLoadRetriesAfterAMissingDataFile) {
647+
const std::string late_path =
648+
::testing::TempDir() + "/module_test_late_data_file.ptd";
649+
std::remove(late_path.c_str());
650+
Module module(
651+
add_mul_path_, std::vector<std::string>{add_mul_data_path_, late_path});
652+
EXPECT_NE(module.load(), Error::Ok);
653+
654+
{
655+
std::ifstream source(linear_data_path_, std::ios::binary);
656+
std::ofstream destination(late_path, std::ios::binary);
657+
destination << source.rdbuf();
658+
}
659+
// The first file must not be added twice by the second attempt.
660+
EXPECT_EQ(module.load(), Error::Ok);
661+
std::remove(late_path.c_str());
662+
}
663+
646664
TEST_F(ModuleTest, TestLoadRetriesAfterAFailedProgramLoad) {
647665
const std::string bad_program_path =
648666
::testing::TempDir() + "/module_test_bad_program.pte";

0 commit comments

Comments
 (0)