Skip to content

Bp5 performance optimization #1756

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Bp5 performance optimization #1756

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2caf62c
Does this really improve serialization?
franzpoeschel May 6, 2025
04982dd
Cache m_is_bp5
franzpoeschel May 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions include/openPMD/IO/ADIOS/ADIOS2File.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,10 @@ class ADIOS2File
* on chosen ADIOS2 engine and can not be explicitly overridden by user.
*/
bool optimizeAttributesStreaming = false;
/*
* Used for a number of BP5-specific optimizations. Written in getEngine().
*/
bool m_is_bp5 = false;

using ParsePreference = Parameter<Operation::OPEN_FILE>::ParsePreference;
ParsePreference parsePreference = ParsePreference::UpFront;
Expand Down
50 changes: 25 additions & 25 deletions src/IO/ADIOS/ADIOS2File.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ void WriteDataset::call(ADIOS2File &ba, detail::BufferedPut &bp)
std::nullopt,
ba.variables());

engine.Put(var, ptr);
auto do_defer =
ba.m_is_bp5 ? adios2::Mode::Sync : adios2::Mode::Deferred;
engine.Put(var, ptr, do_defer);
}
else if constexpr (std::is_same_v<
ptr_type,
Expand Down Expand Up @@ -180,7 +182,9 @@ struct RunUniquePtrPut
bufferedPut.name,
std::nullopt,
ba.variables());
engine.Put(var, ptr);
auto do_defer =
ba.m_is_bp5 ? adios2::Mode::Sync : adios2::Mode::Deferred;
engine.Put(var, ptr, do_defer);
}

static constexpr char const *errorMsg = "RunUniquePtrPut";
Expand Down Expand Up @@ -976,6 +980,14 @@ adios2::Engine &ADIOS2File::getEngine()
{
throw std::runtime_error("[ADIOS2] Failed opening Engine.");
}

m_is_bp5 = m_impl->realEngineType() == "bp5" ||
/* this second check should be sufficient, but we leave the
first check in as a safeguard against renamings in
ADIOS2. Also do a lowerCase transform since the docstring
of `Engine::Type()` claims that the return value is in
lowercase, but for BP5 this does not seem true. */
auxiliary::lowerCase(m_engine->Type()) == "bp5writer";
}
return m_engine.value();
}
Expand Down Expand Up @@ -1096,13 +1108,7 @@ void ADIOS2File::flush_impl(ADIOS2FlushParams flushParams, bool writeLatePuts)
{
case FlushTarget::Disk:
case FlushTarget::Disk_Override:
if (m_impl->realEngineType() == "bp5" ||
/* this second check should be sufficient, but we leave the
first check in as a safeguard against renamings in
ADIOS2. Also do a lowerCase transform since the docstring
of `Engine::Type()` claims that the return value is in
lowercase, but for BP5 this does not seem true. */
auxiliary::lowerCase(engine.Type()) == "bp5writer")
if (m_is_bp5)
{
target = CleanedFlushTarget::Disk;
}
Expand Down Expand Up @@ -1136,9 +1142,13 @@ void ADIOS2File::flush_impl(ADIOS2FlushParams flushParams, bool writeLatePuts)
m_uniquePtrPuts.clear();
m_updateSpans.clear();
break;
case CleanedFlushTarget::Buffer:
engine.PerformPuts();
break;
case CleanedFlushTarget::Buffer: {
if (!m_is_bp5)
{
engine.PerformPuts();
}
}
break;
case CleanedFlushTarget::Step:
if (streamStatus != StreamStatus::DuringStep)
{
Expand Down Expand Up @@ -1246,16 +1256,6 @@ AdvanceStatus ADIOS2File::advance(AdvanceMode mode)
adios2::StepStatus adiosStatus{};
auto &engine = getEngine();

auto check_bp5 = [&]() -> bool {
std::string engineType = engine.Type();
std::transform(
engineType.begin(),
engineType.end(),
engineType.begin(),
[](unsigned char c) { return std::tolower(c); });
return engineType == "bp5writer";
};

if (engine.CurrentStep() == 0)
{
int max_steps_from_env =
Expand All @@ -1273,13 +1273,13 @@ AdvanceStatus ADIOS2File::advance(AdvanceMode mode)

// Check some conditions on which to now cancel operation due to
// unwieldy metadata sizes in BP5 with group encoding
if (this->m_impl->m_handler->m_encoding ==
if (m_is_bp5 &&
this->m_impl->m_handler->m_encoding ==
IterationEncoding::groupBased &&
this->m_max_steps_bp5.has_value() &&
engine.CurrentStep() >= *this->m_max_steps_bp5 &&
(this->m_mode == adios2::Mode::Write ||
this->m_mode == adios2::Mode::Append) &&
check_bp5())
this->m_mode == adios2::Mode::Append))
{
throw error::OperationUnsupportedInBackend(
"ADIOS2",
Expand Down
Loading