From a113333351351e9fc88943ed38950e470e6be07a Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 14:41:15 +0000 Subject: [PATCH 01/32] writer/adios2: add tests first --- src/libpsc/tests/test_mfields_io.cxx | 50 ++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/libpsc/tests/test_mfields_io.cxx b/src/libpsc/tests/test_mfields_io.cxx index cd62490c6..0dc1ae182 100644 --- a/src/libpsc/tests/test_mfields_io.cxx +++ b/src/libpsc/tests/test_mfields_io.cxx @@ -40,6 +40,56 @@ static Grid_t make_grid() return Grid_t{domain, bc, kinds, norm, dt, -1, {2, 2, 2}}; } +#ifdef PSC_HAVE_ADIOS2 +using WriterTestTypes = ::testing::Types; +#else +using WriterTestTypes = ::testing::Types; +#endif + +template +class WriterTest : public ::testing::Test +{}; + +TYPED_TEST_SUITE(WriterTest, WriterTestTypes); + +TYPED_TEST(WriterTest, WriterSingle) +{ + using Writer = TypeParam; + auto grid = make_grid(); + auto mflds = MfieldsSingle{grid, NR_FIELDS, {2, 2, 2}}; + + setupFields(mflds, [](int m, double crd[3]) { + return m + crd[0] + 100 * crd[1] + 10000 * crd[2]; + }); + + Writer writer; + writer.open("test1", "."); + writer.write_step(grid, {}, {}, + mflds.gt().view(_s(2, -2), _s(2, -2), _s(2, -2)), "mflds", + {"c0", "c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8"}); + writer.close(); +} + +TYPED_TEST(WriterTest, WriterMultiple) +// test writing multiple fields, with different numbers of components, to the +// same file +{ + using Writer = TypeParam; + auto grid = make_grid(); + auto mflds1 = MfieldsSingle{grid, 1, {2, 2, 2}}; + auto mflds2 = MfieldsSingle{grid, 2, {2, 2, 2}}; + + Writer writer; + writer.open("test2", "."); + writer.begin_step(grid); + writer.write(mflds1.gt().view(_s(2, -2), _s(2, -2), _s(2, -2)), grid, + "mflds1", {"comp"}); + writer.write(mflds2.gt().view(_s(2, -2), _s(2, -2), _s(2, -2)), grid, + "mflds2", {"comp1", "comp2"}); + writer.end_step(); + writer.close(); +} + template class MfieldsTest : public ::testing::Test {}; From 5c7b38eda5f6e00efea2e89662f847b203cc905c Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 27 May 2026 16:09:46 +0000 Subject: [PATCH 02/32] writer_adios2: consolidate _begin_step() --- src/include/writer_adios2.hxx | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index fec6ad15b..2b41f209e 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -69,20 +69,26 @@ public: dir_.clear(); } + static void _begin_step(kg::io::Engine& file, int step, double time, + const Double3& length, const Double3& corner) + { + file.beginStep(kg::io::StepMode::Append); + file.put("step", step); + file.put("time", time); + file.put("length", length); + file.put("corner", corner); + } + void begin_step(const Grid_t& grid) { int step = grid.timestep(); - double time = grid.time(); int len = dir_.size() + pfx_.size() + 20; char filename[len]; snprintf(filename, len, "%s/%s.%09d.bp", dir_.c_str(), pfx_.c_str(), step); file_ = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); - file_.beginStep(kg::io::StepMode::Append); - file_.put("step", step); - file_.put("time", time); - file_.put("length", grid.domain.length); - file_.put("corner", grid.domain.corner); + _begin_step(file_, step, grid.time(), grid.domain.length, + grid.domain.corner); file_.performPuts(); } @@ -185,11 +191,7 @@ public: prof_stop(pr_lock); auto file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); - file.beginStep(kg::io::StepMode::Append); - file.put("step", step); - file.put("time", time); - file.put("length", length); - file.put("corner", corner); + _begin_step(file, step, time, length, corner); prof_start(pr_adios2); file.put("ib", ib, launch); From 3620c4ab2e6d97b7954b208b74f18bd549301b97 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 02:27:06 +0000 Subject: [PATCH 03/32] writer/adios2: same attribute prefix This makes the USE_THREADS output file follow the same convention as in the unthreaded case. --- src/include/writer_adios2.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 2b41f209e..3e896185f 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -194,10 +194,10 @@ public: _begin_step(file, step, time, length, corner); prof_start(pr_adios2); + file.prefixes_.push_back(name); file.put("ib", ib, launch); file.put("im", im, launch); - file.prefixes_.push_back(name); auto shape = makeDims(n_comps, gdims); for (int p = 0; p < n_patches; p++) { auto start = makeDims(0, patch_off[p]); From 0784f4f95d7de33e0ce1165cf5d1e0a6829b71cc Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 02:41:32 +0000 Subject: [PATCH 04/32] writer/adios2: separate write_combined() function --- src/include/writer_adios2.hxx | 65 ++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 3e896185f..d2af62d45 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -121,6 +121,41 @@ public: prof_stop(pr_write); } + template + static void write_combined(kg::io::Engine& file, int step, double time, + const Double3& length, const Double3& corner, + const Int3& ldims, const Int3& gdims, + const std::vector& patch_off, + const E& h_expr, const std::string& name, + const std::vector& comp_names) + { + auto launch = kg::io::Mode::Blocking; + + int n_comps = h_expr.shape(3); + int n_patches = h_expr.shape(4); + Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; + Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, + -(im[2] - ldims[2]) / 2}; + + _begin_step(file, step, time, length, corner); + file.prefixes_.push_back(name); + file.put("ib", ib, launch); + file.put("im", im, launch); + + auto shape = makeDims(n_comps, gdims); + for (int p = 0; p < n_patches; p++) { + auto start = makeDims(0, patch_off[p]); + auto count = makeDims(n_comps, ldims); + auto _ib = makeDims(0, -ib); + auto _im = makeDims(n_comps, im); + file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, + {start, count}, {_ib, _im}); + } + file.prefixes_.pop_back(); + file.performPuts(); + file.endStep(); + } + template void write_step(const Grid_t& grid, const Int3& rn, const Int3& rx, const E& expr, const std::string& name, @@ -171,45 +206,21 @@ public: prof_start(pr_thread); - int n_comps = h_expr.shape(3); - int n_patches = h_expr.shape(4); - Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, - -(im[2] - ldims[2]) / 2}; - int len = dir_.size() + pfx_.size() + 20; char filename[len]; snprintf(filename, len, "%s/%s.%09d.bp", dir_.c_str(), pfx_.c_str(), step); { - auto launch = kg::io::Mode::Blocking; - // FIXME not sure how necessary this lock really is, it certainly could // spin for a long time if another thread is writing another file prof_start(pr_lock); // std::lock_guard guard(writer_mutex); prof_stop(pr_lock); - auto file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); - - _begin_step(file, step, time, length, corner); prof_start(pr_adios2); - file.prefixes_.push_back(name); - file.put("ib", ib, launch); - file.put("im", im, launch); - - auto shape = makeDims(n_comps, gdims); - for (int p = 0; p < n_patches; p++) { - auto start = makeDims(0, patch_off[p]); - auto count = makeDims(n_comps, ldims); - auto _ib = makeDims(0, -ib); - auto _im = makeDims(n_comps, im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, - {start, count}, {_ib, _im}); // FIXME cast - } - file.prefixes_.pop_back(); - file.performPuts(); - file.endStep(); + auto file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); + write_combined(file, step, time, length, corner, ldims, gdims, + patch_off, h_expr, name, comp_names); file.close(); prof_stop(pr_adios2); } From 9a90493cf59f6d3ddb14805c74e1806bbdda5991 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 02:43:51 +0000 Subject: [PATCH 05/32] writer/adios2: rm unused/unsupported set_subset() --- src/include/writer_adios2.hxx | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index d2af62d45..72d2e6d7f 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -94,8 +94,6 @@ public: void end_step() { file_.close(); } - void set_subset(const Grid_t& grid, Int3 rn, Int3 rx) {} - template void write(const E& expr, const Grid_t& grid, const std::string& name, const std::vector& comp_names) @@ -235,7 +233,6 @@ public: #else prof_start(pr_write); begin_step(grid); - set_subset(grid, rn, rx); write(expr, grid, name, comp_names); end_step(); prof_stop(pr_write); From d2808761873580b6bf01e953fa72de32c626b345 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 02:47:24 +0000 Subject: [PATCH 06/32] writer/adios2: make _begin_step() private --- src/include/writer_adios2.hxx | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 72d2e6d7f..a04e9c6f8 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -69,16 +69,6 @@ public: dir_.clear(); } - static void _begin_step(kg::io::Engine& file, int step, double time, - const Double3& length, const Double3& corner) - { - file.beginStep(kg::io::StepMode::Append); - file.put("step", step); - file.put("time", time); - file.put("length", length); - file.put("corner", corner); - } - void begin_step(const Grid_t& grid) { int step = grid.timestep(); @@ -241,6 +231,16 @@ public: } private: + static void _begin_step(kg::io::Engine& file, int step, double time, + const Double3& length, const Double3& corner) + { + file.beginStep(kg::io::StepMode::Append); + file.put("step", step); + file.put("time", time); + file.put("length", length); + file.put("corner", corner); + } + #ifdef PSC_USE_IO_THREADS void thread_func() { From f498481c709511414cb429c72ce61cf8eb3431d5 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 02:54:43 +0000 Subject: [PATCH 07/32] writer/adios2: use grid.time() --- src/include/writer_adios2.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index a04e9c6f8..e5196e5c1 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -174,7 +174,7 @@ public: #ifdef PSC_USE_IO_THREADS prof_start(pr_write); int step = grid.timestep(); - double time = step * grid.dt; + double time = grid.time(); assert(grid.n_patches() == h_expr.shape(4)); Int3 ldims = grid.ldims; From 2279bc9d6bfe89d7aef6e498538aa89140f2bddf Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 14:51:37 +0000 Subject: [PATCH 08/32] writer/adios2: mv _begin_step() and end_step() up --- src/include/writer_adios2.hxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index e5196e5c1..255c683db 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -125,7 +125,6 @@ public: Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, -(im[2] - ldims[2]) / 2}; - _begin_step(file, step, time, length, corner); file.prefixes_.push_back(name); file.put("ib", ib, launch); file.put("im", im, launch); @@ -141,7 +140,6 @@ public: } file.prefixes_.pop_back(); file.performPuts(); - file.endStep(); } template @@ -207,8 +205,10 @@ public: prof_start(pr_adios2); auto file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); + _begin_step(file, step, time, length, corner); write_combined(file, step, time, length, corner, ldims, gdims, patch_off, h_expr, name, comp_names); + file.endStep(); file.close(); prof_stop(pr_adios2); } From 1780192777c5d31b960e642c83141c46cc381ba3 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 14:54:49 +0000 Subject: [PATCH 09/32] writer/adios2: add missing endStep() call --- src/include/writer_adios2.hxx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 255c683db..e7b8784ae 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -82,7 +82,11 @@ public: file_.performPuts(); } - void end_step() { file_.close(); } + void end_step() + { + file_.endStep(); + file_.close(); + } template void write(const E& expr, const Grid_t& grid, const std::string& name, From 90ab314908f873580711c7f0faa2e47c4e0271e3 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 14:57:10 +0000 Subject: [PATCH 10/32] writer/adios2: consolidated _end_step() --- src/include/writer_adios2.hxx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index e7b8784ae..e5c1d3c02 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -82,11 +82,7 @@ public: file_.performPuts(); } - void end_step() - { - file_.endStep(); - file_.close(); - } + void end_step() { _end_step(file_); } template void write(const E& expr, const Grid_t& grid, const std::string& name, @@ -212,8 +208,7 @@ public: _begin_step(file, step, time, length, corner); write_combined(file, step, time, length, corner, ldims, gdims, patch_off, h_expr, name, comp_names); - file.endStep(); - file.close(); + _end_step(file); prof_stop(pr_adios2); } @@ -245,6 +240,12 @@ private: file.put("corner", corner); } + static void _end_step(kg::io::Engine& file) + { + file.endStep(); + file.close(); + } + #ifdef PSC_USE_IO_THREADS void thread_func() { From 0d8516c503fa24c9bfa1834c23a593ef56742674 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:01:57 +0000 Subject: [PATCH 11/32] writer/adios2: open file in _begin_step() --- src/include/writer_adios2.hxx | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index e5c1d3c02..33615b4b4 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -71,13 +71,7 @@ public: void begin_step(const Grid_t& grid) { - int step = grid.timestep(); - - int len = dir_.size() + pfx_.size() + 20; - char filename[len]; - snprintf(filename, len, "%s/%s.%09d.bp", dir_.c_str(), pfx_.c_str(), step); - file_ = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); - _begin_step(file_, step, grid.time(), grid.domain.length, + _begin_step(file_, grid.timestep(), grid.time(), grid.domain.length, grid.domain.corner); file_.performPuts(); } @@ -192,10 +186,6 @@ public: prof_start(pr_thread); - int len = dir_.size() + pfx_.size() + 20; - char filename[len]; - snprintf(filename, len, "%s/%s.%09d.bp", dir_.c_str(), pfx_.c_str(), - step); { // FIXME not sure how necessary this lock really is, it certainly could // spin for a long time if another thread is writing another file @@ -204,7 +194,7 @@ public: prof_stop(pr_lock); prof_start(pr_adios2); - auto file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); + kg::io::Engine file; _begin_step(file, step, time, length, corner); write_combined(file, step, time, length, corner, ldims, gdims, patch_off, h_expr, name, comp_names); @@ -230,9 +220,14 @@ public: } private: - static void _begin_step(kg::io::Engine& file, int step, double time, - const Double3& length, const Double3& corner) + void _begin_step(kg::io::Engine& file, int step, double time, + const Double3& length, const Double3& corner) { + int len = dir_.size() + pfx_.size() + 20; + char filename[len]; + snprintf(filename, len, "%s/%s.%09d.bp", dir_.c_str(), pfx_.c_str(), step); + file = io_.open(filename, kg::io::Mode::Write, comm_, pfx_); + file.beginStep(kg::io::StepMode::Append); file.put("step", step); file.put("time", time); From 6d666430944f2af81272a619b3f68df9b9040919 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:05:16 +0000 Subject: [PATCH 12/32] writer/adios2: rm unused args --- src/include/writer_adios2.hxx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 33615b4b4..8c4667759 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -104,9 +104,8 @@ public: } template - static void write_combined(kg::io::Engine& file, int step, double time, - const Double3& length, const Double3& corner, - const Int3& ldims, const Int3& gdims, + static void write_combined(kg::io::Engine& file, const Int3& ldims, + const Int3& gdims, const std::vector& patch_off, const E& h_expr, const std::string& name, const std::vector& comp_names) @@ -196,8 +195,7 @@ public: prof_start(pr_adios2); kg::io::Engine file; _begin_step(file, step, time, length, corner); - write_combined(file, step, time, length, corner, ldims, gdims, - patch_off, h_expr, name, comp_names); + write_combined(file, ldims, gdims, patch_off, h_expr, name, comp_names); _end_step(file); prof_stop(pr_adios2); } From 5c4cf0b5c76239d2e88dcd3f50681fb74ceaf716 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:09:44 +0000 Subject: [PATCH 13/32] writer/adios2: use write_combined() always --- src/include/writer_adios2.hxx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 8c4667759..42625b112 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -93,13 +93,15 @@ public: auto&& evaluated = gt::eval(const_cast(expr)); auto&& h_expr = gt::host_mirror(evaluated); gt::copy(evaluated, h_expr); - Mfields> h_mflds(grid, h_expr.shape(3), {}); - h_mflds.gt() = h_expr; prof_stop(pr_eval); prof_start(pr_write); - file_.put(name, h_mflds); - file_.performPuts(); + std::vector patch_off(grid.n_patches()); + for (int p = 0; p < grid.n_patches(); p++) { + patch_off[p] = grid.patches[p].off; + } + write_combined(file_, grid.ldims, grid.domain.gdims, patch_off, h_expr, + name, comp_names); prof_stop(pr_write); } From 6d2614afe59091cf3d6e74ceeff6665cf98e2c18 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:11:50 +0000 Subject: [PATCH 14/32] writer/adios2: rename write_combined() -> write() --- src/include/writer_adios2.hxx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 42625b112..9c206fc60 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -100,17 +100,16 @@ public: for (int p = 0; p < grid.n_patches(); p++) { patch_off[p] = grid.patches[p].off; } - write_combined(file_, grid.ldims, grid.domain.gdims, patch_off, h_expr, - name, comp_names); + write(file_, grid.ldims, grid.domain.gdims, patch_off, h_expr, name, + comp_names); prof_stop(pr_write); } template - static void write_combined(kg::io::Engine& file, const Int3& ldims, - const Int3& gdims, - const std::vector& patch_off, - const E& h_expr, const std::string& name, - const std::vector& comp_names) + static void write(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, + const std::vector& patch_off, const E& h_expr, + const std::string& name, + const std::vector& comp_names) { auto launch = kg::io::Mode::Blocking; @@ -197,7 +196,7 @@ public: prof_start(pr_adios2); kg::io::Engine file; _begin_step(file, step, time, length, corner); - write_combined(file, ldims, gdims, patch_off, h_expr, name, comp_names); + write(file, ldims, gdims, patch_off, h_expr, name, comp_names); _end_step(file); prof_stop(pr_adios2); } From 25196b80ddbad4a32dab2ff3f1e36ac74eba5dbd Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:15:14 +0000 Subject: [PATCH 15/32] writer/adios2: make write() wrapper for _write_4d() --- src/include/writer_adios2.hxx | 55 ++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 9c206fc60..a51d1a912 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -111,29 +111,7 @@ public: const std::string& name, const std::vector& comp_names) { - auto launch = kg::io::Mode::Blocking; - - int n_comps = h_expr.shape(3); - int n_patches = h_expr.shape(4); - Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, - -(im[2] - ldims[2]) / 2}; - - file.prefixes_.push_back(name); - file.put("ib", ib, launch); - file.put("im", im, launch); - - auto shape = makeDims(n_comps, gdims); - for (int p = 0; p < n_patches; p++) { - auto start = makeDims(0, patch_off[p]); - auto count = makeDims(n_comps, ldims); - auto _ib = makeDims(0, -ib); - auto _im = makeDims(n_comps, im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, - {start, count}, {_ib, _im}); - } - file.prefixes_.pop_back(); - file.performPuts(); + _write_4d(file, ldims, gdims, patch_off, h_expr, name, comp_names); } template @@ -240,6 +218,37 @@ private: file.close(); } + template + static void _write_4d(kg::io::Engine& file, const Int3& ldims, + const Int3& gdims, const std::vector& patch_off, + const E& h_expr, const std::string& name, + const std::vector& comp_names) + { + auto launch = kg::io::Mode::Blocking; + + int n_comps = h_expr.shape(3); + int n_patches = h_expr.shape(4); + Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; + Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, + -(im[2] - ldims[2]) / 2}; + + file.prefixes_.push_back(name); + file.put("ib", ib, launch); + file.put("im", im, launch); + + auto shape = makeDims(n_comps, gdims); + for (int p = 0; p < n_patches; p++) { + auto start = makeDims(0, patch_off[p]); + auto count = makeDims(n_comps, ldims); + auto _ib = makeDims(0, -ib); + auto _im = makeDims(n_comps, im); + file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, + {start, count}, {_ib, _im}); + } + file.prefixes_.pop_back(); + file.performPuts(); + } + #ifdef PSC_USE_IO_THREADS void thread_func() { From 02a155189b485e9a4a0cf1dc28519f0f0d8dbab1 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:28:22 +0000 Subject: [PATCH 16/32] writer/adios2: write separate components --- src/include/io_common.h | 7 +++++++ src/include/writer_adios2.hxx | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/include/io_common.h b/src/include/io_common.h index 1040f79c4..c1da14552 100644 --- a/src/include/io_common.h +++ b/src/include/io_common.h @@ -13,4 +13,11 @@ inline kg::io::Dims makeDims(int m, const Int3& dims) static_cast(dims[0])}; } +inline kg::io::Dims makeDims(const Int3& dims) +{ + return kg::io::Dims{static_cast(dims[2]), + static_cast(dims[1]), + static_cast(dims[0])}; +} + } // namespace diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index a51d1a912..8a019ee79 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -111,7 +111,17 @@ public: const std::string& name, const std::vector& comp_names) { +#if 0 _write_4d(file, ldims, gdims, patch_off, h_expr, name, comp_names); +#else + int n_comps = h_expr.shape(3); + assert(n_comps == comp_names.size()); + + for (int m = 0; m < n_comps; m++) { + _write_3d(file, ldims, gdims, patch_off, + h_expr.view(_all, _all, _all, m, _all), comp_names[m]); + } +#endif } template @@ -249,6 +259,35 @@ private: file.performPuts(); } + template + static void _write_3d(kg::io::Engine& file, const Int3& ldims, + const Int3& gdims, const std::vector& patch_off, + const E& h_expr, const std::string& name) + { + auto launch = kg::io::Mode::Blocking; + + int n_patches = h_expr.shape(3); + Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; + Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, + -(im[2] - ldims[2]) / 2}; + + file.prefixes_.push_back(name); + file.put("ib", ib, launch); + file.put("im", im, launch); + + auto shape = makeDims(gdims); + for (int p = 0; p < n_patches; p++) { + auto start = makeDims(patch_off[p]); + auto count = makeDims(ldims); + auto _ib = makeDims(-ib); + auto _im = makeDims(im); + file.putVariable(&h_expr(ib[0], ib[1], ib[2], p), launch, shape, + {start, count}, {_ib, _im}); + } + file.prefixes_.pop_back(); + file.performPuts(); + } + #ifdef PSC_USE_IO_THREADS void thread_func() { From f1b927bb377397c26d4b57cde84b5b5e7fd0b558 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:29:42 +0000 Subject: [PATCH 17/32] writer/adios2: more consistent profiling --- src/include/writer_adios2.hxx | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 8a019ee79..8fc8f817b 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -130,15 +130,13 @@ public: const std::vector& comp_names) { - static int pr, pr_copy, pr_wait, pr_write, pr_thread, pr_lock, pr_adios2; + static int pr, pr_copy, pr_write, pr_thread, pr_lock; if (!pr) { pr = prof_register("write_step", 1., 0, 0); pr_copy = prof_register("ws copy", 1., 0, 0); - pr_wait = prof_register("ws wait", 1., 0, 0); pr_write = prof_register("ws write", 1., 0, 0); pr_thread = prof_register("ws thread", 1., 0, 0); pr_lock = prof_register("ws lock", 1., 0, 0); - pr_adios2 = prof_register("ws adios2", 1., 0, 0); } prof_start(pr); @@ -181,12 +179,12 @@ public: // std::lock_guard guard(writer_mutex); prof_stop(pr_lock); - prof_start(pr_adios2); + prof_start(pr_write); kg::io::Engine file; _begin_step(file, step, time, length, corner); write(file, ldims, gdims, patch_off, h_expr, name, comp_names); _end_step(file); - prof_stop(pr_adios2); + prof_stop(pr_write); } prof_stop(pr_thread); From bd5b9b3d8086e63ceb88744099cabaa2917144f0 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:31:06 +0000 Subject: [PATCH 18/32] writer/adios2: simplify --- src/include/writer_adios2.hxx | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 8fc8f817b..eb5b3015d 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -237,8 +237,7 @@ private: int n_comps = h_expr.shape(3); int n_patches = h_expr.shape(4); Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, - -(im[2] - ldims[2]) / 2}; + Int3 ib = -(im - ldims) / 2; file.prefixes_.push_back(name); file.put("ib", ib, launch); @@ -266,8 +265,7 @@ private: int n_patches = h_expr.shape(3); Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = {-(im[0] - ldims[0]) / 2, -(im[1] - ldims[1]) / 2, - -(im[2] - ldims[2]) / 2}; + Int3 ib = -(im - ldims) / 2; file.prefixes_.push_back(name); file.put("ib", ib, launch); From c94ebcced6f20984762fa0ff208278803ed16839 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Wed, 3 Jun 2026 12:59:07 +0000 Subject: [PATCH 19/32] writer/adios2: write dimensions attr --- src/include/writer_adios2.hxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index eb5b3015d..3a2cff8ec 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -281,6 +281,9 @@ private: {start, count}, {_ib, _im}); } file.prefixes_.pop_back(); + // FIXME, it'd be better to write within the prefix, but xarray-adios2 + // expects a "/" separator instead of "::" + file.put(std::string(name) + "/dimensions", std::string("x y z")); file.performPuts(); } From c6443f080dc0dbe200745a53e5bbf407bd45e688 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Thu, 4 Jun 2026 15:34:47 +0000 Subject: [PATCH 20/32] writer/adios2: write "step_dimension" attr --- src/include/writer_adios2.hxx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 3a2cff8ec..802a721a4 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -218,6 +218,7 @@ private: file.put("time", time); file.put("length", length); file.put("corner", corner); + file.put("step_dimension", std::string("time")); } static void _end_step(kg::io::Engine& file) From 88d46e9639f8544f68b808b26e58cc86fd26575f Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 5 Jun 2026 14:51:03 +0000 Subject: [PATCH 21/32] writer/adios2: mv performPuts -> _begin_step() --- src/include/writer_adios2.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 802a721a4..c5747a661 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -73,7 +73,6 @@ public: { _begin_step(file_, grid.timestep(), grid.time(), grid.domain.length, grid.domain.corner); - file_.performPuts(); } void end_step() { _end_step(file_); } @@ -219,6 +218,7 @@ private: file.put("length", length); file.put("corner", corner); file.put("step_dimension", std::string("time")); + file_.performPuts(); } static void _end_step(kg::io::Engine& file) From 4abf442e1e1a983104a17d84687f36a3a396cfee Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 5 Jun 2026 18:48:12 +0000 Subject: [PATCH 22/32] writer/adios2: write coordinates cell-centered only for now. Also fix the dimensions to be in reverse order (ie., Fortran) while adios2 expects C order. --- src/include/writer_adios2.hxx | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index c5747a661..1b0629a0a 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -72,7 +72,7 @@ public: void begin_step(const Grid_t& grid) { _begin_step(file_, grid.timestep(), grid.time(), grid.domain.length, - grid.domain.corner); + grid.domain.corner, grid.domain.gdims); } void end_step() { _end_step(file_); } @@ -180,7 +180,7 @@ public: prof_start(pr_write); kg::io::Engine file; - _begin_step(file, step, time, length, corner); + _begin_step(file, step, time, length, corner, gdims); write(file, ldims, gdims, patch_off, h_expr, name, comp_names); _end_step(file); prof_stop(pr_write); @@ -205,7 +205,8 @@ public: private: void _begin_step(kg::io::Engine& file, int step, double time, - const Double3& length, const Double3& corner) + const Double3& length, const Double3& corner, + const Int3& gdims) { int len = dir_.size() + pfx_.size() + 20; char filename[len]; @@ -218,6 +219,23 @@ private: file.put("length", length); file.put("corner", corner); file.put("step_dimension", std::string("time")); + + if (file.mpi_rank_ == 0) { + Double3 dx = length / Double3(gdims); + for (int d = 0; d < 3; d++) { + auto crd = gt::zeros({gdims[d]}); + for (int i = 0; i < gdims[d]; i++) { + crd(i) = corner[d] + (i + .5) * dx[d]; + } + std::string name = d == 0 ? "x" : d == 1 ? "y" : "z"; + file.prefixes_.push_back(name); + file.putVariable(crd.data(), kg::io::Mode::Blocking, + kg::io::Dims({gdims[d]}), {}, {}); + file.prefixes_.pop_back(); + file.put(std::string(name) + "/dimensions", name); + } + } + file_.performPuts(); } @@ -284,7 +302,7 @@ private: file.prefixes_.pop_back(); // FIXME, it'd be better to write within the prefix, but xarray-adios2 // expects a "/" separator instead of "::" - file.put(std::string(name) + "/dimensions", std::string("x y z")); + file.put(std::string(name) + "/dimensions", std::string("z y x")); file.performPuts(); } From 5310814254c07568c0b0950a147241e8381d405b Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 5 Jun 2026 18:50:21 +0000 Subject: [PATCH 23/32] writer/adios2: support scalar File::PutVariable --- src/kg/include/kg/io/File.h | 2 ++ src/kg/include/kg/io/File.inl | 8 ++++++++ src/kg/include/kg/io/FileAdios2.inl | 4 ++++ src/kg/testing/io/TestIOAdios2.cxx | 13 +++++++++++++ 4 files changed, 27 insertions(+) diff --git a/src/kg/include/kg/io/File.h b/src/kg/include/kg/io/File.h index b1d81b746..f27a7ba01 100644 --- a/src/kg/include/kg/io/File.h +++ b/src/kg/include/kg/io/File.h @@ -31,6 +31,8 @@ class File void putVariable(const std::string& name, const T* data, Mode launch, const Dims& shape, const Extents& selection, const Extents& memory_selection); + template + void putVariable(const std::string& name, const T& datum); template void getVariable(const std::string& name, T* data, Mode launch, diff --git a/src/kg/include/kg/io/File.inl b/src/kg/include/kg/io/File.inl index 873208e8e..266f02969 100644 --- a/src/kg/include/kg/io/File.inl +++ b/src/kg/include/kg/io/File.inl @@ -37,6 +37,14 @@ inline void File::putVariable(const std::string& name, const T* data, impl_->putVariable(name, dataVar, launch, shape, selection, memory_selection); } +template +inline void File::putVariable(const std::string& name, const T& datum) +{ + assert(impl_); + FileBase::TypeConstPointer datumVar = &datum; + impl_->putVariable(name, datumVar, kg::io::Mode::Blocking, {}, {}, {}); +} + template inline void File::getVariable(const std::string& name, T* data, Mode launch, const Extents& selection, diff --git a/src/kg/include/kg/io/FileAdios2.inl b/src/kg/include/kg/io/FileAdios2.inl index 1926d9678..6a266ae69 100644 --- a/src/kg/include/kg/io/FileAdios2.inl +++ b/src/kg/include/kg/io/FileAdios2.inl @@ -71,6 +71,10 @@ inline void FileAdios2::putVariable(const std::string& name, const T* data, } if (!selection.start.empty()) { v.SetSelection({selection.start, selection.count}); + } else { + if (!shape.empty()) { + v.SetSelection({kg::io::Dims({0}), shape}); + } } if (!memory_selection.start.empty()) { v.SetMemorySelection({memory_selection.start, memory_selection.count}); diff --git a/src/kg/testing/io/TestIOAdios2.cxx b/src/kg/testing/io/TestIOAdios2.cxx index 8ef6c2b33..f5dd31620 100644 --- a/src/kg/testing/io/TestIOAdios2.cxx +++ b/src/kg/testing/io/TestIOAdios2.cxx @@ -68,6 +68,19 @@ TEST(IOAdios2, FilePutGetVariable) } } +TEST(IOAdios2, FilePutGetVariableScalar) +{ + auto io = kg::io::IOAdios2{}; + { + auto file = io.openFile("test3.bp", kg::io::Mode::Write); + file.beginStep(kg::io::StepMode::Append); + double dbl = 99.; + file.putVariable("dbl_scalar", dbl); + file.performPuts(); + file.endStep(); + } +} + TEST(IOAdios2, FilePutGetAttribute) { auto io = kg::io::IOAdios2{}; From db3fdbc120a1c97e3f7201206a2e7856743fabb0 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 5 Jun 2026 18:55:33 +0000 Subject: [PATCH 24/32] writer/adios2: support scalar File::GetVariable() --- src/kg/include/kg/io/File.h | 3 +++ src/kg/include/kg/io/File.inl | 10 ++++++++++ src/kg/testing/io/TestIOAdios2.cxx | 13 +++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/kg/include/kg/io/File.h b/src/kg/include/kg/io/File.h index f27a7ba01..d5361611b 100644 --- a/src/kg/include/kg/io/File.h +++ b/src/kg/include/kg/io/File.h @@ -38,6 +38,9 @@ class File void getVariable(const std::string& name, T* data, Mode launch, const Extents& selection, const Extents& memory_selection); + template + void getVariable(const std::string& name, T& datum); + Dims shapeVariable(const std::string& name) const; template diff --git a/src/kg/include/kg/io/File.inl b/src/kg/include/kg/io/File.inl index 266f02969..8f149555c 100644 --- a/src/kg/include/kg/io/File.inl +++ b/src/kg/include/kg/io/File.inl @@ -55,6 +55,16 @@ inline void File::getVariable(const std::string& name, T* data, Mode launch, impl_->getVariable(name, dataVar, launch, selection, memory_selection); } +template +inline void File::getVariable(const std::string& name, T& datum) +{ + assert(impl_); + auto shape = impl_->shapeVariable(name); + assert(shape.size() == 0); + FileBase::TypePointer dataVar = &datum; + impl_->getVariable(name, dataVar, Mode::Blocking, {}, {}); +} + inline Dims File::shapeVariable(const std::string& name) const { assert(impl_); diff --git a/src/kg/testing/io/TestIOAdios2.cxx b/src/kg/testing/io/TestIOAdios2.cxx index f5dd31620..feafeeaf4 100644 --- a/src/kg/testing/io/TestIOAdios2.cxx +++ b/src/kg/testing/io/TestIOAdios2.cxx @@ -79,6 +79,19 @@ TEST(IOAdios2, FilePutGetVariableScalar) file.performPuts(); file.endStep(); } + { + auto file = io.openFile("test3.bp", kg::io::Mode::Read); + file.beginStep(kg::io::StepMode::Read); + + auto shape = file.shapeVariable("dbl_scalar"); + EXPECT_EQ(shape.size(), 0); + double dbl; + file.getVariable("dbl_scalar", dbl); + file.performGets(); + file.endStep(); + + EXPECT_EQ(dbl, 99.); + } } TEST(IOAdios2, FilePutGetAttribute) From d4df79b4a4d705759c06a899a706d9c4784f96db Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Fri, 5 Jun 2026 19:02:07 +0000 Subject: [PATCH 25/32] writer/adios2: write time as variable --- src/include/writer_adios2.hxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 1b0629a0a..a102e02f7 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -216,6 +216,9 @@ private: file.beginStep(kg::io::StepMode::Append); file.put("step", step); file.put("time", time); + // additionally, write time as a variable (above writes it as an attribute) + // so that xarray-adios2 can use it as a dimension coordinate + file.file_.putVariable("time", time); file.put("length", length); file.put("corner", corner); file.put("step_dimension", std::string("time")); From ec094d1acee54d1d550ed8e00f5876270ed77039 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 01:36:49 +0000 Subject: [PATCH 26/32] writer/adios2: fix a warning --- src/include/writer_adios2.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index a102e02f7..a6fd1d4a0 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -233,7 +233,7 @@ private: std::string name = d == 0 ? "x" : d == 1 ? "y" : "z"; file.prefixes_.push_back(name); file.putVariable(crd.data(), kg::io::Mode::Blocking, - kg::io::Dims({gdims[d]}), {}, {}); + kg::io::Dims({size_t(gdims[d])}), {}, {}); file.prefixes_.pop_back(); file.put(std::string(name) + "/dimensions", name); } From f0518ef183d26b8f5a7a5c57e68ab6ebe4be2d2e Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 01:40:30 +0000 Subject: [PATCH 27/32] refactor(io): move ADIOS write helpers to io_common and rename --- src/include/io_common.h | 61 +++++++++++++++++++++++++++++++ src/include/writer_adios2.hxx | 67 ++--------------------------------- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/src/include/io_common.h b/src/include/io_common.h index c1da14552..5e688df60 100644 --- a/src/include/io_common.h +++ b/src/include/io_common.h @@ -20,4 +20,65 @@ inline kg::io::Dims makeDims(const Int3& dims) static_cast(dims[0])}; } +template +inline void write_4d(kg::io::Engine& file, const Int3& ldims, + const Int3& gdims, const std::vector& patch_off, + const E& h_expr, const std::string& name, + const std::vector& comp_names) +{ + auto launch = kg::io::Mode::Blocking; + + int n_comps = h_expr.shape(3); + int n_patches = h_expr.shape(4); + Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; + Int3 ib = -(im - ldims) / 2; + + file.prefixes_.push_back(name); + file.put("ib", ib, launch); + file.put("im", im, launch); + + auto shape = makeDims(n_comps, gdims); + for (int p = 0; p < n_patches; p++) { + auto start = makeDims(0, patch_off[p]); + auto count = makeDims(n_comps, ldims); + auto _ib = makeDims(0, -ib); + auto _im = makeDims(n_comps, im); + file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, + {start, count}, {_ib, _im}); + } + file.prefixes_.pop_back(); + file.performPuts(); +} + +template +inline void write_3d(kg::io::Engine& file, const Int3& ldims, + const Int3& gdims, const std::vector& patch_off, + const E& h_expr, const std::string& name) +{ + auto launch = kg::io::Mode::Blocking; + + int n_patches = h_expr.shape(3); + Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; + Int3 ib = -(im - ldims) / 2; + + file.prefixes_.push_back(name); + file.put("ib", ib, launch); + file.put("im", im, launch); + + auto shape = makeDims(gdims); + for (int p = 0; p < n_patches; p++) { + auto start = makeDims(patch_off[p]); + auto count = makeDims(ldims); + auto _ib = makeDims(-ib); + auto _im = makeDims(im); + file.putVariable(&h_expr(ib[0], ib[1], ib[2], p), launch, shape, + {start, count}, {_ib, _im}); + } + file.prefixes_.pop_back(); + // FIXME, it'd be better to write within the prefix, but xarray-adios2 + // expects a "/" separator instead of "::" + file.put(std::string(name) + "/dimensions", std::string("z y x")); + file.performPuts(); +} + } // namespace diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index a6fd1d4a0..1ba48a3ee 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -111,14 +111,14 @@ public: const std::vector& comp_names) { #if 0 - _write_4d(file, ldims, gdims, patch_off, h_expr, name, comp_names); + write_4d(file, ldims, gdims, patch_off, h_expr, name, comp_names); #else int n_comps = h_expr.shape(3); assert(n_comps == comp_names.size()); for (int m = 0; m < n_comps; m++) { - _write_3d(file, ldims, gdims, patch_off, - h_expr.view(_all, _all, _all, m, _all), comp_names[m]); + write_3d(file, ldims, gdims, patch_off, + h_expr.view(_all, _all, _all, m, _all), comp_names[m]); } #endif } @@ -248,67 +248,6 @@ private: file.close(); } - template - static void _write_4d(kg::io::Engine& file, const Int3& ldims, - const Int3& gdims, const std::vector& patch_off, - const E& h_expr, const std::string& name, - const std::vector& comp_names) - { - auto launch = kg::io::Mode::Blocking; - - int n_comps = h_expr.shape(3); - int n_patches = h_expr.shape(4); - Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = -(im - ldims) / 2; - - file.prefixes_.push_back(name); - file.put("ib", ib, launch); - file.put("im", im, launch); - - auto shape = makeDims(n_comps, gdims); - for (int p = 0; p < n_patches; p++) { - auto start = makeDims(0, patch_off[p]); - auto count = makeDims(n_comps, ldims); - auto _ib = makeDims(0, -ib); - auto _im = makeDims(n_comps, im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, - {start, count}, {_ib, _im}); - } - file.prefixes_.pop_back(); - file.performPuts(); - } - - template - static void _write_3d(kg::io::Engine& file, const Int3& ldims, - const Int3& gdims, const std::vector& patch_off, - const E& h_expr, const std::string& name) - { - auto launch = kg::io::Mode::Blocking; - - int n_patches = h_expr.shape(3); - Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; - Int3 ib = -(im - ldims) / 2; - - file.prefixes_.push_back(name); - file.put("ib", ib, launch); - file.put("im", im, launch); - - auto shape = makeDims(gdims); - for (int p = 0; p < n_patches; p++) { - auto start = makeDims(patch_off[p]); - auto count = makeDims(ldims); - auto _ib = makeDims(-ib); - auto _im = makeDims(im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], p), launch, shape, - {start, count}, {_ib, _im}); - } - file.prefixes_.pop_back(); - // FIXME, it'd be better to write within the prefix, but xarray-adios2 - // expects a "/" separator instead of "::" - file.put(std::string(name) + "/dimensions", std::string("z y x")); - file.performPuts(); - } - #ifdef PSC_USE_IO_THREADS void thread_func() { From 1f5479fd90d6ec847b020f1be8de254cac3df027 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 01:44:45 +0000 Subject: [PATCH 28/32] refactor(io): don't pass comp_names to write_4d() --- src/include/io_common.h | 13 ++++++------- src/include/writer_adios2.hxx | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/include/io_common.h b/src/include/io_common.h index 5e688df60..cf23ea91b 100644 --- a/src/include/io_common.h +++ b/src/include/io_common.h @@ -21,10 +21,9 @@ inline kg::io::Dims makeDims(const Int3& dims) } template -inline void write_4d(kg::io::Engine& file, const Int3& ldims, - const Int3& gdims, const std::vector& patch_off, - const E& h_expr, const std::string& name, - const std::vector& comp_names) +inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, + const std::vector& patch_off, const E& h_expr, + const std::string& name) { auto launch = kg::io::Mode::Blocking; @@ -51,9 +50,9 @@ inline void write_4d(kg::io::Engine& file, const Int3& ldims, } template -inline void write_3d(kg::io::Engine& file, const Int3& ldims, - const Int3& gdims, const std::vector& patch_off, - const E& h_expr, const std::string& name) +inline void write_3d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, + const std::vector& patch_off, const E& h_expr, + const std::string& name) { auto launch = kg::io::Mode::Blocking; diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index 1ba48a3ee..bb2fcfe31 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -111,7 +111,7 @@ public: const std::vector& comp_names) { #if 0 - write_4d(file, ldims, gdims, patch_off, h_expr, name, comp_names); + write_4d(file, ldims, gdims, patch_off, h_expr, name); #else int n_comps = h_expr.shape(3); assert(n_comps == comp_names.size()); From 6e3de776378dbbf4eeb51a7721ae903359026852 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 02:04:28 +0000 Subject: [PATCH 29/32] refactor(io): make write_4d prefix-neutral --- src/include/io_common.h | 7 +++---- src/include/writer_adios2.hxx | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/include/io_common.h b/src/include/io_common.h index cf23ea91b..58fb066d1 100644 --- a/src/include/io_common.h +++ b/src/include/io_common.h @@ -22,8 +22,7 @@ inline kg::io::Dims makeDims(const Int3& dims) template inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, - const std::vector& patch_off, const E& h_expr, - const std::string& name) + const std::vector& patch_off, const E& h_expr) { auto launch = kg::io::Mode::Blocking; @@ -32,7 +31,6 @@ inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; Int3 ib = -(im - ldims) / 2; - file.prefixes_.push_back(name); file.put("ib", ib, launch); file.put("im", im, launch); @@ -45,7 +43,6 @@ inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, {start, count}, {_ib, _im}); } - file.prefixes_.pop_back(); file.performPuts(); } @@ -54,6 +51,8 @@ inline void write_3d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, const std::vector& patch_off, const E& h_expr, const std::string& name) { + // FIXME, should not pass name, but have prefixes set first, but gotta resolve + // separator inconsistency first auto launch = kg::io::Mode::Blocking; int n_patches = h_expr.shape(3); diff --git a/src/include/writer_adios2.hxx b/src/include/writer_adios2.hxx index bb2fcfe31..fd36ebfbe 100644 --- a/src/include/writer_adios2.hxx +++ b/src/include/writer_adios2.hxx @@ -111,7 +111,9 @@ public: const std::vector& comp_names) { #if 0 - write_4d(file, ldims, gdims, patch_off, h_expr, name); + file.prefixes_.push_back(name); + write_4d(file, ldims, gdims, patch_off, h_expr); + file.prefixes_.pop_back(); #else int n_comps = h_expr.shape(3); assert(n_comps == comp_names.size()); From 287940b08a21f964709da0f101a97ebbe4a07315 Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 02:06:15 +0000 Subject: [PATCH 30/32] refactor(io): reuse shared write_4d for Mfields IO --- src/include/fields3d.hxx | 1 + src/include/fields3d.inl | 16 +++++----------- src/include/io_common.h | 11 ++++++----- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/include/fields3d.hxx b/src/include/fields3d.hxx index 69e3b7d15..d015c3fa0 100644 --- a/src/include/fields3d.hxx +++ b/src/include/fields3d.hxx @@ -353,6 +353,7 @@ struct Mfields const Grid_t& grid() const { return *grid_; } auto gt() { return Base::storage().view(); } + auto gt() const { return Base::storage().view(); } template void Foreach_3d(int l, int r, FUNC&& F) const diff --git a/src/include/fields3d.inl b/src/include/fields3d.inl index 42e32514b..4c90a52c5 100644 --- a/src/include/fields3d.inl +++ b/src/include/fields3d.inl @@ -18,19 +18,13 @@ public: void put(kg::io::Engine& writer, const Mfields& mflds, const kg::io::Mode launch = kg::io::Mode::NonBlocking) { - writer.put("ib", mflds.box().ib(), launch); - writer.put("im", mflds.box().im(), launch); - - auto n_comps = mflds.n_comps(); - auto shape = makeDims(n_comps, mflds.gdims()); + const Grid_t& grid = mflds.grid(); + std::vector patchOffsets; for (int p = 0; p < mflds.n_patches(); p++) { - auto start = makeDims(0, mflds.patchOffset(p)); - auto count = makeDims(n_comps, mflds.ldims()); - auto ib = makeDims(0, -mflds.box().ib()); - auto im = makeDims(n_comps, mflds.box().im()); - writer.putVariable(const_cast(mflds)[p].storage().data(), - launch, shape, {start, count}, {ib, im}); // FIXME cast + patchOffsets.push_back(mflds.patchOffset(p)); } + + write_4d(writer, grid.ldims, grid.domain.gdims, patchOffsets, mflds.gt()); } void get(kg::io::Engine& reader, Mfields& mflds, diff --git a/src/include/io_common.h b/src/include/io_common.h index 58fb066d1..872620ea8 100644 --- a/src/include/io_common.h +++ b/src/include/io_common.h @@ -30,7 +30,8 @@ inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, int n_patches = h_expr.shape(4); Int3 im = {h_expr.shape(0), h_expr.shape(1), h_expr.shape(2)}; Int3 ib = -(im - ldims) / 2; - + std::cout << "ib " << ib[0] << " " << ib[1] << " " << ib[2] << "\n"; + std::cout << "im " << im[0] << " " << im[1] << " " << im[2] << "\n"; file.put("ib", ib, launch); file.put("im", im, launch); @@ -40,8 +41,8 @@ inline void write_4d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, auto count = makeDims(n_comps, ldims); auto _ib = makeDims(0, -ib); auto _im = makeDims(n_comps, im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], 0, p), launch, shape, - {start, count}, {_ib, _im}); + file.putVariable(&h_expr(0, 0, 0, 0, p), launch, shape, {start, count}, + {_ib, _im}); } file.performPuts(); } @@ -69,8 +70,8 @@ inline void write_3d(kg::io::Engine& file, const Int3& ldims, const Int3& gdims, auto count = makeDims(ldims); auto _ib = makeDims(-ib); auto _im = makeDims(im); - file.putVariable(&h_expr(ib[0], ib[1], ib[2], p), launch, shape, - {start, count}, {_ib, _im}); + file.putVariable(&h_expr(0, 0, 0, p), launch, shape, {start, count}, + {_ib, _im}); } file.prefixes_.pop_back(); // FIXME, it'd be better to write within the prefix, but xarray-adios2 From 3cea8d1eb2b557bdfdf282da4a31f5d22af0cdfd Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 02:09:59 +0000 Subject: [PATCH 31/32] refactor(io): reuse write_4d in cuda mfields descr --- src/include/psc_fields_cuda.inl | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/include/psc_fields_cuda.inl b/src/include/psc_fields_cuda.inl index 07d77bf91..56cf48d54 100644 --- a/src/include/psc_fields_cuda.inl +++ b/src/include/psc_fields_cuda.inl @@ -23,27 +23,17 @@ public: const kg::io::Mode launch = kg::io::Mode::NonBlocking) { const auto& grid = mflds_cuda.grid(); - auto n_comps = mflds_cuda.n_comps(); writer.put("ibn", mflds_cuda.ibn(), launch); auto&& h_mflds = gt::host_mirror(mflds_cuda.storage()); gt::copy(mflds_cuda.storage(), h_mflds); - auto shape = makeDims(n_comps, grid.domain.gdims); + std::vector patchOffsets; for (int p = 0; p < mflds_cuda.n_patches(); p++) { - auto start = makeDims(0, grid.patches[p].off); - auto count = makeDims(n_comps, grid.ldims); - auto ib = makeDims(0, mflds_cuda.ibn()); - auto shp = mflds_cuda.storage().shape(); - auto im = makeDims(n_comps, {shp[0], shp[1], shp[2]}); - writer.putVariable(&h_mflds(0, 0, 0, 0, p), launch, shape, {start, count}, - {ib, im}); + patchOffsets.push_back(grid.patches[p].off); } - - // host mirror will go away as this function returns, so need - // to write - writer.performPuts(); + write_4d(writer, grid.ldims, grid.domain.gdims, patchOffsets, h_mflds); } void get(kg::io::Engine& reader, Mfields& mflds_cuda, From ad54d44c92ac63b5c9ffe659b24be2076b5b37ad Mon Sep 17 00:00:00 2001 From: Kai Germaschewski Date: Sat, 6 Jun 2026 02:52:17 +0000 Subject: [PATCH 32/32] writer/adios2: fix issue with local variables --- src/kg/include/kg/io/FileAdios2.inl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kg/include/kg/io/FileAdios2.inl b/src/kg/include/kg/io/FileAdios2.inl index 6a266ae69..036719d70 100644 --- a/src/kg/include/kg/io/FileAdios2.inl +++ b/src/kg/include/kg/io/FileAdios2.inl @@ -72,7 +72,7 @@ inline void FileAdios2::putVariable(const std::string& name, const T* data, if (!selection.start.empty()) { v.SetSelection({selection.start, selection.count}); } else { - if (!shape.empty()) { + if (!shape.empty() && shape[0] != LocalValueDim) { v.SetSelection({kg::io::Dims({0}), shape}); } }