diff --git a/CMakeLists.txt b/CMakeLists.txt index f2dcc98f82..0416c118cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -394,6 +394,7 @@ include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) set(CORE_SOURCE src/config.cpp src/ChunkInfo.cpp + src/CustomHierarchy.cpp src/Dataset.cpp src/Datatype.cpp src/Error.cpp @@ -426,6 +427,11 @@ set(CORE_SOURCE src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp + src/backend/scientific_defaults/ScientificDefaults.cpp + src/backend/scientific_defaults/ScientificDefaults_auxiliary.cpp + src/backend/scientific_defaults/ProcessParsedAttribute.cpp + src/backend/scientific_defaults/AttributeReader.cpp + src/backend/scientific_defaults/ConfigAttribute.cpp src/auxiliary/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp @@ -827,6 +833,7 @@ if(openPMD_BUILD_TESTING) list(APPEND ${out_list} test/Files_Core/automatic_variable_encoding.cpp test/Files_Core/read_nonexistent_attribute.cpp + test/Files_Core/custom_hierarchy.cpp ) endif() endmacro() diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 01bd2fcb33..e38ab3daa9 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -82,6 +82,8 @@ int main() pos.resetDataset(dataset); pos.storeChunk(local_data, Offset{0}, global_extent); } + auto ch = iteration.customHierarchies(); + ch["rabimmel"].setAttribute("rabammel", "rabumm"); iteration.close(); } diff --git a/examples/2_read_serial.cpp b/examples/2_read_serial.cpp index 4eaac05ea2..2a0826a5eb 100644 --- a/examples/2_read_serial.cpp +++ b/examples/2_read_serial.cpp @@ -31,7 +31,8 @@ using namespace openPMD; int main() { Series series = Series( - "../samples/git-sample/data%T.h5", + // "../samples/git-sample/data%T.h5", + "data.h5", Access::READ_ONLY, R"({"defer_iteration_parsing": true})"); cout << "Read a Series with openPMD standard version " << series.openPMD() @@ -99,9 +100,24 @@ int main() auto all_data = E_x.loadChunk(); + auto ch = series.customHierarchies(); + ch.printRecursively(); + std::cout << "READING 200/fields" << std::endl; + ch["data"]["200"]["fields"].read(); + std::cout << "READING 200/particles" << std::endl; + ch["data"]["200"]["particles"].read(); + std::cout << "READING 300" << std::endl; + ch["data"]["300"].read(0); + ch.printRecursively(); + // The iteration can be closed in order to help free up resources. // The iteration's content will be flushed automatically. i.close(); + std::cout << "OPENING 200" << std::endl; + i = series.snapshots()[200].open(); + ch.printRecursively(); + series.snapshots()[300].open(); + cout << "Full E/x starts with:\n\t{"; for (size_t col = 0; col < extent[1] && col < 5; ++col) cout << all_data.get()[col] << ", "; diff --git a/include/openPMD/CustomHierarchy.hpp b/include/openPMD/CustomHierarchy.hpp new file mode 100644 index 0000000000..f29ffa5738 --- /dev/null +++ b/include/openPMD/CustomHierarchy.hpp @@ -0,0 +1,206 @@ +/* Copyright 2023 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/RecordComponent.hpp" +#include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Container.hpp" + +#include +#include +#include +#include + +namespace openPMD +{ +class CustomHierarchy; +namespace internal +{ + using CustomHierarchyData = ContainerData; +} // namespace internal + +class CustomHierarchy; + +/* + * This is its own class, so the return value of asContainerOf() is also + * convsersible again. + */ +template +class ConvertibleContainer : public Container +{ + template + friend class ConversibleContainer; + friend class CustomHierarchy; + +protected: + using Container_t = Container; + using Data_t = internal::ContainerData; + static_assert( + std::is_base_of_v); + + using Container_t::Container_t; + +private: + explicit ConvertibleContainer() = default; + +public: + template + auto asContainerOf() -> ConvertibleContainer + { + if constexpr ( + std::is_same_v || + std::is_same_v || + std::is_same_v || + std::is_same_v) + { + // TODO: If Mesh or ParticleSpecies, create Container on the fly by + // evaluating the meshes/particles path. If RecordComponent, maybe + // use dynamic casting to check if children are datasets? + throw std::runtime_error("UNIMPLEMENTED"); + } + else + { + static_assert( + auxiliary::dependent_false_v, + "[CustomHierarchy::asContainerOf] Type parameter must be " + "one of: CustomHierarchy, RecordComponent, Mesh, " + "ParticleSpecies."); + } + } +}; + +namespace traits +{ + template <> + struct DeferredInitPolicy> + { + template + static void call(Container_const_or_not &); + }; + + template <> + struct GenerationPolicy + { + constexpr static bool is_noop = false; + template + void operator()(Container &cont, Iterator &it) + { + auto &writable = it->second.writable(); + + // These should be different + auto child_shared_data = &it->second.Attributable::get(); + auto parent_shared_data = &cont.Attributable::get(); + if (child_shared_data == parent_shared_data) + { + throw std::runtime_error( + "Trying to emplace object as its own child"); + } + + // These might be different, but might also be the same + // + // For an explanation, ref. the documentation of + // Writable::attributable: This is a pointer back to the first + // created Attributable instance linking this Writable. There might + // be multiple Attributable objects linking the same backend + // Writable object when opening multiple "views" on the same backend + // object, e.g. when a scalar Record is at the same time a + // RecordComponent, or when reopening an object as a + // CustomHierarchy. + // + // Since CustomHierarchy performs no memory management by default, + // we must ensure that the backpointer in Writable::attributable + // remains valid when the frontend instance pointed by + // Writable::attributable *is* the CustomHierarchy instance (happens + // when it is the first frontend object created for that backend + // object). + auto backpointer = writable.attributable; + auto emplaced_pointer = it->second.m_attri.get(); + if (backpointer == emplaced_pointer) + { + (**cont.m_attri) + .m_children_managed_as_custom_hierarchy[it->first] = + it->second; + } + } + }; +} // namespace traits + +// TODO: Use Container internally, but otherwise override members +// such that we have: +// +// operator[](key) -> CustomHierarchy +// +// Or find a better solution for having this automatically.. +class CustomHierarchy : public ConvertibleContainer +{ + friend class Iteration; + friend class Container; + friend class Attributable; + friend struct traits::DeferredInitPolicy>; + +private: + using Parent_t = ConvertibleContainer; + using Container_t = typename Parent_t::Container_t; + using Data_t = typename Parent_t::Data_t; + +protected: + CustomHierarchy(NoInit); + CustomHierarchy(std::shared_ptr other); + CustomHierarchy(Attributable const &other); + + void read(std::vector ¤tPath); + + void flush_internal( + internal::FlushParams const &, std::vector currentPath); + void flush(std::string const &path, internal::FlushParams const &) override; + + /** + * @brief Link with parent. + * + * @param w The Writable representing the parent. + */ + void linkHierarchy(Writable &w) override; + +public: + CustomHierarchy(); + + CustomHierarchy(CustomHierarchy const &other) = default; + CustomHierarchy(CustomHierarchy &&other) = default; + + CustomHierarchy &operator=(CustomHierarchy const &) = default; + CustomHierarchy &operator=(CustomHierarchy &&) = default; + + // TODO maybe make this automatic somehow + // set max_recursion_depth = 0 for infinite cycling + // recursion depth includes the current object + // recursion will not continue expanding into regions that are already known + // (hence not transitively expand into unknown subregions of known regions) + void read(size_t max_recursion_depth = 1); + + void printRecursively(); + +private: + void printRecursively(std::string indent); +}; +} // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,66 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -121,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index abce61eef7..ccaf21d1ae 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -27,6 +27,9 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/PerIterationData.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -120,14 +123,16 @@ namespace internal */ bool allow_reopening_implicitly = false; - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; /** * Cached copy of the key under which this Iteration lives in @@ -145,13 +150,28 @@ namespace internal std::optional m_deferredParseAccess{}; }; } // namespace internal + +class Meshes : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; + +class Particles : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; + /** @brief Logical compilation of data from one snapshot (e.g. a single * simulation cycle). * * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration + : public Attributable + , internal::ScientificDefaults { template friend class Container; @@ -164,6 +184,8 @@ class Iteration : public Attributable friend class StatefulSnapshotsContainer; template friend struct traits::GenerationPolicy; + friend class internal::ScientificDefaults; + friend class Attributable; public: Iteration(Iteration const &) = default; @@ -279,8 +301,10 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + + Meshes meshes{}; + Particles particles{}; virtual ~Iteration() = default; @@ -442,7 +466,7 @@ class Iteration : public Attributable * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w) override; /** * @brief Access an iteration in read mode that has potentially not been @@ -450,6 +474,10 @@ class Iteration : public Attributable * */ void runDeferredParseAccess(); + +protected: + void scientificDefaults_impl( + internal::WriteOrRead, OpenpmdStandard) override; }; // Iteration namespace traits @@ -458,8 +486,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(Iterator &it) + template + void operator()(Container &, Iterator &it) { it->second.get().m_iterationIndex = it->first; } @@ -513,4 +541,10 @@ class IndexedIteration : public Iteration : Iteration(std::forward(it)), iterationIndex(index) {} }; + +class Iterations : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; } // namespace openPMD diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index d0bf81ddef..bf8ad794fb 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -23,7 +23,10 @@ #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults_auxiliary.hpp" #include #include @@ -41,6 +44,8 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class internal::ScientificDefaults; + friend class Attributable; public: Mesh(Mesh const &) = default; @@ -322,12 +327,19 @@ class Mesh : public BaseRecord typename = std::enable_if_t::value>> Mesh &setTimeOffset(T timeOffset); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: Mesh(); void flush_impl(std::string const &, internal::FlushParams const &) override; void read(); + auto retrieveDimensionality() const -> uint64_t; + +protected: + void scientificDefaults_impl( + internal::WriteOrRead, OpenpmdStandard) override; }; // Mesh template diff --git a/include/openPMD/ParticlePatches.hpp b/include/openPMD/ParticlePatches.hpp index f3c4c0b943..ffe539d5f1 100644 --- a/include/openPMD/ParticlePatches.hpp +++ b/include/openPMD/ParticlePatches.hpp @@ -38,6 +38,8 @@ class ParticlePatches : public Container size_t numPatches() const; ~ParticlePatches() override = default; + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: ParticlePatches() = default; void read(); diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 7309afddef..7b13a39784 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -24,23 +24,32 @@ #include "openPMD/Record.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults_auxiliary.hpp" #include namespace openPMD { -class ParticleSpecies : public Container +class ParticleSpecies + : public Container + , internal::ScientificDefaults { friend class Container; friend class Container; friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; + friend class Attributable; public: ParticlePatches particlePatches; + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: ParticleSpecies(); @@ -53,6 +62,10 @@ class ParticleSpecies : public Container { return m_containerData; } + +protected: + void scientificDefaults_impl( + internal::WriteOrRead, OpenpmdStandard) override; }; namespace traits @@ -61,8 +74,8 @@ namespace traits struct GenerationPolicy { constexpr static bool is_noop = false; - template - void operator()(T &it) + template + void operator()(Container &, T &it) { it->second.particlePatches.linkHierarchy(it->second.writable()); } diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 2bb070365f..af397925dc 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -23,6 +23,7 @@ #include "openPMD/RecordComponent.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -34,6 +35,7 @@ class Record : public BaseRecord friend class Container; friend class Iteration; friend class ParticleSpecies; + friend class internal::ScientificDefaults; public: Record(Record const &) = default; @@ -48,6 +50,8 @@ class Record : public BaseRecord template Record &setTimeOffset(T); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: Record(); @@ -55,6 +59,10 @@ class Record : public BaseRecord flush_impl(std::string const &, internal::FlushParams const &) override; [[nodiscard]] internal::HomogenizeExtents read(); + +protected: + void scientificDefaults_impl( + internal::WriteOrRead, OpenpmdStandard) override; }; // Record template diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d06b4213f4..425eab273b 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -27,6 +27,8 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" // comment to prevent this include from being moved by clang-format #include "openPMD/DatatypeMacros.hpp" @@ -110,7 +112,9 @@ namespace internal template class BaseRecord; -class RecordComponent : public BaseRecordComponent +class RecordComponent + : public BaseRecordComponent + , protected internal::ScientificDefaults { template friend class Container; @@ -128,6 +132,9 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; + friend class Attributable; + friend class CustomHierarchy; public: enum class Allocation @@ -482,11 +489,13 @@ class RecordComponent : public BaseRecordComponent auto visit(Args &&...args) -> decltype(Visitor::template call( std::declval(), std::forward(args)...)); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + static constexpr char const *const SCALAR = "\vScalar"; protected: void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void read(); private: /** @@ -534,12 +543,16 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(); template void verifyChunk(Offset const &, Extent const &) const; void verifyChunk(Datatype, Offset const &, Extent const &) const; + +protected: + void scientificDefaults_impl( + internal::WriteOrRead, OpenpmdStandard) override; }; // RecordComponent namespace internal diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..523f4f1e41 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. @@ -129,7 +130,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 603e540c2b..0031f2be5b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -31,7 +31,9 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include "openPMD/backend/PerIterationData.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/version.hpp" @@ -64,7 +66,6 @@ namespace openPMD class ReadIterations; class StatefulIterator; class Series; -class Series; namespace internal { @@ -100,8 +101,8 @@ namespace internal SeriesData &operator=(SeriesData &&) = delete; using IterationIndex_t = Iteration::IterationIndex_t; - using IterationsContainer_t = Container; - IterationsContainer_t iterations{}; + using IterationsContainer_t = Iterations; + Iterations iterations{}; /** * Series::readIterations() returns an iterator type that modifies the @@ -205,14 +206,18 @@ namespace internal * Detected IO format (backend). */ Format m_format; - /** - * Whether a step is currently active for this iteration. - * Used for group-based iteration layout, see SeriesData.hpp for - * iteration-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. + + /* + * This stores data items that are: + * + * 1. global in group and variable encodings + * 2. per-iteration in file encoding + * + * The struct is stored as part of the Series and as part of each + * Iteration. Access must be distinguished by iteration encoding. */ - StepStatus m_stepStatus = StepStatus::NoStep; + PerIterationData m_perIterationData; + /** * True if a user opts into lazy parsing. */ @@ -261,7 +266,6 @@ namespace internal struct RankTableData { - Attributable m_attributable; std::variant< NoSourceSpecified, SourceSpecifiedViaJSON, @@ -385,7 +389,7 @@ class Series : public Attributable * Type for a container of Iterations indexed by IterationIndex_t. */ using IterationsContainer_t = internal::SeriesData::IterationsContainer_t; - IterationsContainer_t iterations; + Iterations iterations; /** * @brief Is this a usable Series object? @@ -447,6 +451,11 @@ class Series : public Attributable * basePath. */ std::string meshesPath() const; + /** + * @return String representing the path to mesh records, relative(!) to + * basePath. + */ + std::optional meshesPathOptional() const; /** Set the path to mesh * records, relative(!) to basePath. @@ -502,6 +511,11 @@ class Series : public Attributable * basePath. */ std::string particlesPath() const; + /** + * @return String representing the path to particle species, relative(!) to + * basePath. + */ + std::optional particlesPathOptional() const; /** Set the path to groups for each particle * species, relative(!) to basePath. @@ -778,6 +792,8 @@ class Series : public Attributable */ void close(); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + /** * This overrides Attributable::iterationFlush() which will fail on Series. */ @@ -898,9 +914,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); - void flushRankTable(); + void flushRankTable(FlushLevel, Attributable &attributable); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ @@ -983,8 +997,9 @@ OPENPMD_private * least one step was written. * * @param doFlush If true, flush the IO handler. + * @param l This operation must only run at flush level write_datasets */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, @@ -995,6 +1010,9 @@ OPENPMD_private Series &setIterationEncoding_internal( IterationEncoding iterationEncoding, internal::default_or_explicit); + Series &setParticlesPath_internal(std::string const &particlesPath); + Series &setMeshesPath_internal(std::string const &meshesPath); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) diff --git a/include/openPMD/auxiliary/Defer.hpp b/include/openPMD/auxiliary/Defer.hpp index c6bc4e0533..505f5fcd39 100644 --- a/include/openPMD/auxiliary/Defer.hpp +++ b/include/openPMD/auxiliary/Defer.hpp @@ -21,7 +21,8 @@ struct defer_type std::move(functor)(); } - explicit defer_type() = default; + explicit defer_type() : do_run_this(false) + {} struct forwarding_tag {}; diff --git a/include/openPMD/auxiliary/MonadicOperations.hpp b/include/openPMD/auxiliary/MonadicOperations.hpp new file mode 100644 index 0000000000..c96f682bc8 --- /dev/null +++ b/include/openPMD/auxiliary/MonadicOperations.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace openPMD::auxiliary +{ +template +auto optional_and_then(Optional &&arg, F &&then) + -> std::invoke_result_t())> +{ + if (!arg.has_value()) + { + return std::nullopt; + } + return std::forward(then)(*std::forward(arg)); +} +} // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index faf2b29472..b395c199e1 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -124,6 +124,22 @@ namespace detail { constexpr static bool value = true; }; + + template + struct ScalarType + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; } // namespace detail template @@ -141,6 +157,15 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsChar_v = detail::IsChar::value; +template +using ScalarType_t = typename detail::ScalarType::type; + +template +using VectorType_t = std::vector>; + +template +using dependent_const = std::conditional_t; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as @@ -203,6 +228,37 @@ namespace detail // little trick to avoid trailing commas in the macro expansions below template using variant_tail_t = std::variant; + + template