From f1829488f9fefb94412fd9e76e5504ac829baa9d Mon Sep 17 00:00:00 2001 From: mfellows Date: Tue, 18 Feb 2020 11:29:36 -0800 Subject: [PATCH 01/40] Changed SQLite open mode to explicitly read-only / no-mutex to improve read-only performance over NFS. --- Source/moja.datarepository/src/providerrelationalsqlite.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index ce31ba8..958c394 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -31,7 +31,7 @@ class SQLiteConnection { } } - if (sqlite3_open(path.c_str(), &_conn) != SQLITE_OK) { + if (sqlite3_open_v2(path.c_str(), &_conn, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, 0) != SQLITE_OK) { BOOST_THROW_EXCEPTION(ConnectionFailedException() << ConnectionError(sqlite3_errmsg(_conn))); } From 2a13d95633f3cb2efe92057e68d4cd42cbee22b8 Mon Sep 17 00:00:00 2001 From: Tlazypanda Date: Thu, 11 Jun 2020 11:26:54 +0530 Subject: [PATCH 02/40] Fix typo in dockerfile.flint.ubuntu.18.04 Signed-off-by: Tlazypanda --- Docker/Dockerfile.flint.ubuntu.18.04 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docker/Dockerfile.flint.ubuntu.18.04 b/Docker/Dockerfile.flint.ubuntu.18.04 index 9d1c720..ea146c6 100644 --- a/Docker/Dockerfile.flint.ubuntu.18.04 +++ b/Docker/Dockerfile.flint.ubuntu.18.04 @@ -1,6 +1,6 @@ # ================================================================================================================== # -# Docker to ubuntu 16.04 image for Moja flint libraries and executables +# Docker to ubuntu 18.04 image for Moja flint libraries and executables # # Building this Docker: # docker build -f Dockerfile.flint.ubuntu.18.04 --build-arg NUM_CPU=4 --build-arg FLINT_BRANCH=develop -t moja/flint:ubuntu-18.04 . From 19df52e8ddef0e41a75c5de79d9ea4e0690d5bd5 Mon Sep 17 00:00:00 2001 From: Tlazypanda Date: Sat, 13 Jun 2020 22:41:03 +0530 Subject: [PATCH 03/40] add badges to readme.md Signed-off-by: Tlazypanda --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a9caee3..1377786 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FLINT Open-source Library -[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors) +[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors) [![License: MPL 2.0](https://img.shields.io/badge/License-MPL%202.0-brightgreen.svg)](https://opensource.org/licenses/MPL-2.0) [![Code Of Conduct](https://img.shields.io/badge/code--of--conduct-Moja%20Global-blue)](https://github.com/moja-global/About_moja_global/blob/master/CODE_OF_CONDUCT.md) [![Contributions](https://img.shields.io/badge/newcomer-friendly-red)](https://github.com/moja-global/About_moja_global/blob/master/CONTRIBUTING.md) [![Chat on Slack](https://img.shields.io/badge/chat-slack-blueviolet)](https://mojaglobal.slack.com/) [![Twitter Handle](https://img.shields.io/badge/twitter-Moja%20Global-darkblue)](https://twitter.com/mojaglobal?lang=en) ## What is `FLINT`? @@ -45,7 +45,7 @@ A fork of a *Vcpkg* repository has been created for the FLINT required libraries ```powershell # bootstrap bootstrap-vcpkg.bat - + # install packages vcpkg.exe install boost-test:x64-windows boost-program-options:x64-windows boost-log:x64-windows turtle:x64-windows zipper:x64-windows poco:x64-windows libpq:x64-windows gdal:x64-windows sqlite3:x64-windows boost-ublas:x64-windows fmt:x64-windows ``` @@ -57,17 +57,17 @@ A fork of a *Vcpkg* repository has been created for the FLINT required libraries cd Source mkdir build cd build - + # now create the Visual Studio Solution (2019) cmake -G "Visual Studio 16 2019" -DCMAKE_INSTALL_PREFIX=C:/Development/Software/moja -DVCPKG_TARGET_TRIPLET=x64-windows -DENABLE_TESTS=OFF -DENABLE_MOJA.MODULES.ZIPPER=OFF -DCMAKE_TOOLCHAIN_FILE=c:\Development\moja-global\vcpkg\scripts\buildsystems\vcpkg.cmake .. - + # OR Visual Studio Solution (2017) cmake -G "Visual Studio 15 2017" -DCMAKE_INSTALL_PREFIX=C:/Development/Software/moja -DVCPKG_TARGET_TRIPLET=x64-windows -DENABLE_TESTS=OFF -DENABLE_MOJA.MODULES.ZIPPER=OFF -DCMAKE_TOOLCHAIN_FILE=c:\Development\moja-global\vcpkg\scripts\buildsystems\vcpkg.cmake .. ``` #### Install Moja Libraries -It is possible to use the Visual Studio moja solution to install built versions of the Moja libraries. To do this you need to set the CMAKE variable '***CMAKE_INSTALL_PREFIX***' to your install path (i.e. "*C:/Development/Software/moja*"). +It is possible to use the Visual Studio moja solution to install built versions of the Moja libraries. To do this you need to set the CMAKE variable '***CMAKE_INSTALL_PREFIX***' to your install path (i.e. "*C:/Development/Software/moja*"). #### Make edits to the Visual Studio Solution using CMake @@ -150,11 +150,11 @@ moja global welcomes a wide range of contributions as explained in [Contributing * You can find FAQs on the [Wiki](https://github.com/moja.global/.github/wiki). * If you have a question about the code, submit [user feedback](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Provide-User-Feedback.md) in the relevant repository -* If you have a general question about a project or repository or moja global, [join moja global](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Join-moja-global.md) and +* If you have a general question about a project or repository or moja global, [join moja global](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Join-moja-global.md) and * [submit a discussion](https://help.github.com/en/articles/about-team-discussions) to the project, repository or moja global [team](https://github.com/orgs/moja-global/teams) - * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](mojaglobal.slack.com). + * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](mojaglobal.slack.com). * If you have other questions, please write to info@moja.global - + ## Contributors @@ -176,4 +176,3 @@ The following people are Maintainers of this repository **Reviewers** check proposed changes before they go to the Maintainers **Ambassadors** are available to provide training related to this repository **Coaches** are available to provide information to new contributors to this repository - From eea8855b9538e12a4e31af233fe11359c7922654 Mon Sep 17 00:00:00 2001 From: Tlazypanda Date: Thu, 11 Jun 2020 21:41:36 +0530 Subject: [PATCH 04/40] Add pull request template for flint Signed-off-by: Tlazypanda --- .github/PULL_REQUEST_TEMPLATE.md | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..65a4f44 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,40 @@ +# Pull Request Template + +## Description + +Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. + +Fixes # (issue) + +## Type of change + +Please delete options that are not relevant. + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] This change requires a documentation update + +## How Has This Been Tested? + +Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration + +## Additional Context (Please include any Screenshots/gifs if relevant) + +... + +## Checklist: + +- [ ] My code follows the style guidelines of this project +- [ ] I have performed a self-review of my own code +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation +- [ ] My changes generate no new warnings +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] New and existing unit tests pass locally with my changes +- [ ] Any dependent changes have been merged and published in downstream modules +- [ ] I have checked my code and corrected any misspellings +- [ ] I have tagged the reviewers in a comment below incase my pull request is ready for a review +- [ ] I have signed the commit message to agree to Developer Certificate of Origin (DCO) (to certify that you wrote or otherwise have the right to submit your contribution to the project.) by adding "--signoff" to my git commit command. + + From 5a749dcd9238086601e55dfa1c1988196208ddfb Mon Sep 17 00:00:00 2001 From: mal Date: Tue, 16 Jun 2020 22:03:09 +1000 Subject: [PATCH 05/40] moved to c++17 --- Source/CMakeLists.txt | 4 ++++ Source/moja.core/CMakeLists.txt | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index dc5e0a5..55d09d1 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -7,6 +7,10 @@ set_property( GLOBAL PROPERTY USE_FOLDERS ON) #turn on parallel builds add_compile_options($<$:/MP>) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + set(MOJA_VERSION_MAJOR "1") set(MOJA_VERSION_MINOR "0") set(MOJA_VERSION_PATCH "0") diff --git a/Source/moja.core/CMakeLists.txt b/Source/moja.core/CMakeLists.txt index 7c8749f..d024677 100644 --- a/Source/moja.core/CMakeLists.txt +++ b/Source/moja.core/CMakeLists.txt @@ -145,8 +145,6 @@ target_compile_definitions(${LIBNAME} FOLLY_NO_CONFIG ) -target_compile_features(${LIBNAME} PUBLIC cxx_std_14) - target_link_libraries(${LIBNAME} PUBLIC Boost::log Boost::log_setup Poco::Foundation Poco::JSON From e1cc2765622fee464379146572b1b57e193713c9 Mon Sep 17 00:00:00 2001 From: Jim Date: Thu, 18 Jun 2020 12:36:49 +1000 Subject: [PATCH 06/40] Fix for OutputerStreamFlux + was missing a handler for on onPostDisturbanceEvent + means disturbance events were not being recorded --- Source/moja.flint/include/moja/flint/outputerstreamflux.h | 1 + Source/moja.flint/src/outputerstreamflux.cpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Source/moja.flint/include/moja/flint/outputerstreamflux.h b/Source/moja.flint/include/moja/flint/outputerstreamflux.h index b3fe790..f3d891d 100644 --- a/Source/moja.flint/include/moja/flint/outputerstreamflux.h +++ b/Source/moja.flint/include/moja/flint/outputerstreamflux.h @@ -28,6 +28,7 @@ class FLINT_API OutputerStreamFlux : public ModuleBase { void onSystemShutdown() override; void onTimingPostInit() override; void onTimingEndStep() override; + void onPostDisturbanceEvent() override; protected: std::string _fileName; diff --git a/Source/moja.flint/src/outputerstreamflux.cpp b/Source/moja.flint/src/outputerstreamflux.cpp index 943c11b..9fdd81f 100644 --- a/Source/moja.flint/src/outputerstreamflux.cpp +++ b/Source/moja.flint/src/outputerstreamflux.cpp @@ -37,6 +37,7 @@ void OutputerStreamFlux::subscribe(NotificationCenter& notificationCenter) { notificationCenter.subscribe(signals::SystemShutdown, &OutputerStreamFlux::onSystemShutdown, *this); notificationCenter.subscribe(signals::TimingPostInit, &OutputerStreamFlux::onTimingPostInit, *this); notificationCenter.subscribe(signals::TimingEndStep, &OutputerStreamFlux::onTimingEndStep, *this); + notificationCenter.subscribe(signals::PostDisturbanceEvent, &OutputerStreamFlux::onPostDisturbanceEvent, *this); } // -------------------------------------------------------------------------------------------- @@ -141,5 +142,9 @@ void OutputerStreamFlux::onTimingPostInit() { outputInit(_output); } void OutputerStreamFlux::onTimingEndStep() { outputEndStep(_output); } +// -------------------------------------------------------------------------------------------- + +void OutputerStreamFlux::onPostDisturbanceEvent() { outputEndStep(_output); } + } // namespace flint } // namespace moja From 17052225e279ced9ff48182e17406d6db1c0009d Mon Sep 17 00:00:00 2001 From: mal Date: Fri, 19 Jun 2020 15:46:17 +1000 Subject: [PATCH 07/40] pre-allocated tuple and persistable vectors in the recordaccumulator --- .../moja.flint/include/moja/flint/recordaccumulator.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index 4067492..adc0141 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -303,6 +303,7 @@ class RecordAccumulatorMap2 { std::vector getPersistableCollection() const { std::vector persistables; + persistables.reserve(_records.size()); for (const auto& rec : _records) { persistables.emplace_back(TRecordConv::asPersistable(rec.first, rec.second)); } @@ -312,15 +313,17 @@ class RecordAccumulatorMap2 { std::vector getPersistableCollectionRange(typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) const { std::vector persistables; + persistables.reserve(std::min(_records.size(), chunkSize)); size_t chunkPosition = 0; for (; (rangeStart != _records.end() && chunkPosition++ < chunkSize); ++rangeStart) { - persistables.push_back(TRecordConv::asPersistable((*rangeStart).first, (*rangeStart).second)); + persistables.emplace_back(TRecordConv::asPersistable((*rangeStart).first, (*rangeStart).second)); } return persistables; } std::vector getTupleCollection() { - std::vector tuples(_records.size()); + std::vector tuples; + tuples.reserve(_records.size()); for (const auto& rec : _records) { tuples.emplace_back(TRecordConv::asTuple(rec.first, rec.second)); } @@ -329,9 +332,10 @@ class RecordAccumulatorMap2 { std::vector getTupleCollectionRange(typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) { std::vector tuples; + tuples.reserve(std::min(_records.size(), chunkSize)); size_t chunkPosition = 0; for (; (rangeStart != _records.end() && chunkPosition++ < chunkSize); ++rangeStart) { - tuples.push_back(TRecordConv::asTuple((*rangeStart).first, (*rangeStart).second)); + tuples.emplace_back(TRecordConv::asTuple((*rangeStart).first, (*rangeStart).second)); } return tuples; } From 264917a28d15274168c71072da434eefa951f199 Mon Sep 17 00:00:00 2001 From: mal Date: Mon, 22 Jun 2020 09:02:37 +1000 Subject: [PATCH 08/40] Added a tuple collection --- .../include/moja/flint/recordaccumulator.h | 102 +++++++++++++++--- 1 file changed, 88 insertions(+), 14 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index adc0141..115c053 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -8,7 +8,10 @@ #include +#include +#include #include +#include namespace moja { namespace flint { @@ -175,23 +178,13 @@ class RecordAccumulator2 { std::vector getPersistableCollection() const { std::vector persistables; + persistables.reserve(_records.size()); for (const auto& record : _records) { - persistables.push_back(record.asPersistable()); + persistables.emplace_back(record.asPersistable()); } return persistables; } - // std::vector getPersistableCollection(size_t startIndex, size_t chunkSize) const { - // std::vector persistables; - // if (startIndex > _records.size()) - // return persistables; - // size_t chunkPosition = 0; - // for (auto it = _records.begin() + startIndex; it != _records.end() && chunkPosition++ < chunkSize; ++it) { - // persistables.push_back(record->asPersistable()); - // } - // return persistables; - //} - void clear() { _recordsIdx.clear(); _records.clear(); @@ -267,6 +260,81 @@ class RecordAccumulatorMap { rec_accu_map _records; }; +template +class list_of_tuples { + using map_type = tlx::btree_map; + const map_type& accumulator_map_; + + public: + template + class tuples_iterator { + using iterator_type = std::conditional_t; + iterator_type iterator_current_; + iterator_type iterator_end_; + TTuple record_{}; + + public: + explicit tuples_iterator(iterator_type iterator_begin, iterator_type iterator_end) + : iterator_current_{iterator_begin}, iterator_end_(iterator_end) { + if (iterator_current_ != iterator_end_) { + const auto& record = *iterator_current_; + record_ = TRecordConv::asTuple(record.first, record.second); + } + } + using difference_type = std::ptrdiff_t; + using value_type = TTuple; + using pointer = std::conditional_t; + using reference = std::conditional_t; + using iterator_category = std::forward_iterator_tag; + + reference operator*() const { return record_; } + pointer operator->() const { return &record_; } + + auto& operator++() { + ++iterator_current_; + if (iterator_current_ != iterator_end_) { + const auto& record = *iterator_current_; + record_ = TRecordConv::asTuple(record.first, record.second); + } + return *this; + } + + auto operator++(int) { + auto result = *this; + ++*this; + return result; + } + // Support comparison between iterator and const_iterator types + template + bool operator==(const tuples_iterator& rhs) const { + return iterator_current_ == rhs.iterator_current_; + } + + template + bool operator!=(const tuples_iterator& rhs) const { + return iterator_current_ != rhs.iterator_current_; + } + + // Support implicit conversion of iterator to const_iterator + // (but not vice versa) + operator tuples_iterator() const { return tuples_iterator(iterator_current_, iterator_end_); } + }; + + using const_iterator = tuples_iterator; + using iterator = tuples_iterator; + + list_of_tuples(const map_type& map) : accumulator_map_{map} {} + + // Begin and end member functions + iterator begin() { return iterator{std::begin(accumulator_map_), std::end(accumulator_map_)}; } + iterator end() { return iterator{std::end(accumulator_map_), std::end(accumulator_map_)}; } + const_iterator begin() const { return const_iterator{std::cbegin(accumulator_map_), std::cend(accumulator_map_)}; } + const_iterator end() const { return const_iterator{std::cend(accumulator_map_), std::cend(accumulator_map_)}; } + + // Other member operations + const auto& front() const { return accumulator_map_.front().asPersistable(); } + size_t size() const { return accumulator_map_.size(); } +}; template class RecordAccumulatorMap2 { @@ -321,6 +389,11 @@ class RecordAccumulatorMap2 { return persistables; } + const auto tuples() const { + return list_of_tuples{_records}; + } + + [[deprecated("Replaced with tuples() method")]] std::vector getTupleCollection() { std::vector tuples; tuples.reserve(_records.size()); @@ -330,7 +403,9 @@ class RecordAccumulatorMap2 { return tuples; } - std::vector getTupleCollectionRange(typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) { + [[deprecated("Replaced with tuples() method")]] + std::vector getTupleCollectionRange( + typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) { std::vector tuples; tuples.reserve(std::min(_records.size(), chunkSize)); size_t chunkPosition = 0; @@ -349,7 +424,6 @@ class RecordAccumulatorMap2 { rec_accu_map _records; }; - } // namespace flint } // namespace moja From 8c959896b8036779fd68ac3621b4c9f487983016 Mon Sep 17 00:00:00 2001 From: mal Date: Tue, 23 Jun 2020 16:00:49 +1000 Subject: [PATCH 09/40] fixed minmax windows macro issue --- Source/moja.flint/include/moja/flint/recordaccumulator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index 115c053..bd1a37d 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -381,7 +381,7 @@ class RecordAccumulatorMap2 { std::vector getPersistableCollectionRange(typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) const { std::vector persistables; - persistables.reserve(std::min(_records.size(), chunkSize)); + persistables.reserve((std::min)(_records.size(), chunkSize)); size_t chunkPosition = 0; for (; (rangeStart != _records.end() && chunkPosition++ < chunkSize); ++rangeStart) { persistables.emplace_back(TRecordConv::asPersistable((*rangeStart).first, (*rangeStart).second)); @@ -407,7 +407,7 @@ class RecordAccumulatorMap2 { std::vector getTupleCollectionRange( typename rec_accu_map::const_iterator& rangeStart, size_t chunkSize) { std::vector tuples; - tuples.reserve(std::min(_records.size(), chunkSize)); + tuples.reserve((std::min)(_records.size(), chunkSize)); size_t chunkPosition = 0; for (; (rangeStart != _records.end() && chunkPosition++ < chunkSize); ++rangeStart) { tuples.emplace_back(TRecordConv::asTuple((*rangeStart).first, (*rangeStart).second)); From ab18b2d7764d83f1c4959fcd34b58d550266bfdc Mon Sep 17 00:00:00 2001 From: mfellows Date: Wed, 24 Jun 2020 10:34:24 -0700 Subject: [PATCH 10/40] Added out of memory handler to give the user more information about the cause of hard crashes in limited cases. --- Source/moja.cli/src/moja.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Source/moja.cli/src/moja.cpp b/Source/moja.cli/src/moja.cpp index 4f50b22..e148291 100644 --- a/Source/moja.cli/src/moja.cpp +++ b/Source/moja.cli/src/moja.cpp @@ -21,6 +21,7 @@ #include #include #include +#include static constexpr const char* CLI_VERSION_STRING = "flint cli version 1.0.0"; @@ -41,7 +42,13 @@ bool checkFilePath(const std::string& filePath) { return true; } +void handleOutOfMemory() { + MOJA_LOG_FATAL << "Failed to allocate memory"; + std::abort(); +} + int main(int argc, char* argv[]) { + std::set_new_handler(handleOutOfMemory); opt::options_description general_opt("General options"); general_opt.add_options() From 72fafac147033b45658956d7a0ff513eeb12eb8e Mon Sep 17 00:00:00 2001 From: Tlazypanda <33183263+Tlazypanda@users.noreply.github.com> Date: Thu, 2 Jul 2020 13:37:17 +0530 Subject: [PATCH 11/40] add welcome bot config Signed-off-by: Tlazypanda <33183263+Tlazypanda@users.noreply.github.com> --- .github/config.yml | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/config.yml diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 0000000..a0b2e87 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,49 @@ +# Configuration for welcome - https://github.com/behaviorbot/welcome + +# Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome + +# Comment to be posted to on first time issues +newIssueWelcomeComment: | + Hello there!πŸ‘‹ Welcome to moja global!πŸ’– + + Thank you and congratulations πŸŽ‰ for opening your very first issue in this project. + Moja global fosters an open and welcoming environment for al our contributors.🌸 Please adhere to our [Code Of Conduct](https://github.com/moja-global/About_moja_global/blob/master/CODE_OF_CONDUCT.md). + + Incase you want to claim this issue, please comment down below! + Please checkout if any of the already existing issue template fits for your isssue or you may open a blank issue as well ✨ + We will try to get back to you as soon as we can.πŸ‘€ + + Feel free to join us on [moja global Private Slack](https://mojaglobal.slack.com/) by dropping an email [here](mailto:info@moja.global).πŸ‘©β€πŸ’» We would love to hear your interesting ideas and engage in discussions.πŸ’– + Moja global is delighted to have you here :) + +# Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome + +# Comment to be posted to on PRs from first time contributors in your repository +newPRWelcomeComment: | + Hello there!πŸ‘‹ Welcome to moja global!πŸ’– + Thank you and congrats πŸŽ‰ for opening your first PR on this project.✨ + We will review it soon! Till then you can checkout the `README.md` for more details on it. + + Moja global fosters an open and welcoming environment for al our contributors.🌸 Please adhere to our [Code Of Conduct](https://github.com/moja-global/About_moja_global/blob/master/CODE_OF_CONDUCT.md). + + Feel free to join us on [moja global Private Slack](https://mojaglobal.slack.com/) by dropping an email [here](mailto:info@moja.global).πŸ‘©β€πŸ’» We would love to hear your interesting ideas and engage in discussions.πŸ’– + Moja global is delighted to have you here :) + +# Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: | + Congratulations on merging your first contribution to moja global!✨ + Your code is officially a part of moja global now!! πŸŽ‰ Please feel free to add yourself as a contributor by following these steps [here](https://github.com/moja-global/About_moja_global/blob/master/CONTRIBUTING.md#how-to-get-credit-for-your-contribution).πŸ™Œ + + Feel free to join us on [moja global Private Slack](https://mojaglobal.slack.com/) by dropping an email [here](mailto:info@moja.global).πŸ‘©β€πŸ’» We would love to hear your interesting ideas and engage in discussions.πŸ’– + +
+ Now that you've completed this, you can help someone else take their first step! +

Help a newcomer with their first pull request by providing feedback! This is their first time too, So be as encouraging as possible!πŸ˜„

+

Help them setup this project and resolve their queries! 🌸

+

Create a welcoming or beginner friendly issue for someone else! πŸ˜„ πŸŽ‰

+
+ + +# It is recommended to include as many gifs and emojis as possible! From dda45fe0bed12d7ec5af2feb7a779ebc7d769e14 Mon Sep 17 00:00:00 2001 From: mfellows Date: Tue, 10 Nov 2020 14:51:47 -0800 Subject: [PATCH 12/40] Added support for outputting properties of DynamicObject-style variables. --- .../src/writevariablegeotiff.cpp | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp index e9c7e97..7885df3 100644 --- a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp +++ b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp @@ -589,18 +589,18 @@ T WriteVariableGeotiff::DataSettingsT::applyValueAdjustment( template void WriteVariableGeotiff::DataSettingsT::setLUValue( std::shared_ptr spatialLocationInfo, int timestep) { - if ((timestep - 1) % _outputInterval != 0) { - return; - } - - initData(spatialLocationInfo, timestep); - if (_variable != nullptr) { - setLUVariableValue(spatialLocationInfo, timestep); - } else if (!_pool.empty()) { - setLUPoolValue(spatialLocationInfo, timestep); - } else if (!_flux.empty()) { - setLUFluxValue(spatialLocationInfo, timestep); - } + if ((timestep - 1) % _outputInterval != 0) { + return; + } + + initData(spatialLocationInfo, timestep); + if (_variable != nullptr) { + setLUVariableValue(spatialLocationInfo, timestep); + } else if (!_pool.empty()) { + setLUPoolValue(spatialLocationInfo, timestep); + } else if (!_flux.empty()) { + setLUFluxValue(spatialLocationInfo, timestep); + } } // -------------------------------------------------------------------------------------------- @@ -608,27 +608,35 @@ void WriteVariableGeotiff::DataSettingsT::setLUValue( template void WriteVariableGeotiff::DataSettingsT::setLUVariableValue( std::shared_ptr spatialLocationInfo, int timestep) { - if (_propertyName != "") { - auto flintDataVariable = _variable->value().extract>(); - if (!_isArray) { - auto variablePropertyValue = flintDataVariable->getProperty(_propertyName); - _data[timestep][spatialLocationInfo->_cellIdx] = - applyValueAdjustment(spatialLocationInfo, timestep, variablePropertyValue.convert()); - } - } else { - auto variableValue = _variable->value(); - if (_isArray) { - auto val = variableValue.extract>>()[_arrayIndex]; - if (val.is_initialized()) { - _data[timestep][spatialLocationInfo->_cellIdx] = - applyValueAdjustment(spatialLocationInfo, timestep, val.value()); - } - } else { - if (!variableValue.isEmpty()) + if (_propertyName != "") { + auto variableValue = _variable->value(); + if (variableValue.isStruct()) { + const auto& structVal = variableValue.extract(); _data[timestep][spatialLocationInfo->_cellIdx] = - applyValueAdjustment(spatialLocationInfo, timestep, variableValue.convert()); - } - } + applyValueAdjustment(spatialLocationInfo, timestep, structVal[_propertyName].convert()); + } else { + auto flintDataVariable = _variable->value().extract>(); + if (!_isArray) { + auto variablePropertyValue = flintDataVariable->getProperty(_propertyName); + _data[timestep][spatialLocationInfo->_cellIdx] = + applyValueAdjustment(spatialLocationInfo, timestep, variablePropertyValue.convert()); + } + } + } else { + auto variableValue = _variable->value(); + if (_isArray) { + auto val = variableValue.extract>>()[_arrayIndex]; + if (val.is_initialized()) { + _data[timestep][spatialLocationInfo->_cellIdx] = + applyValueAdjustment(spatialLocationInfo, timestep, val.value()); + } + } else { + if (!variableValue.isEmpty()) { + _data[timestep][spatialLocationInfo->_cellIdx] = + applyValueAdjustment(spatialLocationInfo, timestep, variableValue.convert()); + } + } + } } // -------------------------------------------------------------------------------------------- From b155a3f3c11c3cb50173f223ba99e2067ab2864c Mon Sep 17 00:00:00 2001 From: Sneha Mishra Date: Thu, 3 Dec 2020 23:07:26 +0530 Subject: [PATCH 13/40] refactor readme with readthedocs Signed-off-by: Sneha Mishra --- README.md | 134 ++++-------------------------------------------------- 1 file changed, 10 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index a9caee3..f75d9e4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # FLINT Open-source Library -[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors) +[![All Contributors](https://img.shields.io/badge/all_contributors-5-orange.svg?style=flat-square)](#contributors) [![License: MPL 2.0](https://img.shields.io/badge/License-MPL%202.0-brightgreen.svg)](https://opensource.org/licenses/MPL-2.0) [![Code Of Conduct](https://img.shields.io/badge/code--of--conduct-Moja%20Global-blue)](https://github.com/moja-global/About_moja_global/blob/master/CODE_OF_CONDUCT.md) [![Contributions](https://img.shields.io/badge/newcomer-friendly-red)](https://github.com/moja-global/About_moja_global/blob/master/CONTRIBUTING.md) [![Chat on Slack](https://img.shields.io/badge/chat-slack-blueviolet)](https://mojaglobal.slack.com/) [![Twitter Handle](https://img.shields.io/badge/twitter-Moja%20Global-darkblue)](https://twitter.com/mojaglobal?lang=en) ## What is `FLINT`? @@ -21,140 +21,27 @@ The FLINT is using the lessons learned from first generation tools, to build a n + development managed using a true open source approach under [moja global](http://moja.global), which will allow users (countries, companies and organizations) to direct strategy and control the budget. + software that allows data processing on local desktops or cloud-based systems +## Installation Docs -## How to use FLINT? +Please checkout the [moja global developer docs](https://docs.moja.global.com) for complete instructions on how to setup the repository. You may also refer this documentation for setting up FLINT.example, GCBM or just get an idea of the moja global workflow! You may also suggest an improvement in the current docs by creating an issue [here](hhttps://github.com/moja-global/GSoD.moja_global_docs). -### Development Environment How-To for Windows - -These instructions are for building the FLINT on Windows using Visual Studio 2017, or Visual Studio 2019. - -#### Required Installs - -##### CMake - -- download [cmake-3.15.2-win64-x64.msi](https://github.com/Kitware/CMake/releases/download/v3.15.2/cmake-3.15.2-win64-x64.msi) - -#### Using vcpkg to install required libraries - -A fork of a *Vcpkg* repository has been created for the FLINT required libraries. To build these libraries you can use the following process: - -+ Clone the Vcpkg repository: https://github.com/moja-global/vcpkg - -+ Start a command shell in the Vcpkg repository folder and use the following commands: - - ```powershell - # bootstrap - bootstrap-vcpkg.bat - - # install packages - vcpkg.exe install boost-test:x64-windows boost-program-options:x64-windows boost-log:x64-windows turtle:x64-windows zipper:x64-windows poco:x64-windows libpq:x64-windows gdal:x64-windows sqlite3:x64-windows boost-ublas:x64-windows fmt:x64-windows - ``` - -+ Once this has completed, start a command shell in you FLINT repository folder. Now use the following commands to create the Visual Studio solution: - - ```powershell - # Create a build folder under the Source folder - cd Source - mkdir build - cd build - - # now create the Visual Studio Solution (2019) - cmake -G "Visual Studio 16 2019" -DCMAKE_INSTALL_PREFIX=C:/Development/Software/moja -DVCPKG_TARGET_TRIPLET=x64-windows -DENABLE_TESTS=OFF -DENABLE_MOJA.MODULES.ZIPPER=OFF -DCMAKE_TOOLCHAIN_FILE=c:\Development\moja-global\vcpkg\scripts\buildsystems\vcpkg.cmake .. - - # OR Visual Studio Solution (2017) - cmake -G "Visual Studio 15 2017" -DCMAKE_INSTALL_PREFIX=C:/Development/Software/moja -DVCPKG_TARGET_TRIPLET=x64-windows -DENABLE_TESTS=OFF -DENABLE_MOJA.MODULES.ZIPPER=OFF -DCMAKE_TOOLCHAIN_FILE=c:\Development\moja-global\vcpkg\scripts\buildsystems\vcpkg.cmake .. - ``` - -#### Install Moja Libraries - -It is possible to use the Visual Studio moja solution to install built versions of the Moja libraries. To do this you need to set the CMAKE variable '***CMAKE_INSTALL_PREFIX***' to your install path (i.e. "*C:/Development/Software/moja*"). - -#### Make edits to the Visual Studio Solution using CMake - -1. Launch the CMake GUI -2. In the '*Where to build the binaries*' field click β€œBrowse Build…” and select the folder you created above (i.e. `C:\Development\moja-global\FLINT\Source\build`)`. The '*Where is the source code:*' field should update, if not, set it correctly. -4. You should be able to edit any CMake setting now (i.e. ENABLE flags like `ENABLE_TESTS`), then click β€œ***Configure***” – assuming all libraries and required software has been installed you should have no errors. Now click ***"Generate"*** and the Solution with adjustments should be ready to load into Visual Studio. - -#### Other Useful Tools - -##### SQLIte Studio - -a simple windows SQLite database manager (http://sqlitestudio.pl/) -[sqlitestudio-3.1.0.zip](http://sqlitestudio.pl/files/sqlitestudio3/complete/win32/sqlitestudio-3.1.0.zip) - -##### TortoiseGit - -[TortoiseGit](https://code.google.com/p/tortoisegit/wiki/Download) - -### Docker for Ubuntu 18:04 - -Containers are a simple way to build FLINT and all required dependencies. Examples of how this can be done are provided for Ubuntu 18.04. See the [Examples docker directory.](https://github.com/moja-global/flint/tree/master/Examples/docker) - -#### Building the containers - -The build has been split into two Dockerfiles, the first to get and build required libraries. The second to get and build the moja FLINT libraries and CLI program. - -```bash -# working from the examples folder "flint/tree/master/Examples/docker" - -# build the base -docker build -f Dockerfile.base.ubuntu.18.04 --build-arg NUM_CPU=4 -t moja/baseimage:ubuntu-18.04 . - -# build the flint container -docker build -f Dockerfile.flint.ubuntu.18.04 --build-arg NUM_CPU=4 --build-arg FLINT_BRANCH=master -t moja/flint:ubuntu-18.04 . - -docker build -f Dockerfile.flint.ubuntu.18.04 --build-arg NUM_CPU=4 --build-arg GITHUB_AT=XXXX --build-arg FLINT_BRANCH=master -t moja/flint:ubuntu-18.04 . -``` - -How to use the final container depends on the task. However, the following command will bash into the flint container and allow you to use the CLI program. - -```bash -# run bash on the flint container -docker run --rm -ti moja/flint:ubuntu-18.04 bash -``` - -Once in, you should be able to run the CLI program `moja.cli` - -``` -# run CLI -moja.cli --help -``` - -That should respond: - -``` -Allowed options: - -General options: - -h [ --help ] produce a help message - --help-section arg produce a help message for a named section - -v [ --version ] output the version number - -Commandline only options: - --logging_config arg path to Moja logging config file - --config_file arg path to Moja run config file - --provider_file arg path to Moja data provider config file - -Configuration file options: - --config arg path to Moja project config files - --config_provider arg path to Moja project config files for data providers -``` +## Installation Videos +We also have a set of installation videos to help you out with the installation. If you prefer video installation procedure as opposed to textual documentation, this will be a perfect starter for you! ## How to Get Involved? -moja global welcomes a wide range of contributions as explained in [Contributing document](https://github.com/moja-global/About-moja-global/blob/master/CONTRIBUTING.md) and in the [About moja-global Wiki](https://github.com/moja-global/.github/wiki). - +moja global welcomes a wide range of contributions as explained in [Contributing document](https://docs.moja.global/en/latest/contributing/index.html). ## FAQ and Other Questions -* You can find FAQs on the [Wiki](https://github.com/moja.global/.github/wiki). +* You can find FAQs on the [FAQs section of our docs](https://docs.moja.global/en/latest/faq.html). * If you have a question about the code, submit [user feedback](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Provide-User-Feedback.md) in the relevant repository -* If you have a general question about a project or repository or moja global, [join moja global](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Join-moja-global.md) and +* If you have a general question about a project or repository or moja global, [join moja global](https://docs.moja.global/en/latest/contact.html) and * [submit a discussion](https://help.github.com/en/articles/about-team-discussions) to the project, repository or moja global [team](https://github.com/orgs/moja-global/teams) - * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](mojaglobal.slack.com). + * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](mojaglobal.slack.com). * If you have other questions, please write to info@moja.global - + ## Contributors @@ -176,4 +63,3 @@ The following people are Maintainers of this repository **Reviewers** check proposed changes before they go to the Maintainers **Ambassadors** are available to provide training related to this repository **Coaches** are available to provide information to new contributors to this repository - From 752aac53a6a76bd46cd0fc4bb1ebbe0555080e2f Mon Sep 17 00:00:00 2001 From: mfellows Date: Mon, 7 Dec 2020 09:18:31 -0800 Subject: [PATCH 14/40] Fixed error in cmake setup in base docker file. --- Docker/Dockerfile.base.ubuntu.18.04 | 1 - 1 file changed, 1 deletion(-) diff --git a/Docker/Dockerfile.base.ubuntu.18.04 b/Docker/Dockerfile.base.ubuntu.18.04 index fb74bc4..58eb49e 100644 --- a/Docker/Dockerfile.base.ubuntu.18.04 +++ b/Docker/Dockerfile.base.ubuntu.18.04 @@ -50,7 +50,6 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && tar xzf cmake-${CMAKE_VERSION}.tar.gz \ && cd cmake-${CMAKE_VERSION} \ && ./bootstrap --system-curl --parallel=$NUM_CPU \ - && .$NUM_CPU \ && make --quiet install \ && make clean \ && cd .. From e450ed55b38fdb6e0ab75c6c9fd99a967361267e Mon Sep 17 00:00:00 2001 From: Shubham Karande Date: Tue, 8 Dec 2020 22:19:10 +0530 Subject: [PATCH 15/40] Fix Dockerfiles to build FLINT images Signed-off-by: Shubham Karande --- Docker/Dockerfile.base.ubuntu.18.04 | 1 - Docker/Dockerfile.flint.ubuntu.18.04 | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Docker/Dockerfile.base.ubuntu.18.04 b/Docker/Dockerfile.base.ubuntu.18.04 index fb74bc4..58eb49e 100644 --- a/Docker/Dockerfile.base.ubuntu.18.04 +++ b/Docker/Dockerfile.base.ubuntu.18.04 @@ -50,7 +50,6 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm && tar xzf cmake-${CMAKE_VERSION}.tar.gz \ && cd cmake-${CMAKE_VERSION} \ && ./bootstrap --system-curl --parallel=$NUM_CPU \ - && .$NUM_CPU \ && make --quiet install \ && make clean \ && cd .. diff --git a/Docker/Dockerfile.flint.ubuntu.18.04 b/Docker/Dockerfile.flint.ubuntu.18.04 index ea146c6..0a7bd66 100644 --- a/Docker/Dockerfile.flint.ubuntu.18.04 +++ b/Docker/Dockerfile.flint.ubuntu.18.04 @@ -7,8 +7,7 @@ # # ================================================================================================================== -#FROM moja/baseimage:ubuntu-18.04 -FROM moja/baseimage:mg +FROM moja/baseimage:ubuntu-18.04 LABEL maintainer="info@moja.global" From f85c141fe5320f89d89c8fccc28e1a4881e4947a Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 9 Dec 2020 11:37:28 +1100 Subject: [PATCH 16/40] Added shrink_to_fit RecordAccumulator2 + allow memory to be released as required --- Source/moja.flint/include/moja/flint/recordaccumulator.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index bd1a37d..c11dc89 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -190,6 +190,11 @@ class RecordAccumulator2 { _records.clear(); } + void shrink_to_fit() { + _recordsIdx.clear(); + _records.shrink_to_fit(); + } + rec_accu_size_type size() const { return _records.size(); } const rec_accu_vec& records() const { return _records; } From db7550478d4505d2e620a2581fefc913e6787294 Mon Sep 17 00:00:00 2001 From: mal Date: Tue, 15 Dec 2020 12:37:32 +1100 Subject: [PATCH 17/40] Added methods to assist large data aggregations --- .../include/moja/flint/recordaccumulator.h | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index c11dc89..f2f6a62 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -12,6 +12,7 @@ #include #include #include +#include namespace moja { namespace flint { @@ -185,6 +186,17 @@ class RecordAccumulator2 { return persistables; } + std::vector getPersistableCollectionRange(typename rec_accu_vec::const_iterator& rangeStart, + size_t chunkSize) const { + std::vector persistables; + persistables.reserve((std::min)(_records.size(), chunkSize)); + size_t chunkPosition = 0; + for (; (rangeStart != _records.end() && chunkPosition++ < chunkSize); ++rangeStart) { + persistables.emplace_back((*rangeStart).asPersistable()); + } + return persistables; + } + void clear() { _recordsIdx.clear(); _records.clear(); @@ -205,6 +217,173 @@ class RecordAccumulator2 { rec_accu_vec _records; }; +template +class list_of_persistables { + using vec_type = std::vector; + const vec_type& accumulator_vec_; + + public: + template + class persistables_iterator { + using iterator_type = std::conditional_t; + iterator_type iterator_current_; + iterator_type iterator_end_; + TPersistable record_{}; + + public: + explicit persistables_iterator(iterator_type iterator_begin, iterator_type iterator_end) + : iterator_current_{iterator_begin}, iterator_end_(iterator_end) { + if (iterator_current_ != iterator_end_) { + const auto& record = *iterator_current_; + record_ = record.asPersistable(); + } + } + using difference_type = std::ptrdiff_t; + using value_type = TPersistable; + using pointer = std::conditional_t; + using reference = std::conditional_t; + using iterator_category = std::forward_iterator_tag; + + reference operator*() const { return record_; } + pointer operator->() const { return &record_; } + + auto& operator++() { + ++iterator_current_; + if (iterator_current_ != iterator_end_) { + const auto& record = *iterator_current_; + record_ = record.asPersistable(); + } + return *this; + } + + auto operator++(int) { + auto result = *this; + ++*this; + return result; + } + // Support comparison between iterator and const_iterator types + template + bool operator==(const persistables_iterator& rhs) const { + return iterator_current_ == rhs.iterator_current_; + } + + template + bool operator!=(const persistables_iterator& rhs) const { + return iterator_current_ != rhs.iterator_current_; + } + + // Support implicit conversion of iterator to const_iterator + // (but not vice versa) + operator persistables_iterator() const { + return persistables_iterator(iterator_current_, iterator_end_); + } + }; + + using const_iterator = persistables_iterator; + using iterator = persistables_iterator; + + list_of_persistables(const vec_type& vec) : accumulator_vec_{vec} {} + + // Begin and end member functions + iterator begin() { return iterator{std::begin(accumulator_vec_), std::end(accumulator_vec_)}; } + iterator end() { return iterator{std::end(accumulator_vec_), std::end(accumulator_vec_)}; } + const_iterator begin() const { return const_iterator{std::cbegin(accumulator_vec_), std::cend(accumulator_vec_)}; } + const_iterator end() const { return const_iterator{std::cend(accumulator_vec_), std::cend(accumulator_vec_)}; } + + // Other member operations + const auto& front() const { return accumulator_vec_.front().asPersistable(); } + [[nodiscard]] bool empty() const noexcept { return accumulator_vec_.empty(); } + + typename vec_type::size_type size() const { return accumulator_vec_.size(); } +}; + +template +class RecordAccumulator3 { + struct RecordComparer { + bool operator()(const TRecord* lhs, const TRecord* rhs) const { return lhs->operator==(*rhs); } + }; + struct RecordHasher { + size_t operator()(const TRecord* record) const { return record->hash(); } + }; + size_t compute_size() { + if (records_.capacity() == 0) { + return (std::max)(64 / sizeof(TRecord), size_t(1)); + } + if (records_.capacity() > 4096 * 32 / sizeof(TRecord)) { + return records_.capacity() * 2; + } + return (records_.capacity() * 3 + 1) / 2; + } + + public: + typedef std::unordered_set rec_accu_set; + typedef std::vector rec_accu_vec; + typedef typename rec_accu_vec::size_type rec_accu_size_type; + + const TRecord* insert(Int64 id, TRecord record) { + if (records_.size() == records_.capacity()) { + records_idx_.clear(); + records_.reserve(compute_size()); + for (auto& rec : records_) { + records_idx_.insert(&rec); + } + } + // ID has been assigned by user, assume that we can run with this + next_id_ = id + 1; // can't guarantee that this will be called in 'id increasing' order but a good guess perhaps + record.setId(id); + records_.push_back(record); + auto& new_record = records_.back(); + records_idx_.insert(&new_record); + return &new_record; + } + + const TRecord* accumulate(TRecord record) { return accumulate(record, next_id_); } + + const TRecord* accumulate(TRecord record, Int64 requestedId) { + auto it = records_idx_.find(&record); + if (it != records_idx_.end()) { + // Found an existing ID for the key. + auto existing = *it; + existing->merge(record); + return existing; + } + return insert(requestedId, record); + } + + const TRecord* search(const TRecord& record) { + auto it = records_idx_.find(&record); + if (it != records_idx_.end()) { + // Found an existing ID for the key. + auto existing = *it; + return existing; + } + return nullptr; + } + + const rec_accu_vec& get_records() const { return records_; } + + void clear() { + records_idx_.clear(); + records_.clear(); + } + + void shrink_to_fit() { + records_idx_.clear(); + records_.shrink_to_fit(); + } + + rec_accu_size_type size() const { return records_.size(); } + + const rec_accu_vec& records() const { return records_; } + + auto persistables() const { return list_of_persistables{records_}; } + + private: + Int64 next_id_ = 1; + rec_accu_set records_idx_; + rec_accu_vec records_; +}; + template class RecordAccumulatorMap { public: From 9c2d99acace79b726455c2736b8098d1cbfb067b Mon Sep 17 00:00:00 2001 From: mfellows Date: Tue, 15 Dec 2020 09:44:22 -0800 Subject: [PATCH 18/40] Fixed occasional error reading layer metadata when using relative paths. --- Source/moja.datarepository/src/rasterreader.cpp | 5 +++-- Source/moja.modules.gdal/src/rasterreadergdal.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/moja.datarepository/src/rasterreader.cpp b/Source/moja.datarepository/src/rasterreader.cpp index 14fa2bd..fbfb3f9 100644 --- a/Source/moja.datarepository/src/rasterreader.cpp +++ b/Source/moja.datarepository/src/rasterreader.cpp @@ -26,13 +26,14 @@ namespace datarepository { FlintMetaDataRasterReader::FlintMetaDataRasterReader(const std::string& path, const std::string& prefix, const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { - _metaPath = (boost::format("%1%%2%%3%.json") % path % Poco::Path::separator() % prefix).str(); + auto filePath = Poco::Path(path); + auto abs = filePath.absolute().toString(); + _metaPath = (boost::format("%1%%2%%3%.json") % abs % Poco::Path::separator() % prefix).str(); } DynamicObject FlintMetaDataRasterReader::readMetaData() const { if (file_exists(_metaPath)) { Poco::JSON::Parser jsonMetadataParser; - // Poco::Dynamic::Var parsedMetadata; std::ifstream metadataFile(_metaPath, std::ifstream::in); jsonMetadataParser.parse(metadataFile); diff --git a/Source/moja.modules.gdal/src/rasterreadergdal.cpp b/Source/moja.modules.gdal/src/rasterreadergdal.cpp index f64ba63..aeee1dc 100644 --- a/Source/moja.modules.gdal/src/rasterreadergdal.cpp +++ b/Source/moja.modules.gdal/src/rasterreadergdal.cpp @@ -45,7 +45,7 @@ MetaDataRasterReaderGDAL::MetaDataRasterReaderGDAL(const std::string& path, cons auto filePath = Poco::Path(path); auto parent = filePath.parent().toString(); auto abs = filePath.parent().absolute().toString(); - _path = (boost::format("%1%%2%.json") % filePath.parent().absolute().toString() % filePath.getBaseName()).str(); + _path = (boost::format("%1%%2%.json") % abs % filePath.getBaseName()).str(); _metaDataRequired = true; if (settings.contains("metadata_required")) { _metaDataRequired = settings["metadata_required"].extract(); From 27503201d3b5bab416aa95c3fd2e07a72f60d449 Mon Sep 17 00:00:00 2001 From: mal Date: Mon, 29 Mar 2021 14:37:04 +1100 Subject: [PATCH 19/40] Fixed issue with column data when nulls are present. Column types change when a null is present in a row so the types cannot be cached for the whole dataset. --- Source/moja.datarepository/src/providerrelationalsqlite.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index 958c394..ba15231 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -127,11 +127,9 @@ class ProviderRelationalSQLite::impl { std::vector result; const int nCols = stmt.n_cols(); - std::vector columnTypes(nCols); std::vector columnNames(nCols); for (int i = 0; i < nCols; i++) { columnNames[i] = stmt.column_name(i); - columnTypes[i] = stmt.column_type(i); } for (; resultCode == SQLITE_ROW || resultCode == SQLITE_BUSY || resultCode == SQLITE_LOCKED || @@ -140,7 +138,7 @@ class ProviderRelationalSQLite::impl { DynamicObject row; for (int i = 0; i < nCols; i++) { - switch (columnTypes[i]) { + switch (stmt.column_type(i)) { case SQLiteStatement::ColumnType::DOUBLE: { row.insert(columnNames[i], stmt.column_double(i)); } break; From 510ffdd246fca40dcc55365266ae111b11b16a1f Mon Sep 17 00:00:00 2001 From: Max Fellows Date: Tue, 6 Apr 2021 15:57:29 -0700 Subject: [PATCH 20/40] Dockerfile cleanup, added BUILD_TYPE arg to switch between debug and release. --- Docker/Dockerfile.flint.ubuntu.18.04 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Docker/Dockerfile.flint.ubuntu.18.04 b/Docker/Dockerfile.flint.ubuntu.18.04 index 0a7bd66..7d48014 100644 --- a/Docker/Dockerfile.flint.ubuntu.18.04 +++ b/Docker/Dockerfile.flint.ubuntu.18.04 @@ -3,7 +3,7 @@ # Docker to ubuntu 18.04 image for Moja flint libraries and executables # # Building this Docker: -# docker build -f Dockerfile.flint.ubuntu.18.04 --build-arg NUM_CPU=4 --build-arg FLINT_BRANCH=develop -t moja/flint:ubuntu-18.04 . +# docker build -f Dockerfile.flint.ubuntu.18.04 --build_arg BUILD_TYPE=RELEASE --build-arg NUM_CPU=4 --build-arg FLINT_BRANCH=develop -t moja/flint:ubuntu-18.04 . # # ================================================================================================================== @@ -14,8 +14,8 @@ LABEL maintainer="info@moja.global" ARG FLINT_BRANCH ARG NUM_CPU=1 ARG DEBIAN_FRONTEND=noninteractive +ARG BUILD_TYPE=DEBUG -ENV ZIPPER_VERSION 1.0.1 ENV ROOTDIR /usr/local WORKDIR $ROOTDIR/src @@ -42,11 +42,10 @@ RUN git clone --recursive https://github.com/sebastiandev/zipper.git \ && make clean \ && cd $ROOTDIR/src - # GET moja.global RUN git clone --recursive --depth 1 -b ${FLINT_BRANCH} https://github.com/moja-global/FLINT.git flint \ && mkdir -p flint/Source/build && cd flint/Source/build \ - && cmake -DCMAKE_BUILD_TYPE=DEBUG \ + && cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE \ -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ -DENABLE_TESTS:BOOL=OFF \ -DENABLE_MOJA.MODULES.GDAL=ON \ @@ -55,7 +54,6 @@ RUN git clone --recursive --depth 1 -b ${FLINT_BRANCH} https://github.com/moja-g -DBUILD_SHARED_LIBS=ON .. \ && make --quiet -j $NUM_CPU \ && make --quiet install \ -# && make clean \ && cd $ROOTDIR/src RUN ln -s $ROOTDIR/lib/libmoja.modules.* $ROOTDIR/bin From 976b6a3954c53d5da1c535c6f7e3a974769b5bae Mon Sep 17 00:00:00 2001 From: Mavneet Kaur Date: Sun, 18 Apr 2021 02:01:47 +0530 Subject: [PATCH 21/40] added video installation links Signed-off-by: Mavneet Kaur --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f75d9e4..11d0348 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,18 @@ The FLINT is using the lessons learned from first generation tools, to build a n ## Installation Docs -Please checkout the [moja global developer docs](https://docs.moja.global.com) for complete instructions on how to setup the repository. You may also refer this documentation for setting up FLINT.example, GCBM or just get an idea of the moja global workflow! You may also suggest an improvement in the current docs by creating an issue [here](hhttps://github.com/moja-global/GSoD.moja_global_docs). +Please checkout the [moja global developer docs](https://docs.moja.global) for complete instructions on how to setup the repository. You may also refer this documentation for setting up FLINT.example, GCBM or just get an idea of the moja global workflow! You may also suggest an improvement in the current docs by creating an issue [here](hhttps://github.com/moja-global/GSoD.moja_global_docs). ## Installation Videos We also have a set of installation videos to help you out with the installation. If you prefer video installation procedure as opposed to textual documentation, this will be a perfect starter for you! +1. [FLINT Core on Visual Studio 2019](https://www.youtube.com/watch?v=BmHltWrxCTY&t=9s) +2. [FLINT Example (RothC model) on Visual Studio](https://www.youtube.com/watch?v=Jfi2-vEhfkg) +3. [FLINT Example (Chapman Richards model) on Visual Studio](https://www.youtube.com/watch?v=JFTyeZQbPjI) +4. [FLINT Docker on Ubuntu 20.04](https://www.youtube.com/watch?v=eiCPhv-SRNc) + +To learn more, you can also visit the [moja global youtube channel](https://www.youtube.com/channel/UCfQUrrNP1Xf-Fv4c8uHYXhQ). + ## How to Get Involved? @@ -39,7 +46,7 @@ moja global welcomes a wide range of contributions as explained in [Contributing * If you have a question about the code, submit [user feedback](https://github.com/moja-global/About-moja-global/blob/master/Contributing/How-to-Provide-User-Feedback.md) in the relevant repository * If you have a general question about a project or repository or moja global, [join moja global](https://docs.moja.global/en/latest/contact.html) and * [submit a discussion](https://help.github.com/en/articles/about-team-discussions) to the project, repository or moja global [team](https://github.com/orgs/moja-global/teams) - * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](mojaglobal.slack.com). + * [submit a message](https://get.slack.help/hc/en-us/categories/200111606#send-messages) to the relevant channel on [moja global's Slack workspace](https://mojaglobal.slack.com). * If you have other questions, please write to info@moja.global From b9fa2d842fccbb282ec86d331b03b53ef41c8880 Mon Sep 17 00:00:00 2001 From: Sree Vidhya <66103871+vidhya001@users.noreply.github.com> Date: Sat, 24 Apr 2021 20:26:21 +0530 Subject: [PATCH 22/40] Update README.md Signed-off-by: Sree Vidhya <66103871+vidhya001@users.noreply.github.com> --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 11d0348..81df1ce 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,15 @@ The FLINT makes developing and operating advanced systems achievable by all coun The FLINT is using the lessons learned from first generation tools, to build a new framework that meets present and future needs. The key improvements compared to the first generation tools include: + a flexible, modular approach, allowing for country specific implementations from [IPCC Tier 1 to 3 ](https://www.reddcompass.org/mgd-content-v1/dita-webhelp/en/Box1.html) + support for [wall-to-wall, sample based, jurisdictional, and supply chain approaches](https://static1.squarespace.com/static/5896200f414fb57d26f3d600/t/59362b028419c2db8f57e747/1496722191543/REDD_nested_projects.pdf) -+ the ability to cover all [land uses and land use changes, and activity-based reporting such as REDD+](https://theredddesk.org/what-redd) ++ the ability to cover all [land uses and land use changes, and activity-based reporting such as REDD+](https://unfccc.int/topics/land-use/workstreams/redd/what-is-redd) + scenario analysis systems to allow the development of projections + the potential to be used for multiple other purposes, including economics, water and biodiversity -+ development managed using a true open source approach under [moja global](http://moja.global), which will allow users (countries, companies and organizations) to direct strategy and control the budget. ++ development managed using a true open source approach under [moja global](https://moja.global), which will allow users (countries, companies and organizations) to direct strategy and control the budget. + software that allows data processing on local desktops or cloud-based systems ## Installation Docs -Please checkout the [moja global developer docs](https://docs.moja.global) for complete instructions on how to setup the repository. You may also refer this documentation for setting up FLINT.example, GCBM or just get an idea of the moja global workflow! You may also suggest an improvement in the current docs by creating an issue [here](hhttps://github.com/moja-global/GSoD.moja_global_docs). +Please checkout the [moja global developer docs](https://docs.moja.global) for complete instructions on how to setup the repository. You may also refer this documentation for setting up [FLINT.example](https://docs.moja.global/en/latest/DevelopmentSetup/FLINT.example_installation.html), [GCBM](https://docs.moja.global/en/latest/GCBMDevelopmentSetup/index.html) or just get an idea of the moja global workflow! You may also suggest an improvement in the current docs by creating an issue [here](hhttps://github.com/moja-global/GSoD.moja_global_docs). ## Installation Videos From 5ee290033f87d68e2b6a0d6553b58ac11ad9939f Mon Sep 17 00:00:00 2001 From: Mal Date: Thu, 6 May 2021 09:26:08 +1000 Subject: [PATCH 23/40] Changes to std::mutex --- .../include/moja/flint/aggregatorfilewriter.h | 4 ++-- .../include/moja/flint/recordaccumulatoruuid.h | 6 ++---- .../moja/flint/recordaccumulatorwithmutex.h | 16 ++++++++-------- .../flint/spatialtiledlocaldomaincontroller.h | 2 +- .../threadedaspatiallocaldomaincontroller.h | 8 +++----- .../include/moja/flint/writesystemconfig.h | 6 +++--- .../include/moja/flint/writevariablegrid.h | 15 +++++++-------- Source/moja.flint/src/aggregatorfilewriter.cpp | 2 +- Source/moja.flint/src/libraryfactory.cpp | 6 +++--- .../src/spatialtiledlocaldomaincontroller.cpp | 9 ++++----- .../threadedaspatiallocaldomaincontroller.cpp | 6 ++---- Source/moja.flint/src/writevariablegrid.cpp | 12 ++++++------ .../moja/modules/gdal/writevariablegeotiff.h | 11 ++++++----- Source/moja.modules.gdal/src/libraryfactory.cpp | 2 +- .../src/writevariablegeotiff.cpp | 5 ++--- 15 files changed, 51 insertions(+), 59 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h b/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h index e62186c..636d62f 100644 --- a/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h +++ b/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h @@ -17,7 +17,7 @@ typedef Poco::Tuple runStatDa class FLINT_API AggregatorFileWriter : public flint::ModuleBase { public: - AggregatorFileWriter(Poco::Mutex& fileMutex, AggregatorLandUnitSharedData& aggregatorLandUnitSharedData, + AggregatorFileWriter(std::mutex& fileMutex, AggregatorLandUnitSharedData& aggregatorLandUnitSharedData, std::shared_ptr> dateDimension, std::shared_ptr> date2Dimension, std::shared_ptr> poolInfoDimension, @@ -47,7 +47,7 @@ class FLINT_API AggregatorFileWriter : public flint::ModuleBase { void writeFlux(); // -- Shared Data - Poco::Mutex& _fileMutex; + std::mutex& _fileMutex; AggregatorLandUnitSharedData& _aggregatorLandUnitSharedData; std::shared_ptr> _dateDimension; std::shared_ptr> _date2Dimension; diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h b/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h index 1425859..961edbd 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h @@ -5,8 +5,6 @@ #include -#include - #include #include @@ -45,7 +43,7 @@ class RecordAccumulatorUuid { RecordAccumulatorUuid() {} const TRecord* accumulate(TRecord record) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { // Found an existing ID for the key. @@ -105,7 +103,7 @@ class RecordAccumulatorUuid { const rec_accu_vec& records() const { return _records; }; private: - Poco::Mutex _mutex; + std::mutex _mutex; rec_accu_set _recordsIdx; rec_accu_vec _records; boost::uuids::random_generator _uuidGenerator; diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h index 7a6e3af..e6dcb7f 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h @@ -6,9 +6,9 @@ #include -#include #include +#include #include namespace moja { @@ -23,7 +23,7 @@ class RecordAccumulatorWithMutex { typedef typename rec_accu_vec::size_type rec_accu_size_type; Record* insert(Int64 id, std::shared_ptr> record) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); // ID has been assigned by user, assume that we can run with this _nextId = id + 1; // can't guarantee that this will be called in 'id increasing' order but a good guess perhaps record->setId(id); @@ -33,7 +33,7 @@ class RecordAccumulatorWithMutex { } Record* accumulate(std::shared_ptr> record) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); auto record_ptr = record.get(); auto it = _recordsIdx.find(record_ptr); if (it != _recordsIdx.end()) { @@ -51,7 +51,7 @@ class RecordAccumulatorWithMutex { } Record* accumulate(std::shared_ptr> record, Int64 requestedId) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); auto record_ptr = record.get(); auto it = _recordsIdx.find(record_ptr); if (it != _recordsIdx.end()) { @@ -105,7 +105,7 @@ class RecordAccumulatorWithMutex { const rec_accu_vec& records() const { return _records; }; private: - Poco::Mutex _mutex; + std::mutex _mutex; Int64 _nextId = 1; rec_accu_set _recordsIdx; rec_accu_vec _records; @@ -139,7 +139,7 @@ class RecordAccumulatorWithMutex2 { RecordAccumulatorWithMutex2() : _nextId(1) {} const TRecord* insert(Int64 id, TRecord record) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); if (_records.size() == _records.capacity()) { _recordsIdx.clear(); _records.reserve(compute_size()); @@ -159,7 +159,7 @@ class RecordAccumulatorWithMutex2 { const TRecord* accumulate(TRecord record) { return accumulate(record, -1); } const TRecord* accumulate(TRecord record, Int64 requestedId) { - Poco::Mutex::ScopedLock lock(_mutex); + std::unique_lock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { // Found an existing ID for the key. @@ -225,7 +225,7 @@ class RecordAccumulatorWithMutex2 { const rec_accu_vec& records() const { return _records; }; private: - Poco::Mutex _mutex; + std::mutex _mutex; std::atomic _nextId; rec_accu_set _recordsIdx; rec_accu_vec _records; diff --git a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h index 75ab107..ae8cf41 100644 --- a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h @@ -138,7 +138,7 @@ class FLINT_API SpatialTiledLocalDomainController : public LocalDomainController // -- General mutable std::shared_ptr _provider; std::shared_ptr _spatiallocationinfo; - Poco::Mutex _blockListMutex; + std::mutex _blockListMutex; std::queue _blockIdxList; // list of block to sim in thread/non-threaded processing. std::map, std::vector> _blockCellIdxList; // list of cells to sim, mapped by blockIdx. Used in thread/non-threaded processing diff --git a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h index 7040a66..2b4f6f7 100644 --- a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h @@ -10,8 +10,6 @@ #include -#include - #include #include @@ -22,7 +20,7 @@ class ThreadedAspatialLocalDomainController; class AspatialLocalDomainThread { public: - AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, Poco::Mutex& tileListMutex, + AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, std::mutex& tileListMutex, std::queue& tileList, const configuration::Configuration* config); @@ -35,7 +33,7 @@ class AspatialLocalDomainThread { ThreadedAspatialLocalDomainController* _parent; configuration::Configuration _threadConfig; int _threadId; - Poco::Mutex& _tileListMutex; + std::mutex& _tileListMutex; std::queue& _tileList; const configuration::Configuration* _config; std::shared_ptr _ldc; @@ -61,7 +59,7 @@ class FLINT_API ThreadedAspatialLocalDomainController final : public flint::Loca std::vector _threads; std::unique_ptr _landscape; - Poco::Mutex _tileListMutex; + std::mutex _tileListMutex; std::queue _tileList; }; diff --git a/Source/moja.flint/include/moja/flint/writesystemconfig.h b/Source/moja.flint/include/moja/flint/writesystemconfig.h index ccae118..acf7921 100644 --- a/Source/moja.flint/include/moja/flint/writesystemconfig.h +++ b/Source/moja.flint/include/moja/flint/writesystemconfig.h @@ -3,7 +3,7 @@ #include -#include +#include namespace moja { namespace flint { @@ -35,7 +35,7 @@ class SpatialLocationInfo; /// class FLINT_API WriteSystemConfig : public flint::ModuleBase { public: - explicit WriteSystemConfig(Poco::Mutex& fileHandlingMutex) + explicit WriteSystemConfig(std::mutex& fileHandlingMutex) : ModuleBase(), _fileHandlingMutex(fileHandlingMutex), notificationType(OnNotificationType::TimingInit), @@ -76,7 +76,7 @@ class FLINT_API WriteSystemConfig : public flint::ModuleBase { private: // Mutexes - Poco::Mutex& _fileHandlingMutex; + std::mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; diff --git a/Source/moja.flint/include/moja/flint/writevariablegrid.h b/Source/moja.flint/include/moja/flint/writevariablegrid.h index 080efc7..1761212 100644 --- a/Source/moja.flint/include/moja/flint/writevariablegrid.h +++ b/Source/moja.flint/include/moja/flint/writevariablegrid.h @@ -7,10 +7,9 @@ #include "moja/flint/ipool.h" #include "moja/flint/modulebase.h" -#include - #include #include +#include namespace moja { namespace flint { @@ -19,7 +18,7 @@ class SpatialLocationInfo; class FLINT_API WriteVariableGrid : public ModuleBase { public: - explicit WriteVariableGrid(Poco::Mutex& fileHandlingMutex) + explicit WriteVariableGrid(std::mutex& fileHandlingMutex) : ModuleBase(), _fileHandlingMutex(fileHandlingMutex), _useIndexesForFolderName(false), @@ -60,7 +59,7 @@ class FLINT_API WriteVariableGrid : public ModuleBase { // --- Base classs for data layer class DataSettingsB { public: - DataSettingsB(Poco::Mutex& fileHandlingMutex, int hdrDataType) + DataSettingsB(std::mutex& fileHandlingMutex, int hdrDataType) : notificationType(OnNotificationType::TimingInit), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -121,14 +120,14 @@ class FLINT_API WriteVariableGrid : public ModuleBase { const flint::IVariable* _variable; std::vector _pool; std::string _tileFolderPath; - Poco::Mutex& _fileHandlingMutex; + std::mutex& _fileHandlingMutex; }; // --- Templated version of Base classs for data layer types template class DataSettingsT : public DataSettingsB { public: - DataSettingsT(Poco::Mutex& fileHandlingMutex, int hdrDataType) : DataSettingsB(fileHandlingMutex, hdrDataType){}; + DataSettingsT(std::mutex& fileHandlingMutex, int hdrDataType) : DataSettingsB(fileHandlingMutex, hdrDataType){}; ~DataSettingsT() = default; virtual void configure(std::string& globalOutputPath, bool useIndexesForFolderName, bool forceVariableFolderName, @@ -157,7 +156,7 @@ class FLINT_API WriteVariableGrid : public ModuleBase { private: // Mutexes - Poco::Mutex& _fileHandlingMutex; + std::mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; @@ -175,4 +174,4 @@ class FLINT_API WriteVariableGrid : public ModuleBase { } // namespace flint } // namespace moja -#endif // MOJA_FLINT_WRITESPATIALDATAGRID_H_ \ No newline at end of file +#endif // MOJA_FLINT_WRITESPATIALDATAGRID_H_ diff --git a/Source/moja.flint/src/aggregatorfilewriter.cpp b/Source/moja.flint/src/aggregatorfilewriter.cpp index 0e4042a..14a2fc0 100644 --- a/Source/moja.flint/src/aggregatorfilewriter.cpp +++ b/Source/moja.flint/src/aggregatorfilewriter.cpp @@ -177,7 +177,7 @@ void AggregatorFileWriter::writeFlux() { // -- Write Flux Facts // append to end of file using _fileMutex to keep things safe - Poco::ScopedLockWithUnlock lock(_fileMutex); + std::unique_lock lock(_fileMutex); // append to file here // Poco::OutputLineEndingConverter output(stream, Poco::LineEnding::NEWLINE_CRLF); diff --git a/Source/moja.flint/src/libraryfactory.cpp b/Source/moja.flint/src/libraryfactory.cpp index 88f55d2..bc194b5 100644 --- a/Source/moja.flint/src/libraryfactory.cpp +++ b/Source/moja.flint/src/libraryfactory.cpp @@ -79,9 +79,9 @@ struct FLINTAggregationSharedDataObject { FLINTAggregationSharedDataObject flint_aggregation_shared_data; -Poco::Mutex _fileHandlingMutexVarGridWriter; -Poco::Mutex _fileHandlingMutexConfigWriter; -Poco::Mutex _fileHandlingMutexAggregationFileWriter; +std::mutex _fileHandlingMutexVarGridWriter; +std::mutex _fileHandlingMutexConfigWriter; +std::mutex _fileHandlingMutexAggregationFileWriter; // Flint Data Factory std::shared_ptr createEventQueueFactory(const std::string& eventTypeStr, int id, const std::string& name, diff --git a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp index 6ac1ca7..5a61a3e 100644 --- a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp +++ b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp @@ -24,7 +24,6 @@ #include #include -#include #include #include @@ -230,7 +229,7 @@ void StatsUnitRecord::mojaLog(const std::string& levelStr, int localDomainId, da class SpatialTiledLocalDomainController::InternalThreadBlocks { public: - InternalThreadBlocks(int threadId, std::string runId, Poco::Mutex& blockIdxListMutex, + InternalThreadBlocks(int threadId, std::string runId, std::mutex& blockIdxListMutex, std::queue& blockIdxList, int blockIdxListSize, std::map, std::vector>& blockCellIdxList, const configuration::Configuration* config, @@ -248,7 +247,7 @@ class SpatialTiledLocalDomainController::InternalThreadBlocks { private: int _threadId; std::string _runId; - Poco::Mutex& _blockIdxListMutex; + std::mutex& _blockIdxListMutex; std::queue& _blockIdxList; std::map, std::vector>& _blockCellIdxList; const configuration::Configuration* _config; @@ -259,7 +258,7 @@ class SpatialTiledLocalDomainController::InternalThreadBlocks { // -------------------------------------------------------------------------------------------- SpatialTiledLocalDomainController::InternalThreadBlocks::InternalThreadBlocks( - int threadId, std::string runId, Poco::Mutex& blockIdxListMutex, std::queue& blockIdxList, + int threadId, std::string runId, std::mutex& blockIdxListMutex, std::queue& blockIdxList, int blockIdxListSize, std::map, std::vector>& blockCellIdxList, const configuration::Configuration* config, std::shared_ptr>& globalStatsDimension, @@ -313,7 +312,7 @@ void SpatialTiledLocalDomainController::InternalThreadBlocks::operator()() { while (keepRunning) { // Pop a block index from the queue. // ToDo: could make the number of blocks to pop here configurable - Poco::ScopedLockWithUnlock lock(_blockIdxListMutex); + std::unique_lock lock(_blockIdxListMutex); if (_blockIdxList.size() > 0) { _ldc->_blockIdxListPosition = int(_blockIdxList.size()); auto blockIdx = _blockIdxList.front(); diff --git a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp index 6400ccc..3d24c58 100644 --- a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp @@ -11,8 +11,6 @@ #include #include -#include - #include using moja::flint::ILocalDomainController; @@ -24,7 +22,7 @@ namespace moja { namespace flint { AspatialLocalDomainThread::AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, - Poco::Mutex& tileListMutex, + std::mutex& tileListMutex, std::queue& tileList, const configuration::Configuration* config) : _parent(parent), @@ -49,7 +47,7 @@ void AspatialLocalDomainThread::operator()() { auto keepRunning = true; while (keepRunning) { // Pop a block index from the queue - Poco::ScopedLockWithUnlock lock(_tileListMutex); + std::unique_lock lock(_tileListMutex); if (_tileList.size() > 0) { auto tile = _tileList.front(); _tileList.pop(); diff --git a/Source/moja.flint/src/writevariablegrid.cpp b/Source/moja.flint/src/writevariablegrid.cpp index b9289f9..2396040 100644 --- a/Source/moja.flint/src/writevariablegrid.cpp +++ b/Source/moja.flint/src/writevariablegrid.cpp @@ -15,11 +15,11 @@ #include #include -#include -#include +#include #include #include +#include namespace moja { namespace flint { @@ -381,7 +381,7 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitInit( } Poco::File tileFolder(_tileFolderPath); - Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + std::unique_lock lock(_fileHandlingMutex); try { tileFolder.createDirectories(); } catch ( @@ -403,7 +403,7 @@ void WriteVariableGrid::DataSettingsT::initializeData(std::shared_ptr void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( std::shared_ptr spatialLocationInfo) { - Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + std::unique_lock lock(_fileHandlingMutex); // for (const auto& timestepData : _data) { typename std::unordered_map>::iterator itPrev; @@ -463,8 +463,7 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( fwrite(timestepData.second.data(), sizeof(T), numCells, pFile); } - boost::iostreams::stream_buffer buf(filenameHdr); - std::ostream out(&buf); + std::ofstream out(filenameHdr); out << "ENVI" << std::endl; out << "description = { " << filenameGrd << " }" << std::endl; @@ -493,6 +492,7 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( "\"Greenwich\",0],UNIT[\"Degree\",0.017453292519943295]] }" << std::endl; out << "band names = { Band 1 }" << std::endl; + out.close(); itPrev = it; } _data.clear(); diff --git a/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h b/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h index 25241ff..6b54f0a 100644 --- a/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h +++ b/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h @@ -9,6 +9,7 @@ #include #include +#include namespace moja { @@ -33,7 +34,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { float64 = 7, }; - explicit WriteVariableGeotiff(Poco::Mutex& fileHandlingMutex) + explicit WriteVariableGeotiff(std::mutex& fileHandlingMutex) : _fileHandlingMutex(fileHandlingMutex), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -57,7 +58,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { // --- Base class for data layer class DataSettingsB { public: - DataSettingsB(Poco::Mutex& fileHandlingMutex, data_type dataType) + DataSettingsB(std::mutex& fileHandlingMutex, data_type dataType) : notificationType(OnNotificationType::TimingInit), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -117,14 +118,14 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { const flint::IVariable* _variable; std::vector _pool; std::string _tileFolderPath; - Poco::Mutex& _fileHandlingMutex; + std::mutex& _fileHandlingMutex; }; // --- Templated version of Base class for data layer types template class DataSettingsT : public DataSettingsB { public: - DataSettingsT(Poco::Mutex& fileHandlingMutex, data_type dataType) + DataSettingsT(std::mutex& fileHandlingMutex, data_type dataType) : DataSettingsB(fileHandlingMutex, dataType){}; ~DataSettingsT() = default; @@ -157,7 +158,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { private: // Mutexes - Poco::Mutex& _fileHandlingMutex; + std::mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; diff --git a/Source/moja.modules.gdal/src/libraryfactory.cpp b/Source/moja.modules.gdal/src/libraryfactory.cpp index 4e8f11c..3686ba5 100644 --- a/Source/moja.modules.gdal/src/libraryfactory.cpp +++ b/Source/moja.modules.gdal/src/libraryfactory.cpp @@ -21,7 +21,7 @@ namespace gdal { extern "C" { -Poco::Mutex _fileHandlingMutexVarGeotiffWriter; +std::mutex _fileHandlingMutexVarGeotiffWriter; MOJA_LIB_API int getModuleRegistrations(ModuleRegistration* outModuleRegistrations) { int index = 0; diff --git a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp index 7885df3..a8f432e 100644 --- a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp +++ b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp @@ -13,7 +13,6 @@ #include #include -#include #include #include @@ -386,7 +385,7 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitInit( } Poco::File tileFolder(_tileFolderPath); - Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + std::unique_lock lock(_fileHandlingMutex); try { tileFolder.createDirectories(); } catch ( @@ -481,7 +480,7 @@ static std::shared_ptr create_gdalraster(const Poco::File& path, template void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown( std::shared_ptr spatialLocationInfo) { - Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + std::unique_lock lock(_fileHandlingMutex); Poco::File tileFolder(_tileFolderPath); std::string folderLocStr; From aa30f6b5fe2d24acdefb932c25f76c4fb07b3d2a Mon Sep 17 00:00:00 2001 From: Mal Date: Thu, 6 May 2021 09:26:30 +1000 Subject: [PATCH 24/40] fixed env issue --- Source/moja.core/src/environment_win32u.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Source/moja.core/src/environment_win32u.cpp b/Source/moja.core/src/environment_win32u.cpp index e0ebdbf..15956aa 100644 --- a/Source/moja.core/src/environment_win32u.cpp +++ b/Source/moja.core/src/environment_win32u.cpp @@ -9,14 +9,14 @@ namespace moja { -static std::string EnvironmentImpl::startProcessFolderImpl() { - static wchar_t path[512] = ""; - if (!path[0]) { - // Get directory this executable was launched from. - GetModuleFileNameW(NULL, path, sizeof(path) - 1); - } - Poco::UnicodeConverter::toUTF8(path, result); - auto folder = Poco::Path(path).parent(); +std::string EnvironmentImpl::startProcessFolderImpl() { + + std::wstring mod_path; + mod_path.resize(MAX_PATH); + mod_path.resize(::GetModuleFileNameW(NULL, &mod_path[0], mod_path.size())); + std::string result; + Poco::UnicodeConverter::toUTF8(mod_path, result); + auto folder = Poco::Path(result).parent().absolute(); return folder.toString(); } From 34f5c467d9963abbfc36408082dc687589fa6a34 Mon Sep 17 00:00:00 2001 From: Mal Date: Thu, 6 May 2021 09:26:53 +1000 Subject: [PATCH 25/40] Turned on gdal by default --- Source/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 55d09d1..6809175 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -28,11 +28,11 @@ set(Moja_COMPONENTS "") # Allow enabling and disabling components option(ENABLE_MOJA.MODULES.ZIPPER "moja.modules.zipper" OFF) -option(ENABLE_MOJA.MODULES.POCO "moja.modules.poco" OFF) -option(ENABLE_MOJA.MODULES.LIBPQ "moja.modules.libpq" OFF) -option(ENABLE_MOJA.MODULES.GDAL "moja.modules.gdal" OFF) -option(ENABLE_MOJA.CLI "moja.cli" ON) -option(ENABLE_MOJA.SYSTEMTEST "moja.systemtest" OFF) +option(ENABLE_MOJA.MODULES.POCO "moja.modules.poco" OFF) +option(ENABLE_MOJA.MODULES.LIBPQ "moja.modules.libpq" ON) +option(ENABLE_MOJA.MODULES.GDAL "moja.modules.gdal" ON) +option(ENABLE_MOJA.CLI "moja.cli" ON) +option(ENABLE_MOJA.SYSTEMTEST "moja.systemtest" OFF) if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "C:/Development/Software/${PROJECT_NAME}" CACHE PATH "..." FORCE) From e9e352e0338f1380f64675e5ad9c2430170e2ea7 Mon Sep 17 00:00:00 2001 From: Mal Date: Fri, 14 May 2021 17:50:29 +1000 Subject: [PATCH 26/40] linux fix --- Source/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 6809175..e4b9747 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10.0) -project(moja VERSION 1.0.6 LANGUAGES CXX) +project(moja VERSION 1.0.6 LANGUAGES C CXX) #turn on using solution folders set_property( GLOBAL PROPERTY USE_FOLDERS ON) From 8dd7d9f5b4c2a913da3889f33925efda02dbfb79 Mon Sep 17 00:00:00 2001 From: Mal Date: Fri, 14 May 2021 17:53:22 +1000 Subject: [PATCH 27/40] Removed ublas --- Source/moja.flint/CMakeLists.txt | 25 +- Source/moja.flint/src/landunitcontroller.cpp | 5 +- .../src/spinuplandunitcontroller.cpp | 1 - Source/moja.flint/tests/CMakeLists.txt | 2 - .../moja.flint/tests/src/matrixtestseigen.cpp | 301 ------------------ .../moja.flint/tests/src/matrixtestsublas.cpp | 159 --------- .../tests/src/operationmanagertestutils.cpp | 18 -- .../tests/src/operationmanagerublastests.cpp | 155 --------- 8 files changed, 2 insertions(+), 664 deletions(-) delete mode 100644 Source/moja.flint/tests/src/matrixtestseigen.cpp delete mode 100644 Source/moja.flint/tests/src/matrixtestsublas.cpp delete mode 100644 Source/moja.flint/tests/src/operationmanagerublastests.cpp diff --git a/Source/moja.flint/CMakeLists.txt b/Source/moja.flint/CMakeLists.txt index 44e88a2..ad9a6f4 100644 --- a/Source/moja.flint/CMakeLists.txt +++ b/Source/moja.flint/CMakeLists.txt @@ -179,18 +179,6 @@ set(MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS include/moja/flint/poolsimplecache.h ) -set(MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS - include/moja/flint/matrixublas.h - include/moja/flint/operationmanagerublas.h - include/moja/flint/operationresultfluxublas.h - include/moja/flint/operationresultfluxiteratorublas.h - include/moja/flint/operationresultublas.h - include/moja/flint/operationproportionalublas.h - include/moja/flint/operationstockublas.h - include/moja/flint/operationtransferublas.h - include/moja/flint/poolublas.h -) - set(MOJA_FLINT_MODULE_SOURCES src/aggregatorfluxstep.cpp src/aggregatorstockstep.cpp @@ -271,14 +259,6 @@ set(MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE src/operationstocksimplecache.cpp ) -set(MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE - src/poolublas.cpp - src/operationmanagerublas.cpp - src/operationproportionalublas.cpp - src/operationresultublas.cpp - src/operationstockublas.cpp -) - source_group("header files\\other" FILES ${MOJA_FLINT_HEADERS}) source_group("source files\\other" FILES ${MOJA_FLINT_SOURCES}) source_group("header files\\modules" FILES ${MOJA_FLINT_MODULE_HEADERS}) @@ -292,8 +272,6 @@ source_group("header files\\op_man_simple" FILES ${MOJA_FLINT_OPERATION_MANAGE source_group("source files\\op_man_simple" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_SOURCE}) source_group("header files\\op_man_simplecache" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS}) source_group("source files\\op_man_simplecache" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE}) -source_group("header files\\op_man_ublas" FILES ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS}) -source_group("source files\\op_man_ublas" FILES ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE}) set (SRCS ${MOJA_FLINT_GENERATED_HEADERS} ${MOJA_FLINT_SOURCES} ${MOJA_FLINT_HEADERS} @@ -302,8 +280,7 @@ set (SRCS ${MOJA_FLINT_GENERATED_HEADERS} ${MOJA_FLINT_FLINTDATA_SOURCES} ${MOJA_FLINT_FLINTDATA_HEADERS} ${MOJA_FLINT_IOPERATION_MANAGER_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_SOURCE} - ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE} - ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE}) + ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE} ) add_library( ${LIBNAME} ${LIB_MODE} ${SRCS} ${ProductVersionFiles}) diff --git a/Source/moja.flint/src/landunitcontroller.cpp b/Source/moja.flint/src/landunitcontroller.cpp index 1b14b14..7fa49f5 100644 --- a/Source/moja.flint/src/landunitcontroller.cpp +++ b/Source/moja.flint/src/landunitcontroller.cpp @@ -7,7 +7,6 @@ #include "moja/flint/ivariable.h" #include "moja/flint/operationmanagersimple.h" #include "moja/flint/operationmanagersimplecache.h" -#include "moja/flint/operationmanagerublas.h" #include #include @@ -37,9 +36,7 @@ void LandUnitController::configure(const configuration::Configuration& config) { auto const operationManagerConfig = config.localDomain()->operationManagerObject(); - if (operationManagerConfig->name() == "Ublas") { - _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); - } else if (operationManagerConfig->name() == "Simple") { + if (operationManagerConfig->name() == "Simple") { _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); } else if (operationManagerConfig->name() == "SimpleCache") { _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); diff --git a/Source/moja.flint/src/spinuplandunitcontroller.cpp b/Source/moja.flint/src/spinuplandunitcontroller.cpp index 86dddbe..8258efc 100644 --- a/Source/moja.flint/src/spinuplandunitcontroller.cpp +++ b/Source/moja.flint/src/spinuplandunitcontroller.cpp @@ -5,7 +5,6 @@ #include "moja/flint/ioperation.h" #include "moja/flint/ivariable.h" #include "moja/flint/landunitcontroller.h" -#include "moja/flint/operationmanagerublas.h" #include diff --git a/Source/moja.flint/tests/CMakeLists.txt b/Source/moja.flint/tests/CMakeLists.txt index 3bf5f88..f46cf07 100644 --- a/Source/moja.flint/tests/CMakeLists.txt +++ b/Source/moja.flint/tests/CMakeLists.txt @@ -14,11 +14,9 @@ set(TEST_SRCS src/flinttests.cpp src/landunitcontrollertests.cpp src/lookuptransformtests.cpp - src/matrixtestsublas.cpp src/operationmanagertestutils.cpp src/operationmanagersimpletests.cpp src/operationmanagersimplecachetests.cpp - src/operationmanagerublastests.cpp src/sqlquerytransformtests.cpp src/recordaccumulatorintegrationtests.cpp ) diff --git a/Source/moja.flint/tests/src/matrixtestseigen.cpp b/Source/moja.flint/tests/src/matrixtestseigen.cpp deleted file mode 100644 index db57fc2..0000000 --- a/Source/moja.flint/tests/src/matrixtestseigen.cpp +++ /dev/null @@ -1,301 +0,0 @@ -#include -#include "moja/flint/timing.h" -#include "moja/datetime.h" -#include "moja/flint/matrixeigen.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include // std::setprecision - -// warning C4996: 'std::_Copy_impl': Function call with parameters that may be unsafe - this call relies on the caller to check that the passed values are correct. -#pragma warning( push ) -#pragma warning( disable : 4996 ) -#include "Eigen/Dense" -#include "Eigen/Sparse" -#pragma warning( pop ) - -using namespace moja; -using namespace moja::flint; - -BOOST_AUTO_TEST_SUITE(MatrixTestsEigen); - -typedef Eigen::SparseMatrix EigenSparseMat; // declares a column-major sparse matrix type of double -typedef Eigen::DiagonalMatrix EigenDiagonal; -typedef Eigen::MatrixXd EigenMat; // declares a dynamic matrix of type double -typedef Eigen::VectorXd EigenVec; // declares a dynamic vector of type double -typedef Eigen::MatrixXd moja_identity; - -typedef Eigen::Triplet test_eigen_triplet; -typedef std::vector test_eigen_tripletlist; - - -void eigen_transferPool(EigenMat& _matrix, int source, int sink, double proportion) { - //Eigen::Triplet t(source, sink, proportion); - - _matrix(source, sink) += proportion; - _matrix(source, source) -= proportion; -} - -void eigen_transferPool_Sparse(test_eigen_tripletlist& tripletList, int source, int sink, double value) { - tripletList.push_back(moja_eigen_triplet(sink, source, value)); -} - -template -inline T sum_kahan3e(const EigenMat& xs) { - if (xs.size() == 0) return 0; - T sumP(0); - T sumN(0); - T tP(0); - T tN(0); - T cP(0); - T cN(0); - T yP(0); - T yN(0); - for (size_t i = 0, size = xs.size(); i < size; i++) { - if ((*(xs.data() + i)) > 0) { - yP = (*(xs.data() + i)) - cP; - tP = sumP + yP; - cP = (tP - sumP) - yP; - sumP = tP; - } - else { - yN = (*(xs.data() + i)) - cN; - tN = sumN + yN; - cN = (tN - sumN) - yN; - sumN = tN; - } - } - return sumP + sumN; -} - - -// -------------------------------------------------------------------------------------------- - -#if 0 - -BOOST_AUTO_TEST_CASE(flint_Matrix_eigen_DoSomeMatrixStuff) { - - // Shows setIdentity blanks matrix in bounds of the 1's being inserted in diag - - Eigen::Matrix4i m1 = Eigen::Matrix4i::Zero(); - m1.block<3, 3>(1, 0).setIdentity(); - std::cout << "m1" << std::endl; - std::cout << m1 << std::endl; - - Eigen::Matrix4i m2 = Eigen::Matrix4i::Zero(); - m2(0, 1) = 9; - m2(0, 2) = 7; - m2.block<3, 3>(1, 0).setIdentity(); - std::cout << "m2" << std::endl; - std::cout << m2 << std::endl; - - Eigen::Matrix4i m3 = Eigen::Matrix4i::Zero(); - m3(0, 1) = 9; - m3(0, 2) = 7; - m3(1, 2) = 5; - m3.setIdentity(); - std::cout << "m3" << std::endl; - std::cout << m3 << std::endl; - - // Play with Vectors - - Eigen::Matrix4i m4 = Eigen::Matrix4i::Identity(); - - Eigen::Vector2i v1 = Eigen::Vector2i::Zero(); - std::cout << "v1" << std::endl; - std::cout << v1 << std::endl; - - Eigen::Vector4i v2 = Eigen::Vector4i::Identity(); - std::cout << "v2" << std::endl; - std::cout << v2 << std::endl; - - Eigen::Vector4i r_v1 = m4 * v2; - std::cout << "r_v1" << std::endl; - std::cout << r_v1 << std::endl; - - //Eigen::Vector4i r_v2 = m4 * v1; - //std::cout << "r_v2" << std::endl; - //std::cout << r_v2 << std::endl; -} - -#endif - -// -------------------------------------------------------------------------------------------- - -BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_iterate_data) { - Eigen::Matrix test = Eigen::Matrix::Random(5, 5); - - auto now = time(0); - sum_kahan3e(test); -} - -void simpleConstructAndCalc(int length) { - auto testName = boost::unit_test::framework::current_test_case().p_name; - auto testSuiteName = (boost::unit_test::framework::get(boost::unit_test::framework::current_test_case().p_parent_id)).full_name(); - - auto startConstruct = clock(); - - moja_eigen_matrix _testMatrix; - _testMatrix.resize(length, length); - _testMatrix.setIdentity(); - //_testMatrix.setZero(); - moja_eigen_vector _testVector(length); - _testVector.setRandom(length); - - auto finishConstruct = clock(); - - //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << ": Mat : " << _testMatrix << std::endl; - //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << ": Vec : " << _testVector << std::endl; - - auto startCalcs = clock(); - - for (int i = 0; i < 100; i++) - for (int j = 0; j < 10000; j++) { - moja_eigen_vector result = MOJA_EIGEN_MAT_PROD(_testMatrix, _testVector); - } - - auto finishCalcs = clock(); - auto lengthCalcs = finishCalcs - startCalcs; - auto lengthConstruct = finishConstruct - startConstruct; - - // CLOCKS_PER_SEC - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << " : For size (" << length << "): construction : " << lengthConstruct << std::endl; - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << " : For size (" << length << "): calculations : " << lengthCalcs << std::endl; -} - -// -------------------------------------------------------------------------------------------- - -BOOST_AUTO_TEST_CASE(flint_Matrix_eigen_SpeedChecks) { - - for (int i = 10; i <= 60; i+=10) - simpleConstructAndCalc(i); -} - - -// -------------------------------------------------------------------------------------------- - -BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_001) { - const int numVars = 6; - EigenVec pools(numVars); - pools.setZero(); - - pools[0] = 0.02; - pools[1] = 8.681; - pools[2] = 0.05; - pools[3] = 0; - pools[4] = 20.96; - pools[5] = 0; - - EigenMat _I(numVars, numVars); - _I.setIdentity(); - - EigenDiagonal diag = pools.asDiagonal(); - EigenMat forOutput = diag; - - EigenMat matrix(numVars, numVars); - matrix.setIdentity(); - - eigen_transferPool(matrix, 0, 5, 0.250852); - eigen_transferPool(matrix, 1, 5, 0.012596); - eigen_transferPool(matrix, 2, 5, 0); - eigen_transferPool(matrix, 3, 5, 0.0190627); - eigen_transferPool(matrix, 4, 5, 0.00126915); - eigen_transferPool(matrix, 0, 2, 0.0397643); - eigen_transferPool(matrix, 1, 2, 0.00199667); - eigen_transferPool(matrix, 3, 2, 0.00302176); - eigen_transferPool(matrix, 4, 3, 0); - eigen_transferPool(matrix, 0, 4, 0.0413873); - eigen_transferPool(matrix, 1, 4, 0.00207817); - eigen_transferPool(matrix, 2, 4, 0); - eigen_transferPool(matrix, 3, 4, 0.00314509); - eigen_transferPool(matrix, 4, 4, 0.000410574); - - auto a = matrix - _I; - auto b = diag * a; - auto flux = b * 1; // timeScale - - pools = pools.transpose() * matrix; // timeScale - - BOOST_CHECK_CLOSE(pools[0], 0.013359927999999998, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[1], 8.5362804379599986, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[2], 0.068128378269999998, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[3], 0, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[4], 20.952266955769996, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[5], 0.14096429999999999, 0.000000000000001); -} - -// -------------------------------------------------------------------------------------------- -// NOTE: Eigen will NOT remove the 0.0 results from the final sparse matrix - -BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_zero_transfer_test) { - auto testName = boost::unit_test::framework::current_test_case().p_name; - auto testSuiteName = (boost::unit_test::framework::get(boost::unit_test::framework::current_test_case().p_parent_id)).full_name(); - - const int numVars = 6; - EigenVec pools(numVars); - pools.setZero(); - - pools[0] = 0.02; - pools[1] = 8.681; - pools[2] = 0.05; - pools[3] = 0; - pools[4] = 20.96; - pools[5] = 0; - - EigenMat _I(numVars, numVars); - _I.setIdentity(); - - EigenDiagonal diag = pools.asDiagonal(); - test_eigen_tripletlist tripletList; - EigenSparseMat matrix; - matrix.resize(numVars, numVars); - matrix.reserve(5); - //matrix.setZero(); - - //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix blank" << std::endl; - //std::cout << matrix << std::endl; - - eigen_transferPool_Sparse(tripletList, 0, 5, 0.0); - eigen_transferPool_Sparse(tripletList, 0, 4, 0.1); - - matrix.setFromTriplets(tripletList.begin(), tripletList.end()); - - //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix set" << std::endl; - //std::cout << matrix << std::endl; - - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix iteration" << std::endl; - for (int k = 0; k < matrix.outerSize(); ++k) { - for (EigenSparseMat::InnerIterator flux(matrix, k); flux; ++flux) { - auto srcIx = flux.col(); - auto dstIx = flux.row(); - auto val = flux.value(); - - std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; - } - } - - EigenSparseMat fluxes = matrix * diag; - - //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": fluxes" << std::endl; - //std::cout << fluxes << std::endl; - - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": fluxes iteration" << std::endl; - for (int k = 0; k < fluxes.outerSize(); ++k) { - for (EigenSparseMat::InnerIterator flux(fluxes, k); flux; ++flux) { - auto srcIx = flux.col(); - auto dstIx = flux.row(); - auto val = flux.value(); - - std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; - } - } - -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/Source/moja.flint/tests/src/matrixtestsublas.cpp b/Source/moja.flint/tests/src/matrixtestsublas.cpp deleted file mode 100644 index e8df16d..0000000 --- a/Source/moja.flint/tests/src/matrixtestsublas.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include - -// warning C4996: 'std::_Uninitialized_copy0': Function call with parameters that may be unsafe - this call relies on -// the caller to check that the passed values are correct. -#pragma warning(push) -#pragma warning(disable : 4996) -// Work-around needed for ublas when using boost >= 1.64 -//#include -#include -#include -#include -#include -#include -#pragma warning(pop) - -#include - -#include -#include -#include // std::setprecision -#include -#include -#include -#include -#include -#include - -BOOST_AUTO_TEST_SUITE(MatrixTestsuBlas); - -namespace ublas = boost::numeric::ublas; -typedef ublas::vector moja_vector; -typedef ublas::compressed_matrix moja_matrix; -typedef ublas::diagonal_matrix moja_diagonal; -typedef ublas::identity_matrix moja_identity; -typedef moja_matrix::iterator1 it1_t; -typedef moja_matrix::iterator2 it2_t; - -void transferPool(moja_matrix& _matrix, int source, int sink, double proportion) { - _matrix(source, sink) += proportion; - _matrix(source, source) -= proportion; -} - -BOOST_AUTO_TEST_CASE(flint_Matrix_flux) { - const int test_size = 6; - - moja_vector pools(test_size); - pools[0] = 0.02; - pools[1] = 8.681; - pools[2] = 0.05; - pools[3] = 0; - pools[4] = 20.96; - pools[5] = 0; - - moja_identity _I(test_size); - moja_diagonal diag(test_size, pools.data()); - moja_matrix matrix(_I); - - transferPool(matrix, 0, 5, 0.250852); - transferPool(matrix, 1, 5, 0.012596); - transferPool(matrix, 2, 5, 0); - transferPool(matrix, 3, 5, 0.0190627); - transferPool(matrix, 4, 5, 0.00126915); - transferPool(matrix, 0, 2, 0.0397643); - transferPool(matrix, 1, 2, 0.00199667); - transferPool(matrix, 3, 2, 0.00302176); - transferPool(matrix, 4, 3, 0); - transferPool(matrix, 0, 4, 0.0413873); - transferPool(matrix, 1, 4, 0.00207817); - transferPool(matrix, 2, 4, 0); - transferPool(matrix, 3, 4, 0.00314509); - transferPool(matrix, 4, 4, 0.000410574); - - // Flux - //-0.00664007 0 0.000795285 0 0.000827746 0.00501704 - // 0 - 0.144719 0.0173331 0 0.0180406 0.109346 - // 0 0 0 0 0 0 - // 0 0 0 0 0 0 - // 0 0 0 0 - 0.0266013 0.0266013 - // 0 0 0 0 0 0 - - auto a = matrix - _I; - auto b = ublas::prec_prod(diag, a); - auto flux = b * 1; // timeScale - - pools = ublas::prec_prod(pools, matrix) * 1; // timeScale - - BOOST_CHECK_CLOSE(pools[0], 0.013359927999999998, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[1], 8.5362804379599986, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[2], 0.068128378269999998, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[3], 0, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[4], 20.952266955769996, 0.000000000000001); - BOOST_CHECK_CLOSE(pools[5], 0.14096429999999999, 0.000000000000001); -} - -// -------------------------------------------------------------------------------------------- -// NOTE: UBLAS will remove the 0.0 results from the final sparse matrix - -BOOST_AUTO_TEST_CASE(moja_Matrix_ublas_zero_transfer_test) { - auto testName = boost::unit_test::framework::current_test_case().p_name; - auto testSuiteName = (boost::unit_test::framework::get( - boost::unit_test::framework::current_test_case().p_parent_id)) - .full_name(); - - const int test_size = 6; - moja_vector pools(test_size); - - pools[0] = 0.02; - pools[1] = 8.681; - pools[2] = 0.05; - pools[3] = 0; - pools[4] = 20.96; - pools[5] = 0; - - moja_diagonal diag(test_size, pools.data()); - - moja_matrix matrix(test_size, test_size, test_size); - - // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << - // testName << ": matrix blank" << std::endl; std::cout << matrix << std::endl; - - matrix(0, 5) = 0.0; - matrix(0, 4) = 0.1; - - // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << - // testName << ": matrix set" << std::endl; std::cout << matrix << std::endl; - - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') - << testName << ": matrix iteration" << std::endl; - for (it1_t it1 = matrix.begin1(); it1 != matrix.end1(); ++it1) { - for (it2_t it2 = it1.begin(); it2 != it1.end(); ++it2) { - // std::cout << "(" << it2.index1() << "," << it2.index2() << ") = "; - // std::cout << *it2 << std::endl; - auto srcIx = it2.index1(); - auto dstIx = it2.index2(); - auto val = *it2; - std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; - } - } - - moja_matrix fluxes = ublas::prec_prod(diag, matrix); - - // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << - // testName << ": fluxes" << std::endl; std::cout << fluxes << std::endl; - - std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') - << testName << ": fluxes iteration" << std::endl; - for (it1_t it1 = fluxes.begin1(); it1 != fluxes.end1(); ++it1) { - for (it2_t it2 = it1.begin(); it2 != it1.end(); ++it2) { - // std::cout << "(" << it2.index1() << "," << it2.index2() << ") = "; - // std::cout << *it2 << std::endl; - auto srcIx = it2.index1(); - auto dstIx = it2.index2(); - auto val = *it2; - std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; - } - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/Source/moja.flint/tests/src/operationmanagertestutils.cpp b/Source/moja.flint/tests/src/operationmanagertestutils.cpp index 4ec3165..83e3f9f 100644 --- a/Source/moja.flint/tests/src/operationmanagertestutils.cpp +++ b/Source/moja.flint/tests/src/operationmanagertestutils.cpp @@ -234,24 +234,6 @@ void test_NoResultFluxIteration(mf::IOperationManager& manager, mf::IModule& mod BOOST_CHECK_EQUAL(manager.operationResultsPending().size(), 1); - // UBLAS will remove the zero result flux in the product, other implementations won't - - // if (manager.operationResultsPending().size() == 0) - // return; - - // auto operationResult = *(manager.operationResultsPending().begin()); - - // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << - // testName << ": fluxes" << std::endl; - - // int count = 0; - // for (auto p : operationResult->operationResultFluxCollection()) { - // std::cout << "Src: "<< p->source() << ", Snk: " << p->sink() << ", Val: " << p->value() << std::endl; - // count++; - //} - - // BOOST_CHECK_EQUAL(count, 0); - data.clear(); } diff --git a/Source/moja.flint/tests/src/operationmanagerublastests.cpp b/Source/moja.flint/tests/src/operationmanagerublastests.cpp deleted file mode 100644 index aa6a26a..0000000 --- a/Source/moja.flint/tests/src/operationmanagerublastests.cpp +++ /dev/null @@ -1,155 +0,0 @@ -#include "flinttests.h" -#include "operationmanagertestutils.h" - -#include -#include -#include - -#include - -#include - -#include - -using namespace moja; -using namespace moja::flint; -namespace conf = moja::flint::configuration; - -// -------------------------------------------------------------------------------------------- -struct PoolInitValueAndResult { - std::string name; - double value; - double result; - - std::shared_ptr poolHandle; -}; - -// -------------------------------------------------------------------------------------------- - -#define OP_MANAGER_TO_USE OperationManagerUblas - -// -------------------------------------------------------------------------------------------- - -struct UblasFixture { - Timing timing; - ModuleBase module; - DynamicObject config; - OperationManagerUblas manager; - - UblasFixture() : config({{"use_kahan", true}}), manager(timing, config) { - timing.setStartDate(DateTime(2001, 1, 1)); - timing.setStartDate(DateTime(2013, 12, 31)); - timing.init(); - - auto& metadata = module.metaData(); - metadata.setDefaults(); - metadata.moduleName = "testModuleUblas"; - } -}; - -BOOST_FIXTURE_TEST_SUITE(Ublas_operationmanagertests, UblasFixture); - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_NoPoolIteration) { test_NoPoolIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SinglePoolIteration) { test_SinglePoolIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_MultiplePoolIteration) { test_MultiplePoolIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_NoResultIteration) { test_NoResultIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SingleResultIteration) { test_SingleResultIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_MultipleResultIteration) { test_MultipleResultIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_NoResultFluxIteration) { test_NoResultFluxIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SingleResultFluxIteration) { test_SingleResultFluxIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_MultipleResultFluxIteration) { test_MultipleResultFluxIteration(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SingleProportionTransfer) { test_SingleProportionTransfer(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SingleStockTransfer) { test_SingleStockTransfer(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_DoubleProportionalTransfer) { test_DoubleProportionalTransfer(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_DoubleStockTransfer) { test_DoubleStockTransfer(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_DoubleStockAndApplyTransfer) { test_DoubleStockAndApplyTransfer(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_TwoOperationsStockAndProportional) { -// test_TwoOperationsStockAndProportional(manager, module); -// } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_Kahan_summation_issues_Proportion) { -// test_Kahan_summation_issues_Proportion(manager, module); -// } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_Kahan_summation_issues_Proportion_with_SpinUp) { -// test_Kahan_summation_issues_Proportion_with_SpinUp(manager, module); -// } - -#if 0 // don't do performance testing, too slow for blas - -// -------------------------------------------------------------------------------------------- -BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestProportionalSLEEK) { - test_PerformanceTestProportionalSLEEK(manager, module); -} - -// -------------------------------------------------------------------------------------------- -BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestStockSLEEK) { - test_PerformanceTestStockSLEEK(manager, module); -} - -// -------------------------------------------------------------------------------------------- -BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestCBM) { - test_PerformanceTestCBM(manager, module); -} - -#endif - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_SubmitOperationAddsToPendingQueue) { SubmitOperationAddsToPendingQueue(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ClearLastAppliedOperationResults) { ClearLastAppliedOperationResults(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsAppendsToCommittedQueue) { -// ApplyOperationsAppendsToCommittedQueue(manager, module); -// } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsCorrectlyUpdatesPoolsForSimpleCase) { -// ApplyOperationsCorrectlyUpdatesPoolsForSimpleCase(manager, module); -// } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsCorrectlyUpdatesPoolsForComplexCase) { -// ApplyOperationsCorrectlyUpdatesPoolsForComplexCase(manager, module); -// } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ApplyAndGetOperationsLastApplied) { ApplyAndGetOperationsLastApplied(manager, module); } - -// // -------------------------------------------------------------------------------------------- -// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsClearsPendingQueue) { ApplyOperationsClearsPendingQueue(manager, module); } - -BOOST_AUTO_TEST_SUITE_END(); From 8b380285fcc79424bfc5bda95182fb17f2f21a2b Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 19 May 2021 10:36:53 +1000 Subject: [PATCH 28/40] Updates to RecordAccumulatorWithMutex2 + added Id type to template + added shrink to fit --- .devcontainer/Dockerfile | 23 +++++++++++++++++- .devcontainer/devcontainer.json | 24 ++++++++++++++++++- .vscode/settings.json | 8 +++++-- .../moja/flint/recordaccumulatorwithmutex.h | 15 ++++++++---- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index bb02d19..eb4370a 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,6 +1,27 @@ -FROM mojaglobal/vscode-baseimage:bionic +# FROM mojaglobal/vscode-baseimage:bionic +FROM mulliongroup/vscode-baseimage_standalone_vcpkg:bionic ENV DEBIAN_FRONTEND=noninteractive +ARG NUM_CPU=6 + +ENV ROOTDIR /usr/local +ENV PATH $ROOTDIR/bin:$PATH +ENV LD_LIBRARY_PATH $ROOTDIR/lib:$ROOTDIR/src/vcpkg/installed/x64-linux/lib:$ROOTDIR/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV PYTHONPATH $ROOTDIR/lib:$PYTHONPATH +ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt +ENV GDAL_DATA=$ROOTDIR/share/gdal +ENV GDAL_HTTP_VERSION 2 +ENV GDAL_HTTP_MERGE_CONSECUTIVE_RANGES YES +ENV GDAL_HTTP_MULTIPLEX YES + +WORKDIR $ROOTDIR/src + +RUN apt-get update \ + && export DEBIAN_FRONTEND=noninteractive \ + && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + +RUN apt-get install -y gettext libzstd1-dev sqlformat + # Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 331d76b..0d755fc 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,12 +3,26 @@ { "name": "C++", "dockerFile": "Dockerfile", - "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined" + ], // Set *default* container specific settings.json values on container create. "settings": { "terminal.integrated.shell.linux": "/bin/bash" }, + "build": { + "args": { + "NUM_CPU": "8", + "FLINT_BRANCH": "develop", + "MYARGFROMENVVAR": "${localEnv:VARIABLE_NAME}" + } + }, + "containerEnv": { + "GOOGLE_APPLICATION_CREDENTIALS": "/workspaces/moja.mulliongroup_develop/sensitive/datalayer.json" + }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ @@ -18,6 +32,8 @@ "ms-vscode.cmake-tools" ] + "workspaceFolder": "${containerWorkspaceFolder}", + // Use 'forwardPorts' to make a list of ports inside the container available locally. // "forwardPorts": [], @@ -27,4 +43,10 @@ // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. // "remoteUser": "vscode" + "extensions": [ + "ms-vscode.cpptools", + "austin.code-gnu-global", + "twxs.cmake", + "ms-vscode.cmake-tools" + ] } \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 3e3cbd7..f063712 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,15 @@ { "cmake.configureArgs": [ "-DCMAKE_BUILD_TYPE=DEBUG", + "-DDEBUG=ON", "-DCMAKE_INSTALL_PREFIX=/usr/local", "-DENABLE_TESTS:BOOL=OFF", "-DENABLE_MOJA.MODULES.GDAL=ON", "-DENABLE_MOJA.MODULES.LIBPQ=ON", "-DBoost_USE_STATIC_LIBS=OFF", - "-DBUILD_SHARED_LIBS=ON" - ] + "-DBUILD_SHARED_LIBS=ON", + "-DCMAKE_TOOLCHAIN_FILE=/usr/local/src/vcpkg/scripts/buildsystems/vcpkg.cmake" + ], + "cmake.sourceDirectory": "${workspaceFolder}/Source", + "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild" } \ No newline at end of file diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h index e6dcb7f..9688585 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h @@ -111,7 +111,7 @@ class RecordAccumulatorWithMutex { rec_accu_vec _records; }; -template +template class RecordAccumulatorWithMutex2 { struct RecordComparer { bool operator()(const TRecord* lhs, const TRecord* rhs) const { return lhs->operator==(*rhs); } @@ -138,7 +138,7 @@ class RecordAccumulatorWithMutex2 { RecordAccumulatorWithMutex2() : _nextId(1) {} - const TRecord* insert(Int64 id, TRecord record) { + const TRecord* insert(ID_Type id, TRecord record) { std::unique_lock lock(_mutex); if (_records.size() == _records.capacity()) { _recordsIdx.clear(); @@ -158,7 +158,7 @@ class RecordAccumulatorWithMutex2 { const TRecord* accumulate(TRecord record) { return accumulate(record, -1); } - const TRecord* accumulate(TRecord record, Int64 requestedId) { + const TRecord* accumulate(TRecord record, ID_Type requestedId) { std::unique_lock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { @@ -200,7 +200,7 @@ class RecordAccumulatorWithMutex2 { return nullptr; } - const TRecord* searchId(Int64 id) const { + const TRecord* searchId(ID_Type id) const { for (auto& record : _records) { if (record.getId() == id) return &record; } @@ -220,13 +220,18 @@ class RecordAccumulatorWithMutex2 { _records.clear(); } + void shrink_to_fit() { + _recordsIdx.clear(); + _records.shrink_to_fit(); + } + rec_accu_size_type size() { return _records.size(); } const rec_accu_vec& records() const { return _records; }; private: std::mutex _mutex; - std::atomic _nextId; + std::atomic _nextId; rec_accu_set _recordsIdx; rec_accu_vec _records; }; From 27d53a7785dcdd972d3b30b5accce2ac05f20bff Mon Sep 17 00:00:00 2001 From: Jim Date: Wed, 19 May 2021 10:55:49 +1000 Subject: [PATCH 29/40] Reversed typename for ID in template --- .../include/moja/flint/recordaccumulatorwithmutex.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h index 9688585..bfd9cc1 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h @@ -111,7 +111,8 @@ class RecordAccumulatorWithMutex { rec_accu_vec _records; }; -template +// template +template class RecordAccumulatorWithMutex2 { struct RecordComparer { bool operator()(const TRecord* lhs, const TRecord* rhs) const { return lhs->operator==(*rhs); } @@ -138,7 +139,7 @@ class RecordAccumulatorWithMutex2 { RecordAccumulatorWithMutex2() : _nextId(1) {} - const TRecord* insert(ID_Type id, TRecord record) { + const TRecord* insert(Int64 id, TRecord record) { std::unique_lock lock(_mutex); if (_records.size() == _records.capacity()) { _recordsIdx.clear(); @@ -158,7 +159,7 @@ class RecordAccumulatorWithMutex2 { const TRecord* accumulate(TRecord record) { return accumulate(record, -1); } - const TRecord* accumulate(TRecord record, ID_Type requestedId) { + const TRecord* accumulate(TRecord record, Int64 requestedId) { std::unique_lock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { @@ -200,7 +201,7 @@ class RecordAccumulatorWithMutex2 { return nullptr; } - const TRecord* searchId(ID_Type id) const { + const TRecord* searchId(Int64 id) const { for (auto& record : _records) { if (record.getId() == id) return &record; } @@ -231,7 +232,7 @@ class RecordAccumulatorWithMutex2 { private: std::mutex _mutex; - std::atomic _nextId; + std::atomic _nextId; rec_accu_set _recordsIdx; rec_accu_vec _records; }; From 4794ef27689e498e4038048dfaf31a2cfbdec225 Mon Sep 17 00:00:00 2001 From: Jim Date: Mon, 31 May 2021 15:31:59 +1000 Subject: [PATCH 30/40] Small fix for OutputerStreamFlux + was not outputting disturbance_type --- Source/moja.flint/src/outputerstreamflux.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/Source/moja.flint/src/outputerstreamflux.cpp b/Source/moja.flint/src/outputerstreamflux.cpp index 9fdd81f..ff0c756 100644 --- a/Source/moja.flint/src/outputerstreamflux.cpp +++ b/Source/moja.flint/src/outputerstreamflux.cpp @@ -5,6 +5,8 @@ #include "moja/flint/ipool.h" #include "moja/flint/itiming.h" +// #include +#include #include #include #include @@ -71,8 +73,15 @@ void OutputerStreamFlux::outputInit(std::ostream& stream) const { const auto srcPool = _landUnitData->getPool(srcIx); const auto dstPool = _landUnitData->getPool(dstIx); + auto fluxTypeInfoRecordId = flint::FluxType::Unclassified; + if (operationResult->hasDataPackage()) { + auto dataPacket = operationResult->dataPackage().extract>(); + fluxTypeInfoRecordId = dataPacket->_fluxType; + } + stream << timingL->step() << DL_CHR << timingL->curStartDate() << DL_CHR; stream << mdata->moduleName << DL_CHR; + stream << int(fluxTypeInfoRecordId) << DL_CHR; stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val << std::endl; } @@ -95,8 +104,16 @@ void OutputerStreamFlux::outputEndStep(std::ostream& stream) const { const auto srcPool = _landUnitData->getPool(srcIx); const auto dstPool = _landUnitData->getPool(dstIx); - stream << timingL->step() << DL_CHR << timingL->curStartDate() << DL_CHR; + auto fluxTypeInfoRecordId = flint::FluxType::Unclassified; + if (operationResult->hasDataPackage()) { + auto dataPacket = operationResult->dataPackage().extract>(); + fluxTypeInfoRecordId = dataPacket->_fluxType; + } + + stream << timingL->step() << DL_CHR; + stream << timingL->curStartDate() << DL_CHR; stream << mdata->moduleName << DL_CHR; + stream << int(fluxTypeInfoRecordId) << DL_CHR; stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val << std::endl; } From e9d000206b86b9aba090c97494973cfe8a159e00 Mon Sep 17 00:00:00 2001 From: Jim Date: Mon, 31 May 2021 15:32:28 +1000 Subject: [PATCH 31/40] DevContainer stuff + got it working with vscode again --- .devcontainer/Dockerfile | 132 +++++++++++++++++++++++++++---- .devcontainer/devcontainer.json | 77 +++++++++++------- .devcontainer/docker-compose.yml | 96 ++++++++++++++++++++++ .vscode/settings.json | 9 ++- 4 files changed, 267 insertions(+), 47 deletions(-) create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index eb4370a..44b3650 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,27 +1,127 @@ -# FROM mojaglobal/vscode-baseimage:bionic -FROM mulliongroup/vscode-baseimage_standalone_vcpkg:bionic +#FROM mulliongroup/moja.mulliongroup.baseimage:ubuntu-20.04 +FROM gcr.io/flintpro-212105/flintpro/moja.mulliongroup.baseimage_vscode:ubuntu-20.04 as builder ENV DEBIAN_FRONTEND=noninteractive -ARG NUM_CPU=6 -ENV ROOTDIR /usr/local -ENV PATH $ROOTDIR/bin:$PATH -ENV LD_LIBRARY_PATH $ROOTDIR/lib:$ROOTDIR/src/vcpkg/installed/x64-linux/lib:$ROOTDIR/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH -ENV PYTHONPATH $ROOTDIR/lib:$PYTHONPATH -ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt -ENV GDAL_DATA=$ROOTDIR/share/gdal -ENV GDAL_HTTP_VERSION 2 -ENV GDAL_HTTP_MERGE_CONSECUTIVE_RANGES YES -ENV GDAL_HTTP_MULTIPLEX YES +ARG CACHE_DATE=2021-05-27.1 +ARG FLINT_BRANCH=develop -WORKDIR $ROOTDIR/src +ENV ROOTDIR /usr RUN apt-get update \ && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 + && apt-get -y install --no-install-recommends gdb apt-utils dialog gettext sqlformat postgresql-client 2>&1 + +#RUN apt-get install -y apt-utils dialog gettext sqlformat + +# +# GET moja.global - Mullion Fork. Build Debug and Release +# +# RUN git clone --recursive --depth 1 -b ${FLINT_BRANCH} https://github.com/MullionGroup/FLINT.git flint \ +# && mkdir -p flint/Source/build && cd flint/Source/build \ +# && cmake -DCMAKE_BUILD_TYPE=DEBUG \ +# -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ +# -DENABLE_TESTS:BOOL=OFF \ +# -DENABLE_MOJA.MODULES.GDAL=ON \ +# -DDEBUG=ON \ +# -DBoost_USE_STATIC_LIBS=OFF \ +# -DBUILD_SHARED_LIBS=ON \ +# -DCMAKE_TOOLCHAIN_FILE=$ROOTDIR/src/vcpkg/scripts/buildsystems/vcpkg.cmake \ +# .. \ +# && make --quiet -j$(nproc) \ +# && make --quiet install \ +# && cd $ROOTDIR/src + +# RUN mkdir -p flint/Source/build_release && cd flint/Source/build_release \ +# && cmake -DCMAKE_BUILD_TYPE=RELEASE \ +# -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ +# -DENABLE_TESTS:BOOL=OFF \ +# -DENABLE_MOJA.MODULES.GDAL=ON \ +# -DBoost_USE_STATIC_LIBS=OFF \ +# -DBUILD_SHARED_LIBS=ON \ +# -DCMAKE_TOOLCHAIN_FILE=$ROOTDIR/src/vcpkg/scripts/buildsystems/vcpkg.cmake \ +# .. \ +# && make --quiet -j$(nproc) \ +# && make --quiet install \ +# && cd $ROOTDIR/src + +# PROJ dependencies +RUN apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsqlite3-0 libtiff5 libcurl4 \ + curl unzip ca-certificates + +# GDAL dependencies +RUN apt-get update -y; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + python3-numpy libpython3.8 \ + libexpat1 \ + libxerces-c3.2 \ + libwebp6 \ + libzstd1 bash libpq5 libssl1.1 libopenjp2-7 + +RUN apt-get update; \ + DEBIAN_FRONTEND=noninteractive apt-get install -y python-is-python3 python3-pip + +WORKDIR /usr/src/vcpkg +RUN ./vcpkg install --triplet x64-linux boost-format boost-thread boost-filesystem boost-iostreams boost-program-options boost-random boost-crc boost-test libzip bzip2 liblzma zlib zstd snappy avro-cpp[snappy] + +# set environment variables +ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt +ENV GDAL_DATA /usr/share/gdal +ENV GDAL_HTTP_VERSION 2 + +ENV PATH $ROOTDIR/bin:$PATH +ENV LD_LIBRARY_PATH $ROOTDIR/lib:$ROOTDIR/src/vcpkg/installed/x64-linux/lib:$LD_LIBRARY_PATH +ENV PYTHONPATH /usr/lib:$PYTHONPATH + +RUN ln -s $ROOTDIR/lib/libmoja.modules.* $ROOTDIR/bin + +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 -RUN apt-get install -y gettext libzstd1-dev sqlformat +RUN ldconfig +### # TEST TEST +### FROM mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 as devtools +### +### # PROJ dependencies +### RUN apt-get update; \ +### DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ +### libsqlite3-0 libtiff5 libcurl4 \ +### curl unzip ca-certificates +### +### # GDAL dependencies +### RUN apt-get update -y; \ +### DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ +### python3-numpy libpython3.8 \ +### libexpat1 \ +### libxerces-c3.2 \ +### libwebp6 \ +### libzstd1 bash libpq5 libssl1.1 libopenjp2-7 +### +### RUN apt-get update; \ +### DEBIAN_FRONTEND=noninteractive apt-get install -y python-is-python3 python3-pip +### +### # set environment variables +### ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt +### ENV GDAL_DATA /usr/share/gdal +### ENV GDAL_HTTP_VERSION 2 +### +### ENV PATH /usr/bin:$PATH +### ENV LD_LIBRARY_PATH /usr/lib:$LD_LIBRARY_PATH +### ENV PYTHONPATH /usr/lib:$PYTHONPATH +### +### # ================================================================================================================== +### # Moja and Mullion libraries and Binaries +### # ================================================================================================================== +### +### COPY --from=builder /usr/lib/ /usr/lib/ +### COPY --from=builder /usr/bin/ /usr/bin/ +### COPY --from=builder /usr/share/ /usr/share/ +### COPY --from=builder /usr/include/gdal_version.h /usr/include +### +### RUN ldconfig +### # RUN ln -s /usr/lib/libmoja.modules.*.so /usr/bin/ -# Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0d755fc..09a5784 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,19 +1,23 @@ -// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.117.1/containers/cpp +// See https://aka.ms/vscode-remote/devcontainer.json for format details. { + // See https://aka.ms/vscode-remote/devcontainer.json for format details. "name": "C++", - "dockerFile": "Dockerfile", - "runArgs": [ - "--cap-add=SYS_PTRACE", - "--security-opt", - "seccomp=unconfined" - ], + // "dockerFile": "Dockerfile", + "dockerComposeFile": "docker-compose.yml", + "service": "app", + "shutdownAction": "stopCompose", - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, + // "runArgs": [ + // "--cap-add=SYS_PTRACE", + // "--security-opt", + // "seccomp=unconfined", + // "--network", + // "rasterizer_net_backend" + // ], "build": { + // "dockerfile": "docker-compose.yml", + "dockerComposeFile": "docker-compose.yml", + "service": "app", "args": { "NUM_CPU": "8", "FLINT_BRANCH": "develop", @@ -21,32 +25,49 @@ } }, "containerEnv": { - "GOOGLE_APPLICATION_CREDENTIALS": "/workspaces/moja.mulliongroup_develop/sensitive/datalayer.json" + // "GOOGLE_APPLICATION_CREDENTIALS": "/workspaces/moja.mulliongroup_develop/sensitive/datalayer.json" }, + "mounts": [ + // "source=${localWorkspaceFolder}/Run_Envs/bashhistory,target=/commandhistory,type=bind,consistency=cached" + ], + + // Default path to open when attaching to a new container. + // "workspaceFolder": "/workspaces/moja.mulliongroup_develop", + // "workspaceFolder": "${containerWorkspaceFolder}", + "workspaceFolder": "/workspaces", - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-vscode.cpptools", - "austin.code-gnu-global", - "twxs.cmake", - "ms-vscode.cmake-tools" - ] + // Sets the run context to one level up instead of the .devcontainer folder. + // "context": "..", + + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + // "dockerFile": "../Build/DockerFile.moja.global.ubuntu", + // "dockerFile": "../Build/DockerFile.moja.global.ubuntu.18.04", - "workspaceFolder": "${containerWorkspaceFolder}", + // The optional 'runArgs' property can be used to specify additional runtime arguments. + // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + // "runArgs": [ + // // Uncomment the next line if you want to use Docker from the container. See the docker-in-docker definition for details. + // // "-v","/var/run/docker.sock:/var/run/docker.sock", - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // // Uncomment the next line if you will be using a ptrace-based debugger like C++, Go, and Rust. + // "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" + // ], - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "gcc -v", + // Uncomment the next line if you want to publish any ports. + // "appPort": [], + + // Uncomment the next line if you want to add in default container specific settings.json values + // "settings": { "workbench.colorTheme": "Quiet Light" }, - // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" + // Uncomment the next line to run commands after the container is created. + // "postCreateCommand": "uname -a", + // Add the IDs of any extensions you want installed in the array below. "extensions": [ "ms-vscode.cpptools", "austin.code-gnu-global", "twxs.cmake", - "ms-vscode.cmake-tools" + "ms-vscode.cmake-tools", + "mutantdino.resourcemonitor" ] } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 0000000..7a396a2 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,96 @@ +version: '3' + +services: + app: + # Using a Dockerfile is optional, but included for completeness. + security_opt: + - seccomp:unconfined + cap_add: + - SYS_PTRACE + build: + context: . + dockerfile: Dockerfile + # [Optional] You can use build args to set options. e.g. 'VARIANT' below affects the image in the Dockerfile + args: + NUM_CPU: 8 + FLINT_BRANCH: develop + # FLINT_BRANCH: feature-collection-updates + + volumes: + # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json + # - ..:/workspaces:cached + - ..:/workspaces:delegated + + # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details. + # - /var/run/docker.sock:/var/run/docker.sock + + environment: + - GOOGLE_APPLICATION_CREDENTIALS=/workspaces/sensitive/datalayer.json + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "while sleep 1000; do :; done" + + # Runs app on the same network as the service container, allows "forwardPorts" in devcontainer.json function. + # network_mode: service:rasterizer_net_backend + # network_mode: service:another-service + networks: + - net_backend + + # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. + # (Adding the "ports" property to this file will not forward from a Codespace.) + + # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details. + # user: vscode + + # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + +# flintpro-rasterizer: +# #image: gcr.io/flintpro-212105/flintpro-rasterizer:v20201015.03 +# # Thsi version is the new client side rasterizer +# image: gcr.io/flintpro-212105/flintpro-rasterizer:v20210525.01 +# hostname: flintpro-rasterizer +# ports: +# - 50051:50051 +# depends_on: +# - cloudsql +# command: '/run/rasterizer_server -p proxy:9090' +# restart: unless-stopped +# networks: +# - net_backend + +# proxy: +# image: gcr.io/flintpro-212105/proxy:v20200507.1 +# hostname: proxy +# # your own configurations for that app +# ports: +# - 9090:9090 +# depends_on: +# - cloudsql +# command: ["./wait-for-it.sh", "cloudsql:5432", "--", "/go/bin/featureservice-proxy", "-grpc-port", "9090", "-lookupConfigFile", "./proxy-config/lookup_config.json"] +# volumes: +# - ../Run_Envs/rasterizer/proxy-config:/proxy-config +# restart: unless-stopped +# networks: +# - net_backend + +# cloudsql: +# image: gcr.io/cloudsql-docker/gce-proxy:1.22.0 +# networks: +# - net_backend +# command: /cloud_sql_proxy -instances=flintpro-212105:us-central1:flintpro-qa-3=tcp:0.0.0.0:5432,flintpro-212105:us-central1:flintpro-prod=tcp:0.0.0.0:5433 -credential_file=/config/datalayer.json +# volumes: +# - ../sensitive:/config +# # - /mnt/c/Development/MullionGroup/cloud_sql:/config +# # - c:\Development\MullionGroup\cloud_sql:/config +# ports: +# - 5432:5432 +# restart: unless-stopped + +networks: + net_backend: + + # As in the "app" service, use "forwardPorts" in **devcontainer.json** to forward an app port locally. \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index f063712..008e712 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,12 +4,15 @@ "-DDEBUG=ON", "-DCMAKE_INSTALL_PREFIX=/usr/local", "-DENABLE_TESTS:BOOL=OFF", - "-DENABLE_MOJA.MODULES.GDAL=ON", + "-DENABLE_MOJA.MODULES.GDAL=OFF", "-DENABLE_MOJA.MODULES.LIBPQ=ON", "-DBoost_USE_STATIC_LIBS=OFF", "-DBUILD_SHARED_LIBS=ON", - "-DCMAKE_TOOLCHAIN_FILE=/usr/local/src/vcpkg/scripts/buildsystems/vcpkg.cmake" + "-DCMAKE_TOOLCHAIN_FILE=/usr/src/vcpkg/scripts/buildsystems/vcpkg.cmake" ], "cmake.sourceDirectory": "${workspaceFolder}/Source", - "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild" + "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild", + "files.associations": { + "iterator": "cpp" + } } \ No newline at end of file From bae1ed5e69c16f37a10e1c92488c60bf33be27a2 Mon Sep 17 00:00:00 2001 From: Jim Date: Mon, 31 May 2021 15:36:22 +1000 Subject: [PATCH 32/40] dev container + keeps config build folders separate + helps with build speed --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 008e712..dfde3a4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,7 +11,7 @@ "-DCMAKE_TOOLCHAIN_FILE=/usr/src/vcpkg/scripts/buildsystems/vcpkg.cmake" ], "cmake.sourceDirectory": "${workspaceFolder}/Source", - "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild", + "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild/${buildKit}/${buildType}", "files.associations": { "iterator": "cpp" } From ca1e5f44798569231ac8e63c799c129843bd0420 Mon Sep 17 00:00:00 2001 From: Jim Date: Mon, 31 May 2021 22:10:02 +1000 Subject: [PATCH 33/40] More cleanup of OutputerStream and OutputerStreamFlux + combined multiple output methods into single + improved flux col header names --- .../include/moja/flint/outputerstream.h | 3 +- .../include/moja/flint/outputerstreamflux.h | 3 +- Source/moja.flint/src/outputerstream.cpp | 70 ++++++------------- Source/moja.flint/src/outputerstreamflux.cpp | 63 +++++------------ 4 files changed, 41 insertions(+), 98 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/outputerstream.h b/Source/moja.flint/include/moja/flint/outputerstream.h index 9a9a533..c2a3b78 100644 --- a/Source/moja.flint/include/moja/flint/outputerstream.h +++ b/Source/moja.flint/include/moja/flint/outputerstream.h @@ -27,8 +27,7 @@ class FLINT_API OutputerStream : public ModuleBase { virtual void subscribe(NotificationCenter& notificationCenter) override; virtual void outputHeader(std::ostream& stream) const; - virtual void outputInit(std::ostream& stream); - virtual void outputEndStep(const std::string& notification, std::ostream& stream); + virtual void outputOnNotification(const std::string& notification, std::ostream& stream); virtual void outputShutdown(std::ostream& stream); void onSystemInit() override; diff --git a/Source/moja.flint/include/moja/flint/outputerstreamflux.h b/Source/moja.flint/include/moja/flint/outputerstreamflux.h index f3d891d..4d24c3b 100644 --- a/Source/moja.flint/include/moja/flint/outputerstreamflux.h +++ b/Source/moja.flint/include/moja/flint/outputerstreamflux.h @@ -16,8 +16,7 @@ class FLINT_API OutputerStreamFlux : public ModuleBase { explicit OutputerStreamFlux() : ModuleBase(), _outputToScreen(false), _outputInfoHeader(false) {} virtual ~OutputerStreamFlux() {} - void outputInit(std::ostream& stream) const; - void outputEndStep(std::ostream& stream) const; + void outputOnNotification(const std::string& notification, std::ostream& stream) const; void outputShutdown(std::ostream& stream) const; void outputHeader(std::ostream& stream) const; diff --git a/Source/moja.flint/src/outputerstream.cpp b/Source/moja.flint/src/outputerstream.cpp index e4c5215..4f081bd 100644 --- a/Source/moja.flint/src/outputerstream.cpp +++ b/Source/moja.flint/src/outputerstream.cpp @@ -142,12 +142,16 @@ void OutputerStream::outputHeader(std::ostream& stream) const { // -------------------------------------------------------------------------------------------- -void OutputerStream::outputInit(std::ostream& stream) { +void OutputerStream::outputOnNotification(const std::string& notification, std::ostream& stream) { const auto& timingL = *_landUnitData->timing(); - stream << "onTimingPostInit" << DL_CHR << timingL.step() << DL_CHR << timingL.curEndDate().addMicroseconds(-1) - << DL_CHR << timingL.fractionOfStep() << DL_CHR << timingL.stepLengthInYears() << DL_CHR; - stream << std::setprecision(STOCK_PRECISION); + stream << notification << DL_CHR << + timingL.step() << DL_CHR << + // timingL.curEndDate().addMicroseconds(-1) << DL_CHR << + timingL.curStartDate() << DL_CHR << + timingL.fractionOfStep() << DL_CHR << + timingL.stepLengthInYears() << DL_CHR << + std::setprecision(STOCK_PRECISION); auto pools = _landUnitData->poolCollection(); for (auto& it : pools) { @@ -185,47 +189,6 @@ void OutputerStream::outputInit(std::ostream& stream) { // -------------------------------------------------------------------------------------------- -void OutputerStream::outputEndStep(const std::string& notification, std::ostream& stream) { - const auto& timingL = *_landUnitData->timing(); - stream << notification << DL_CHR << timingL.step() << DL_CHR << timingL.curEndDate().addMicroseconds(-1) << DL_CHR - << timingL.fractionOfStep() << DL_CHR << timingL.stepLengthInYears() << DL_CHR; - stream << std::setprecision(STOCK_PRECISION); - auto pools = _landUnitData->poolCollection(); - for (auto& it : pools) { - stream << it->value() << DL_CHR; - } - for (auto var : _variables) { - auto varPtr = std::get<3>(var); - auto varName = std::get<1>(var); - auto varProp = std::get<2>(var); - - if (varPtr == nullptr) { - stream << "(missing variable)"; - } else if (varProp != "") { - auto varValue = varPtr->value(); - if (varValue.type() == typeid(std::shared_ptr)) { - auto flintDataVariable = varPtr->value().extract>(); - auto propValue = flintDataVariable->getProperty(varProp); - outputDynamicToStream(stream, propValue); - } else { - if (varValue.isStruct()) { - auto varStruct = varValue.extract(); - auto varStructProp = varStruct.contains(varProp) ? varStruct[varProp] : DynamicVar(); - outputDynamicToStream(stream, varStructProp); - } else - outputDynamicToStream(stream, varValue); - } - } else { - // TODO: extend this to do property of array/struct objects - outputDynamicToStream(stream, varPtr->value()); - } - stream << DL_CHR; - } - stream << std::endl; -} - -// -------------------------------------------------------------------------------------------- - void OutputerStream::outputShutdown(std::ostream& stream) { using namespace std::chrono; @@ -268,28 +231,35 @@ void OutputerStream::onLocalDomainInit() { // -------------------------------------------------------------------------------------------- -void OutputerStream::onTimingPostInit() { outputInit(_output); } +void OutputerStream::onTimingPostInit() { + outputOnNotification("onTimingPostInit", _output); +} // -------------------------------------------------------------------------------------------- void OutputerStream::onOutputStep() { if (_outputOnOutputStep) { const auto& timingL = *_landUnitData->timing(); - - if (!_outputAnnually || timingL.curStartDate().month() == 12) outputEndStep("onOutputStep", _output); + if (!_outputAnnually || timingL.curStartDate().month() == 12) { + outputOnNotification("onOutputStep", _output); + } } } // -------------------------------------------------------------------------------------------- void OutputerStream::onTimingEndStep() { - if (_outputOnTimingEndStep) outputEndStep("onTimingEndStep", _output); + if (_outputOnTimingEndStep) { + outputOnNotification("onTimingEndStep", _output); + } } // -------------------------------------------------------------------------------------------- void OutputerStream::onPostDisturbanceEvent() { - if (_outputOnPostDisturbanceEvent) outputEndStep("onPostDisturbanceEvent", _output); + if (_outputOnPostDisturbanceEvent) { + outputOnNotification("onPostDisturbanceEvent", _output); + } } } // namespace flint diff --git a/Source/moja.flint/src/outputerstreamflux.cpp b/Source/moja.flint/src/outputerstreamflux.cpp index ff0c756..3b895a8 100644 --- a/Source/moja.flint/src/outputerstreamflux.cpp +++ b/Source/moja.flint/src/outputerstreamflux.cpp @@ -52,45 +52,19 @@ void OutputerStreamFlux::outputHeader(std::ostream& stream) const { stream << "Started:" << start << std::endl; stream << "==========================================================================" << std::endl; } - stream << "step" << DL_CHR << "step date" << DL_CHR << "module name" << DL_CHR << "disturbance_type" << DL_CHR - << "source pool" DL_CHR << "sink pool" << DL_CHR << "value" << std::endl; + stream << + "step" << DL_CHR << + "step_date" << DL_CHR << + "module_name" << DL_CHR << + "disturbance_type" << DL_CHR << + "source_pool" << DL_CHR << + "sink_pool" << DL_CHR << + "value" << std::endl; } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::outputInit(std::ostream& stream) const { - const auto timingL = _landUnitData->timing(); - - for (auto operationResult : _landUnitData->getOperationLastAppliedIterator()) { - const auto mdata = operationResult->metaData(); - for (auto f : operationResult->operationResultFluxCollection()) { - const auto srcIx = f->source(); - const auto dstIx = f->sink(); - if (srcIx == dstIx) // don't process diagonal - continue; - - const auto val = f->value(); - const auto srcPool = _landUnitData->getPool(srcIx); - const auto dstPool = _landUnitData->getPool(dstIx); - - auto fluxTypeInfoRecordId = flint::FluxType::Unclassified; - if (operationResult->hasDataPackage()) { - auto dataPacket = operationResult->dataPackage().extract>(); - fluxTypeInfoRecordId = dataPacket->_fluxType; - } - - stream << timingL->step() << DL_CHR << timingL->curStartDate() << DL_CHR; - stream << mdata->moduleName << DL_CHR; - stream << int(fluxTypeInfoRecordId) << DL_CHR; - stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val - << std::endl; - } - } -} - -// -------------------------------------------------------------------------------------------- - -void OutputerStreamFlux::outputEndStep(std::ostream& stream) const { +void OutputerStreamFlux::outputOnNotification(const std::string& notification, std::ostream& stream) const { const auto& timingL = _landUnitData->timing(); for (auto operationResult : _landUnitData->getOperationLastAppliedIterator()) { const auto mdata = operationResult->metaData(); @@ -110,12 +84,13 @@ void OutputerStreamFlux::outputEndStep(std::ostream& stream) const { fluxTypeInfoRecordId = dataPacket->_fluxType; } - stream << timingL->step() << DL_CHR; - stream << timingL->curStartDate() << DL_CHR; - stream << mdata->moduleName << DL_CHR; - stream << int(fluxTypeInfoRecordId) << DL_CHR; - stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val - << std::endl; + stream << timingL->step() << DL_CHR << + timingL->curStartDate() << DL_CHR << + mdata->moduleName << DL_CHR << + int(fluxTypeInfoRecordId) << DL_CHR << + srcPool->name() << DL_CHR << + dstPool->name() << DL_CHR << + std::setprecision(STOCK_PRECISION) << val << std::endl; } } } @@ -153,15 +128,15 @@ void OutputerStreamFlux::onSystemShutdown() { // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onTimingPostInit() { outputInit(_output); } +void OutputerStreamFlux::onTimingPostInit() { outputOnNotification("onTimingPostInit", _output); } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onTimingEndStep() { outputEndStep(_output); } +void OutputerStreamFlux::onTimingEndStep() { outputOnNotification("onTimingEndStep", _output); } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onPostDisturbanceEvent() { outputEndStep(_output); } +void OutputerStreamFlux::onPostDisturbanceEvent() { outputOnNotification("onPostDisturbanceEvent", _output); } } // namespace flint } // namespace moja From 8f286bb6e053a08cc13b188bf5d2a0aa037e6bda Mon Sep 17 00:00:00 2001 From: Mal Date: Mon, 7 Jun 2021 11:26:15 +1000 Subject: [PATCH 34/40] More removal of Poco for std::filesystem --- Source/moja.core/src/environment_unix.cpp | 10 ++--- Source/moja.core/src/environment_win32.cpp | 24 ++++++------ Source/moja.core/src/environment_win32u.cpp | 22 +++++------ Source/moja.core/src/logging.cpp | 41 +++++++++++---------- 4 files changed, 49 insertions(+), 48 deletions(-) diff --git a/Source/moja.core/src/environment_unix.cpp b/Source/moja.core/src/environment_unix.cpp index b4b5cba..63f85f0 100644 --- a/Source/moja.core/src/environment_unix.cpp +++ b/Source/moja.core/src/environment_unix.cpp @@ -1,14 +1,15 @@ #include "moja/environment_unix.h" -#include - #include #include +#include #include namespace moja { std::string EnvironmentImpl::startProcessFolderImpl() { + namespace fs = std::filesystem; + static bool bHaveResult = false; static char path[1024] = ""; @@ -20,9 +21,8 @@ std::string EnvironmentImpl::startProcessFolderImpl() { } bHaveResult = true; } - auto folder = Poco::Path(path).parent(); - return folder.toString(); - ; + fs::path path(path); + return path.remove_filename().string(); } } // namespace moja diff --git a/Source/moja.core/src/environment_win32.cpp b/Source/moja.core/src/environment_win32.cpp index adc67fc..ad4725d 100644 --- a/Source/moja.core/src/environment_win32.cpp +++ b/Source/moja.core/src/environment_win32.cpp @@ -1,23 +1,23 @@ #include "moja/environment_win32.h" -#include -#include - #include +#include #include -#pragma comment(lib, "psapi.lib") - namespace moja { std::string EnvironmentImpl::startProcessFolderImpl() { - static char path[512] = ""; - if (!path[0]) { - // Get directory this executable was launched from. - GetModuleFileNameA(NULL, path, sizeof(path) - 1); - } - auto folder = Poco::Path(path).parent(); - return folder.toString(); + namespace fs = std::filesystem; + + std::vector charBuffer; + auto size = MAX_PATH; + do { + size *= 2; + charBuffer.resize(size); + } while (GetModuleFileNameA(NULL, charBuffer.data(), size) == size); + + fs::path path(charBuffer.data()); // Contains the full path including .exe + return path.remove_filename().string(); } } // namespace moja diff --git a/Source/moja.core/src/environment_win32u.cpp b/Source/moja.core/src/environment_win32u.cpp index 15956aa..d9c6967 100644 --- a/Source/moja.core/src/environment_win32u.cpp +++ b/Source/moja.core/src/environment_win32u.cpp @@ -1,23 +1,23 @@ #include "moja/environment_win32u.h" -#include -#include -#include - #include +#include #include namespace moja { std::string EnvironmentImpl::startProcessFolderImpl() { + namespace fs = std::filesystem; + + std::vector charBuffer; + auto size = MAX_PATH; + do { + size *= 2; + charBuffer.resize(size); + } while (GetModuleFileNameW(NULL, charBuffer.data(), size) == size); - std::wstring mod_path; - mod_path.resize(MAX_PATH); - mod_path.resize(::GetModuleFileNameW(NULL, &mod_path[0], mod_path.size())); - std::string result; - Poco::UnicodeConverter::toUTF8(mod_path, result); - auto folder = Poco::Path(result).parent().absolute(); - return folder.toString(); + fs::path path(charBuffer.data()); // Contains the full path including .exe + return path.remove_filename().string(); } } // namespace moja diff --git a/Source/moja.core/src/logging.cpp b/Source/moja.core/src/logging.cpp index ce20133..d6d3302 100644 --- a/Source/moja.core/src/logging.cpp +++ b/Source/moja.core/src/logging.cpp @@ -2,14 +2,12 @@ #include "moja/environment.h" -#include -#include - #include #include #include #include +#include #include namespace moja { @@ -17,13 +15,14 @@ namespace moja { const std::string Logging::_defaultFileName = "logging.conf"; bool Logging::_explicitConfigurationFileSet = false; bool Logging::_explicitConfigurationTextSet = false; -std::string Logging::_explicitConfigurationFile = ""; -std::string Logging::_explicitConfigurationText = ""; +std::string Logging::_explicitConfigurationFile; +std::string Logging::_explicitConfigurationText; std::string Logging::_loggingConfigurationFile = "unknown"; void Logging::init() { namespace logging = boost::log; namespace attrs = boost::log::attributes; + namespace fs = std::filesystem; static auto initialized = false; if (initialized) { @@ -34,7 +33,8 @@ void Logging::init() { logging::register_simple_filter_factory("Severity"); // Determine which log config to load, searching the working folder then the exe folder. - if (_explicitConfigurationFileSet && Poco::File(_explicitConfigurationFile).exists()) { + fs::path explicit_configuration_file(_explicitConfigurationFile); + if (_explicitConfigurationFileSet && fs::exists(explicit_configuration_file)) { std::ifstream loggingConfig(_explicitConfigurationFile); logging::init_from_stream(loggingConfig); _loggingConfigurationFile = _explicitConfigurationFile; @@ -44,25 +44,26 @@ void Logging::init() { boost::log::init_from_stream(s); _loggingConfigurationFile = "internal text"; } else { - std::string filenameToCheck = Poco::Path::current() + _defaultFileName; - if (Poco::File(filenameToCheck).exists()) { - std::ifstream loggingConfig(filenameToCheck); + auto current_path = fs::current_path().append(_defaultFileName); + if (fs::exists(current_path)) { + std::ifstream loggingConfig(current_path); logging::init_from_stream(loggingConfig); - _loggingConfigurationFile = filenameToCheck; + _loggingConfigurationFile = current_path.string(); } else { - filenameToCheck = moja::Environment::startProcessFolder() + _defaultFileName; - if (Poco::File(filenameToCheck).exists()) { - std::ifstream loggingConfig(filenameToCheck); + auto start_path = fs::path(Environment::startProcessFolder() + _defaultFileName); + if (fs::exists(start_path)) { + std::ifstream loggingConfig(start_path); logging::init_from_stream(loggingConfig); - _loggingConfigurationFile = filenameToCheck; + _loggingConfigurationFile = start_path.string(); } else { std::stringstream s; - s << "[Sinks.console]" << std::endl; - s << "Destination=Console" << std::endl; - s << "Asynchronous = false" << std::endl; - s << "AutoFlush = true" << std::endl; - s << "Format = \"<%TimeStamp%> (%Severity%) - %Message%\"" << std::endl; - s << "Filter = \"%Severity% >= info\"" << std::endl; + s << R"([Sinks.console] + Destination=Console + Asynchronous = false + AutoFlush = true + Format = "<%TimeStamp%> (%Severity%) - %Message%" + Filter = "%Severity% >= info")" + << std::endl; boost::log::init_from_stream(s); _loggingConfigurationFile = "internal default"; } From 34e8687d1f78a7a3d3d7211a1671384db099c7ab Mon Sep 17 00:00:00 2001 From: Mal Date: Mon, 7 Jun 2021 13:57:50 +1000 Subject: [PATCH 35/40] Fixed Linux compile issue --- Source/moja.core/src/environment_unix.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/moja.core/src/environment_unix.cpp b/Source/moja.core/src/environment_unix.cpp index 63f85f0..471bc87 100644 --- a/Source/moja.core/src/environment_unix.cpp +++ b/Source/moja.core/src/environment_unix.cpp @@ -11,17 +11,17 @@ std::string EnvironmentImpl::startProcessFolderImpl() { namespace fs = std::filesystem; static bool bHaveResult = false; - static char path[1024] = ""; + static char charBuffer[1024] = ""; if (!bHaveResult) { - if (readlink("/proc/self/exe", path, sizeof(path) - 1) == -1) { + if (readlink("/proc/self/exe", charBuffer, sizeof(charBuffer) - 1) == -1) { int ErrNo = errno; // unreachable return ""; } bHaveResult = true; } - fs::path path(path); + fs::path path(charBuffer); return path.remove_filename().string(); } From 663cee5a7be948678dc5e9ecf401c08d8c097cd6 Mon Sep 17 00:00:00 2001 From: Mal Date: Mon, 7 Jun 2021 14:04:27 +1000 Subject: [PATCH 36/40] Another linux fix for std filesystem --- Source/CMakeLists.txt | 4 ++++ Source/moja.core/CMakeLists.txt | 1 + 2 files changed, 5 insertions(+) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index e4b9747..7f025ca 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -127,6 +127,10 @@ if (CMAKE_SYSTEM MATCHES "Linux" ) else() message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++17 support. Please use a different C++ compiler.") endif() + + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(CXX_FILESYSTEM_LIBRARIES "stdc++fs") + endif() endif(CMAKE_SYSTEM MATCHES "Linux" ) add_subdirectory(moja.core) diff --git a/Source/moja.core/CMakeLists.txt b/Source/moja.core/CMakeLists.txt index d024677..04f8258 100644 --- a/Source/moja.core/CMakeLists.txt +++ b/Source/moja.core/CMakeLists.txt @@ -148,6 +148,7 @@ target_compile_definitions(${LIBNAME} target_link_libraries(${LIBNAME} PUBLIC Boost::log Boost::log_setup Poco::Foundation Poco::JSON + ${CXX_FILESYSTEM_LIBRARIES} PRIVATE ${SYSLIBS} ) From c0dd260f9ec9c2bec81738a5b957b893e77c815e Mon Sep 17 00:00:00 2001 From: Mal Date: Wed, 16 Jun 2021 12:37:39 +1000 Subject: [PATCH 37/40] Cleaned up exceptions and move to std::filesystem --- Source/moja.cli/src/moja.cpp | 42 ++- Source/moja.core/CMakeLists.txt | 7 +- .../moja.core/include/moja/coreexceptions.h | 29 --- Source/moja.core/include/moja/exception.h | 243 ------------------ Source/moja.core/include/moja/filesystem.h | 14 + Source/moja.core/src/datetime.cpp | 2 - Source/moja.core/src/exception.cpp | 114 -------- Source/moja.core/src/filesystem.cpp | 11 + Source/moja.core/src/logging.cpp | 4 +- Source/moja.core/src/pocojsonutils.cpp | 79 ++---- .../datarepository/rasterreaderinterface.h | 15 +- .../src/providerrelationalsqlite.cpp | 8 +- .../moja.datarepository/src/rasterreader.cpp | 24 +- .../moja.flint.configuration/CMakeLists.txt | 1 - .../configuration/configurationexceptions.h | 55 ---- .../src/configblock.cpp | 9 +- .../src/configcell.cpp | 5 +- .../src/configtile.cpp | 9 +- .../src/configuration.cpp | 8 +- .../src/json2configurationprovider.cpp | 50 ++-- .../moja.flint.configuration/src/provider.cpp | 13 +- .../moja.flint.configuration/src/spinup.cpp | 5 +- .../include/moja/flint/flintexceptions.h | 35 --- .../moja/flint/operationmanagersimplecache.h | 6 +- .../include/moja/flint/writesystemconfig.h | 16 -- .../include/moja/flint/writevariablegrid.h | 16 -- .../moja.flint/src/aggregatorfilewriter.cpp | 24 +- .../src/aspatiallocaldomaincontroller.cpp | 5 - Source/moja.flint/src/compositetransform.cpp | 17 +- Source/moja.flint/src/externalvariable.cpp | 7 +- Source/moja.flint/src/landunitcontroller.cpp | 6 +- Source/moja.flint/src/librarymanager.cpp | 53 ++-- .../src/localdomaincontrollerbase.cpp | 15 +- .../moja.flint/src/lookuprandomtransform.cpp | 14 +- Source/moja.flint/src/lookuptransform.cpp | 17 +- .../moja.flint/src/operationmanagersimple.cpp | 9 +- .../src/operationmanagersimplecache.cpp | 6 +- Source/moja.flint/src/outputerstream.cpp | 12 +- Source/moja.flint/src/outputerstreamflux.cpp | 10 +- Source/moja.flint/src/poolsimplecache.cpp | 2 +- .../src/spatialtiledlocaldomaincontroller.cpp | 166 +----------- .../src/spinuplandunitcontroller.cpp | 4 +- Source/moja.flint/src/sqlquerytransform.cpp | 11 +- Source/moja.flint/src/writesystemconfig.cpp | 81 +++--- Source/moja.flint/src/writevariablegrid.cpp | 138 ++++------ .../src/rasterreadergdal.cpp | 7 +- .../src/writevariablegeotiff.cpp | 144 ++++------- 47 files changed, 378 insertions(+), 1190 deletions(-) delete mode 100644 Source/moja.core/include/moja/coreexceptions.h delete mode 100644 Source/moja.core/include/moja/exception.h create mode 100644 Source/moja.core/include/moja/filesystem.h delete mode 100644 Source/moja.core/src/exception.cpp create mode 100644 Source/moja.core/src/filesystem.cpp delete mode 100644 Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h diff --git a/Source/moja.cli/src/moja.cpp b/Source/moja.cli/src/moja.cpp index e148291..73ad5c8 100644 --- a/Source/moja.cli/src/moja.cpp +++ b/Source/moja.cli/src/moja.cpp @@ -9,15 +9,12 @@ #include "moja/flint/moduleproxybase.h" #include "moja/flint/ioperationmanager.h" -#include "moja/signals.h" -#include "moja/exception.h" -#include "moja/logging.h" +#include +#include +#include -#include #include -#include - #include #include #include @@ -32,9 +29,10 @@ namespace conf = mf::configuration; using mf::configuration::LocalDomainType; using mf::ILocalDomainController; +namespace fs = moja::filesystem; + bool checkFilePath(const std::string& filePath) { - Poco::File file(filePath); - if (!file.exists()) { + if (!fs::exists(filePath)) { std::cerr << "File not found: " << filePath; return false; } @@ -295,23 +293,13 @@ int main(int argc, char* argv[]) { ldc->run(); ldc->shutdown(); ldc->_notificationCenter.postNotification(moja::signals::SystemShutdown); - } - catch (const moja::Exception& e) { - MOJA_LOG_FATAL << e.displayText(); - return EXIT_FAILURE; - } catch (const boost::exception& e) { - MOJA_LOG_FATAL << boost::diagnostic_information(e); - return EXIT_FAILURE; - } catch (const Poco::Exception& e) { - MOJA_LOG_FATAL << e.message(); - return EXIT_FAILURE; - } catch (const std::exception& e) { - MOJA_LOG_FATAL << e.what(); - return EXIT_FAILURE; - } catch (...) { - MOJA_LOG_FATAL << "Unknown exception"; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; + } catch (const std::exception& e) { + MOJA_LOG_FATAL << e.what(); + return EXIT_FAILURE; + } catch (...) { + MOJA_LOG_FATAL << "Unknown exception"; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; } diff --git a/Source/moja.core/CMakeLists.txt b/Source/moja.core/CMakeLists.txt index 04f8258..e63cb9e 100644 --- a/Source/moja.core/CMakeLists.txt +++ b/Source/moja.core/CMakeLists.txt @@ -6,6 +6,7 @@ include(${CMAKE_MODULE_PATH}/generate_product_version.cmake) find_package(Poco REQUIRED Foundation JSON) find_package(Boost COMPONENTS log log_setup REQUIRED) +find_package(fmt REQUIRED) if (MSVC) generate_product_version(ProductVersionFiles @@ -83,7 +84,7 @@ set(MOJA_Core_headers include/moja/dispatcher.h include/moja/dynamic.h include/moja/environment.h - include/moja/exception.h + include/moja/filesystem.h include/moja/floatcmp.h include/moja/hash.h include/moja/logging.h @@ -98,13 +99,12 @@ set(MOJA_Core_headers include/moja/version.h include/moja/publisher.h include/moja/utility.h - include/moja/coreexceptions.h ) set(MOJA_Core_sources src/datetime.cpp src/environment.cpp - src/exception.cpp + src/filesystem.cpp src/floatcmp.cpp src/logging.cpp src/mathex.cpp @@ -149,6 +149,7 @@ target_link_libraries(${LIBNAME} PUBLIC Boost::log Boost::log_setup Poco::Foundation Poco::JSON ${CXX_FILESYSTEM_LIBRARIES} + fmt::fmt-header-only PRIVATE ${SYSLIBS} ) diff --git a/Source/moja.core/include/moja/coreexceptions.h b/Source/moja.core/include/moja/coreexceptions.h deleted file mode 100644 index 3bb63aa..0000000 --- a/Source/moja.core/include/moja/coreexceptions.h +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef MOJA_CORE_COREEXCEPTIONS_H_ -#define MOJA_CORE_COREEXCEPTIONS_H_ - -#include "_core_exports.h" - -#include - -#include -#include - -namespace moja { - -struct CORE_API CoreException : virtual std::exception, virtual boost::exception {}; - -struct CORE_API CoreAssertionViolationException : virtual CoreException {}; -typedef boost::error_info AssertMsg; - -struct CORE_API CoreFileNotFoundException : virtual CoreException {}; -typedef boost::error_info FileName; - -struct CORE_API CoreNotImplementedException : virtual CoreException {}; -typedef boost::error_info Message; - -struct CORE_API CoreUnhandledType : virtual CoreException {}; -typedef boost::error_info TypeErrorMsg; - -} // namespace moja - -#endif // MOJA_CORE_COREEXCEPTIONS_H_ \ No newline at end of file diff --git a/Source/moja.core/include/moja/exception.h b/Source/moja.core/include/moja/exception.h deleted file mode 100644 index 4e1e654..0000000 --- a/Source/moja.core/include/moja/exception.h +++ /dev/null @@ -1,243 +0,0 @@ -#ifndef MOJA_CORE_EXCEPTION_H_ -#define MOJA_CORE_EXCEPTION_H_ - -#include "_core_exports.h" - -#include - -namespace moja { - -/** - * - * This is the base class for all exceptions defined in the moja Core class library. - * - */ -class CORE_API Exception : public std::exception { - public: - /** - * Creates an exception. - * - * The message. - * (Optional) the code. - */ - Exception(const std::string& msg, int code = 0); - - /** - * Creates an exception. - * - * The message. - * The extended message. - * (Optional) the code. - */ - Exception(const std::string& msg, const std::string& arg, int code = 0); - - /** - * Creates an exception and stores a clone of the nested exception. - * - * The message. - * The nested. - * (Optional) the code. - */ - Exception(const std::string& msg, const Exception& nested, int code = 0); - - /** - * Copy constructor. - */ - Exception(const Exception& exc); - - /** Destroys the exception and deletes the nested exception. */ - ~Exception() throw(); - - /** - * Assignment operator. - */ - Exception& operator=(const Exception& exc); - - /** - * Returns a static string describing the exception. - */ - virtual const char* name() const throw(); - - /** - * Returns the name of the exception class. - */ - virtual const char* className() const throw(); - - /** - * - * Returns a static string describing the exception. - * - * Same as name(), but for compatibility with std::exception. - * - */ - virtual const char* what() const throw(); - - /** - * - * Returns a pointer to the nested exception, or null if no nested exception exists. - * - */ - const Exception* nested() const; - - /** - * Returns the message text. - */ - const std::string& message() const; - - /** - * Returns the exception code if defined. - */ - int code() const; - - /** - * Returns a string consisting of the message name and the message text. - */ - std::string displayText() const; - - /** - * - * Creates an exact copy of the exception. - * - * The copy can later be thrown again by invoking rethrow() on it. - * - */ - virtual Exception* clone() const; - - /** - * - * (Re)Throws the exception. - * - * This is useful for temporarily storing a copy of an exception (see clone()), then throwing it - * again. - * - */ - virtual void rethrow() const; - - protected: - /** - * Standard constructor. - * - * (Optional) the code. - */ - Exception(int code = 0); - - /** - * Sets the message for the exception. - * - * The message. - */ - void message(const std::string& msg); - - /** - * Sets the extended message for the exception. - * - * The extended message. - */ - void extendedMessage(const std::string& arg); - - private: - std::string _msg; - Exception* _pNested; - int _code; -}; - -inline const Exception* Exception::nested() const { return _pNested; } - -inline const std::string& Exception::message() const { return _msg; } - -inline void Exception::message(const std::string& msg) { _msg = msg; } - -inline int Exception::code() const { return _code; } - -// Macros for quickly declaring and implementing exception classes. -// Unfortunately, we cannot use a template here because character -// pointers (which we need for specifying the exception name) -// are not allowed as template arguments. -#define MOJA_DECLARE_EXCEPTION_CODE(API, CLS, BASE, CODE) \ - class API CLS : public BASE { \ - public: \ - CLS(int code = CODE); \ - CLS(const std::string& msg, int code = CODE); \ - CLS(const std::string& msg, const std::string& arg, int code = CODE); \ - CLS(const std::string& msg, const moja::Exception& exc, int code = CODE); \ - CLS(const CLS& exc); \ - ~CLS() throw(); \ - CLS& operator=(const CLS& exc); \ - const char* name() const throw(); \ - const char* className() const throw(); \ - moja::Exception* clone() const; \ - void rethrow() const; \ - }; - -#define MOJA_DECLARE_EXCEPTION(API, CLS, BASE) MOJA_DECLARE_EXCEPTION_CODE(API, CLS, BASE, 0) - -#define MOJA_IMPLEMENT_EXCEPTION(CLS, BASE, NAME) \ - CLS::CLS(int code) : BASE(code) {} \ - CLS::CLS(const std::string& msg, int code) : BASE(msg, code) {} \ - CLS::CLS(const std::string& msg, const std::string& arg, int code) : BASE(msg, arg, code) {} \ - CLS::CLS(const std::string& msg, const moja::Exception& exc, int code) : BASE(msg, exc, code) {} \ - CLS::CLS(const CLS& exc) : BASE(exc) {} \ - CLS::~CLS() throw() {} \ - CLS& CLS::operator=(const CLS& exc) { \ - BASE::operator=(exc); \ - return *this; \ - } \ - const char* CLS::name() const throw() { return NAME; } \ - const char* CLS::className() const throw() { return typeid(*this).name(); } \ - moja::Exception* CLS::clone() const { return new CLS(*this); } \ - void CLS::rethrow() const { throw *this; } - -// Standard exception classes. -MOJA_DECLARE_EXCEPTION(CORE_API, LogicException, Exception) -MOJA_DECLARE_EXCEPTION(CORE_API, AssertionViolationException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, NullPointerException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, NullValueException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, BugcheckException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, InvalidArgumentException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, NotImplementedException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, RangeException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, IllegalStateException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, InvalidAccessException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, SignalException, LogicException) -MOJA_DECLARE_EXCEPTION(CORE_API, UnhandledException, LogicException) - -MOJA_DECLARE_EXCEPTION(CORE_API, RuntimeException, Exception) -MOJA_DECLARE_EXCEPTION(CORE_API, NotFoundException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, ExistsException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, TimeoutException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, SystemException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, RegularExpressionException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, LibraryLoadException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, LibraryAlreadyLoadedException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, LibrarySymbolLoadException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, NoThreadAvailableException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, PropertyNotSupportedException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, PoolOverflowException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, NoPermissionException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, OutOfMemoryException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, DataException, RuntimeException) - -MOJA_DECLARE_EXCEPTION(CORE_API, DataFormatException, DataException) -MOJA_DECLARE_EXCEPTION(CORE_API, SyntaxException, DataException) -MOJA_DECLARE_EXCEPTION(CORE_API, CircularReferenceException, DataException) -MOJA_DECLARE_EXCEPTION(CORE_API, PathSyntaxException, SyntaxException) -MOJA_DECLARE_EXCEPTION(CORE_API, IOException, RuntimeException) -MOJA_DECLARE_EXCEPTION(CORE_API, ProtocolException, IOException) -MOJA_DECLARE_EXCEPTION(CORE_API, FileException, IOException) -MOJA_DECLARE_EXCEPTION(CORE_API, FileExistsException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, FileNotFoundException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, PathNotFoundException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, FileReadOnlyException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, FileAccessDeniedException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, CreateFileException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, OpenFileException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, WriteFileException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, ReadFileException, FileException) -MOJA_DECLARE_EXCEPTION(CORE_API, UnknownURISchemeException, RuntimeException) - -MOJA_DECLARE_EXCEPTION(CORE_API, ApplicationException, Exception) -MOJA_DECLARE_EXCEPTION(CORE_API, BadCastException, RuntimeException) - -} // namespace moja - -#endif // MOJA_CORE_EXCEPTION_H_ \ No newline at end of file diff --git a/Source/moja.core/include/moja/filesystem.h b/Source/moja.core/include/moja/filesystem.h new file mode 100644 index 0000000..b549d20 --- /dev/null +++ b/Source/moja.core/include/moja/filesystem.h @@ -0,0 +1,14 @@ +#pragma once + +#include "_core_exports.h" + +#include + +namespace moja::filesystem { + +using namespace std::filesystem; + +inline path CORE_API expand(const std::string& path); +inline path CORE_API expand(const path& path); + +} // namespace moja::filesystem diff --git a/Source/moja.core/src/datetime.cpp b/Source/moja.core/src/datetime.cpp index a290b67..07f24d5 100644 --- a/Source/moja.core/src/datetime.cpp +++ b/Source/moja.core/src/datetime.cpp @@ -1,7 +1,5 @@ #include "moja/datetime.h" -#include "moja/exception.h" - using namespace date; using namespace std::chrono; diff --git a/Source/moja.core/src/exception.cpp b/Source/moja.core/src/exception.cpp deleted file mode 100644 index ea7aa3b..0000000 --- a/Source/moja.core/src/exception.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include "moja/exception.h" - -#include - -namespace moja { - -Exception::Exception(int code) : _pNested(0), _code(code) {} - -Exception::Exception(const std::string& msg, int code) : _msg(msg), _pNested(0), _code(code) {} - -Exception::Exception(const std::string& msg, const std::string& arg, int code) : _msg(msg), _pNested(0), _code(code) { - if (!arg.empty()) { - _msg.append(": "); - _msg.append(arg); - } -} - -Exception::Exception(const std::string& msg, const Exception& nested, int code) - : _msg(msg), _pNested(nested.clone()), _code(code) {} - -Exception::Exception(const Exception& exc) : std::exception(exc), _msg(exc._msg), _code(exc._code) { - _pNested = exc._pNested ? exc._pNested->clone() : 0; -} - -Exception::~Exception() throw() { delete _pNested; } - -Exception& Exception::operator=(const Exception& exc) { - if (&exc != this) { - delete _pNested; - _msg = exc._msg; - _pNested = exc._pNested ? exc._pNested->clone() : 0; - _code = exc._code; - } - - return *this; -} - -const char* Exception::name() const throw() { return "Exception"; } - -const char* Exception::className() const throw() { return typeid(*this).name(); } - -const char* Exception::what() const throw() { return name(); } - -std::string Exception::displayText() const { - std::string txt = name(); - if (!_msg.empty()) { - txt.append(": "); - txt.append(_msg); - } - return txt; -} - -void Exception::extendedMessage(const std::string& arg) { - if (!arg.empty()) { - if (!_msg.empty()) _msg.append(": "); - _msg.append(arg); - } -} - -Exception* Exception::clone() const { return new Exception(*this); } - -void Exception::rethrow() const { throw *this; } - -MOJA_IMPLEMENT_EXCEPTION(LogicException, Exception, "Logic exception") -MOJA_IMPLEMENT_EXCEPTION(AssertionViolationException, LogicException, "Assertion violation") -MOJA_IMPLEMENT_EXCEPTION(NullPointerException, LogicException, "Null pointer") -MOJA_IMPLEMENT_EXCEPTION(NullValueException, LogicException, "Null value") -MOJA_IMPLEMENT_EXCEPTION(BugcheckException, LogicException, "Bugcheck") -MOJA_IMPLEMENT_EXCEPTION(InvalidArgumentException, LogicException, "Invalid argument") -MOJA_IMPLEMENT_EXCEPTION(NotImplementedException, LogicException, "Not implemented") -MOJA_IMPLEMENT_EXCEPTION(RangeException, LogicException, "Out of range") -MOJA_IMPLEMENT_EXCEPTION(IllegalStateException, LogicException, "Illegal state") -MOJA_IMPLEMENT_EXCEPTION(InvalidAccessException, LogicException, "Invalid access") -MOJA_IMPLEMENT_EXCEPTION(SignalException, LogicException, "Signal received") -MOJA_IMPLEMENT_EXCEPTION(UnhandledException, LogicException, "Unhandled exception") - -MOJA_IMPLEMENT_EXCEPTION(RuntimeException, Exception, "Runtime exception") -MOJA_IMPLEMENT_EXCEPTION(NotFoundException, RuntimeException, "Not found") -MOJA_IMPLEMENT_EXCEPTION(ExistsException, RuntimeException, "Exists") -MOJA_IMPLEMENT_EXCEPTION(TimeoutException, RuntimeException, "Timeout") -MOJA_IMPLEMENT_EXCEPTION(SystemException, RuntimeException, "System exception") -MOJA_IMPLEMENT_EXCEPTION(RegularExpressionException, RuntimeException, "Error in regular expression") -MOJA_IMPLEMENT_EXCEPTION(LibraryLoadException, RuntimeException, "Cannot load library") -MOJA_IMPLEMENT_EXCEPTION(LibraryAlreadyLoadedException, RuntimeException, "Library already loaded") -MOJA_IMPLEMENT_EXCEPTION(LibrarySymbolLoadException, RuntimeException, "Cannot load library symbol") -MOJA_IMPLEMENT_EXCEPTION(NoThreadAvailableException, RuntimeException, "No thread available") -MOJA_IMPLEMENT_EXCEPTION(PropertyNotSupportedException, RuntimeException, "Property not supported") -MOJA_IMPLEMENT_EXCEPTION(PoolOverflowException, RuntimeException, "Pool overflow") -MOJA_IMPLEMENT_EXCEPTION(NoPermissionException, RuntimeException, "No permission") -MOJA_IMPLEMENT_EXCEPTION(OutOfMemoryException, RuntimeException, "Out of memory") -MOJA_IMPLEMENT_EXCEPTION(DataException, RuntimeException, "Data error") - -MOJA_IMPLEMENT_EXCEPTION(DataFormatException, DataException, "Bad data format") -MOJA_IMPLEMENT_EXCEPTION(SyntaxException, DataException, "Syntax error") -MOJA_IMPLEMENT_EXCEPTION(CircularReferenceException, DataException, "Circular reference") -MOJA_IMPLEMENT_EXCEPTION(PathSyntaxException, SyntaxException, "Bad path syntax") -MOJA_IMPLEMENT_EXCEPTION(IOException, RuntimeException, "I/O error") -MOJA_IMPLEMENT_EXCEPTION(ProtocolException, IOException, "Protocol error") -MOJA_IMPLEMENT_EXCEPTION(FileException, IOException, "File access error") -MOJA_IMPLEMENT_EXCEPTION(FileExistsException, FileException, "File exists") -MOJA_IMPLEMENT_EXCEPTION(FileNotFoundException, FileException, "File not found") -MOJA_IMPLEMENT_EXCEPTION(PathNotFoundException, FileException, "Path not found") -MOJA_IMPLEMENT_EXCEPTION(FileReadOnlyException, FileException, "File is read-only") -MOJA_IMPLEMENT_EXCEPTION(FileAccessDeniedException, FileException, "Access to file denied") -MOJA_IMPLEMENT_EXCEPTION(CreateFileException, FileException, "Cannot create file") -MOJA_IMPLEMENT_EXCEPTION(OpenFileException, FileException, "Cannot open file") -MOJA_IMPLEMENT_EXCEPTION(WriteFileException, FileException, "Cannot write file") -MOJA_IMPLEMENT_EXCEPTION(ReadFileException, FileException, "Cannot read file") -MOJA_IMPLEMENT_EXCEPTION(UnknownURISchemeException, RuntimeException, "Unknown URI scheme") - -MOJA_IMPLEMENT_EXCEPTION(ApplicationException, Exception, "Application exception") -MOJA_IMPLEMENT_EXCEPTION(BadCastException, RuntimeException, "Bad cast exception") - -} // namespace moja diff --git a/Source/moja.core/src/filesystem.cpp b/Source/moja.core/src/filesystem.cpp new file mode 100644 index 0000000..2d25e68 --- /dev/null +++ b/Source/moja.core/src/filesystem.cpp @@ -0,0 +1,11 @@ +#include "moja/filesystem.h" + +#include + +namespace moja::filesystem { + +path expand(const std::string& path) { return Poco::Path::expand(path); } + +path expand(const path& path) { return expand(path.string()); } + +} // namespace moja::filesystem \ No newline at end of file diff --git a/Source/moja.core/src/logging.cpp b/Source/moja.core/src/logging.cpp index d6d3302..697b5ca 100644 --- a/Source/moja.core/src/logging.cpp +++ b/Source/moja.core/src/logging.cpp @@ -1,13 +1,13 @@ #include "moja/logging.h" #include "moja/environment.h" +#include "moja/filesystem.h" #include #include #include #include -#include #include namespace moja { @@ -22,7 +22,7 @@ std::string Logging::_loggingConfigurationFile = "unknown"; void Logging::init() { namespace logging = boost::log; namespace attrs = boost::log::attributes; - namespace fs = std::filesystem; + namespace fs = moja::filesystem; static auto initialized = false; if (initialized) { diff --git a/Source/moja.core/src/pocojsonutils.cpp b/Source/moja.core/src/pocojsonutils.cpp index d00fe9c..6b89677 100644 --- a/Source/moja.core/src/pocojsonutils.cpp +++ b/Source/moja.core/src/pocojsonutils.cpp @@ -1,16 +1,8 @@ #include "moja/pocojsonutils.h" -#include "moja/coreexceptions.h" -#include "moja/exception.h" - #include #include -#include - -using moja::FileName; -using moja::FileNotFoundException; -using moja::NotImplementedException; namespace moja { @@ -40,44 +32,37 @@ DynamicVector parsePocoJSONToDynamic(const Poco::JSON::Array::Ptr& val) { } else { auto object = val->get(i); if (object.isArray()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Array")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Array"); } else if (object.isBoolean()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isDeque()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Deque")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Deque"); } else if (object.isEmpty()) { } else if (object.isInteger()) { try { auto value = object.extract(); arrayDocument.push_back(value); - } catch (Poco::BadCastException) { + } catch (Poco::BadCastException&) { auto value = object.extract(); arrayDocument.push_back(value); } } else if (object.isList()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - List")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - List"); } else if (object.isNumeric()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isSigned()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Signed")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Signed"); } else if (object.isString()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isStruct()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Struct")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Struct"); } else if (object.isVector()) { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Vector")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Vector"); } else { - auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Unexpected type")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Unexpected type"); } } } @@ -110,44 +95,38 @@ DynamicVar parsePocoJSONToDynamic(const Poco::JSON::Object::Ptr& val) { document[var.first] = subDocument; } else { if (var.second.isArray()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Array")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Array"); } else if (var.second.isBoolean()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isDeque()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Deque")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Deque"); } else if (var.second.isEmpty()) { } else if (var.second.isInteger()) { try { auto value = var.second.extract(); document[var.first] = value; - } catch (Poco::BadCastException) { + } catch (Poco::BadCastException&) { auto value = var.second.extract(); document[var.first] = value; } } else if (var.second.isList()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - List")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - List"); + } else if (var.second.isNumeric()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isSigned()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Signed")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Signed"); } else if (var.second.isString()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isStruct()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Struct")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Struct"); } else if (var.second.isVector()) { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Vector")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Vector"); } else { - auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Unexpected type")).str(); - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic - Unexpected type"); } } } @@ -185,8 +164,7 @@ DynamicVar parsePocoJSONToDynamic(const DynamicVar& data) { DynamicVar result = data.extract(); return result; } - auto msg = "Unhandled data type in parse of json into dynamic"; - BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); + throw std::runtime_error("Unhandled data type in parse of json into dynamic"); } // -------------------------------------------------------------------------------------------- @@ -204,7 +182,7 @@ DynamicVar parsePocoVarToDynamic(const DynamicVar& var) { if (var.isInteger()) { try { return DynamicVar(var.extract()); - } catch (Poco::BadCastException) { + } catch (Poco::BadCastException&) { return DynamicVar(var.convert()); } } @@ -249,24 +227,19 @@ DynamicVar parsePocoVarToDynamic(const DynamicVar& var) { } } DynamicObject result; - for (auto item : s) { - result[item.first] = parsePocoVarToDynamic(item.second); + for (auto [key, value] : s) { + result[key] = parsePocoVarToDynamic(value); } return result; } - try { - auto obj = var.extract(); - DynamicObject result; - for (auto kvp : *obj) { - result[kvp.first] = parsePocoVarToDynamic(kvp.second); - } - return result; - } catch (std::exception) { + auto obj = var.extract(); + DynamicObject result; + for (auto [key, value] : *obj) { + result[key] = parsePocoVarToDynamic(value); } + return result; - auto msg = (boost::format("Unhandled data type in parse of Poco::Dynamic::Var into Dynamic")).str(); - BOOST_THROW_EXCEPTION(CoreUnhandledType() << moja::TypeErrorMsg(msg)); } } // namespace moja \ No newline at end of file diff --git a/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h b/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h index 825a315..d32f079 100644 --- a/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h +++ b/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h @@ -5,9 +5,7 @@ #include #include - -#include -#include +#include #include #include @@ -17,6 +15,8 @@ #include #include +namespace fs = moja::filesystem; + namespace moja { namespace datarepository { @@ -42,8 +42,7 @@ class DATAREPOSITORY_API MetaDataRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool MetaDataRasterReaderInterface::file_exists(const std::string& path) { - Poco::File pf(path); - return pf.exists(); + return fs::exists(path); } // -------------------------------------------------------------------------------------------- @@ -88,8 +87,7 @@ class DATAREPOSITORY_API TileRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool TileRasterReaderInterface::file_exists(const std::string& path) { - Poco::File pf(path); - return pf.exists(); + return fs::exists(path); } // -------------------------------------------------------------------------------------------- @@ -134,8 +132,7 @@ class DATAREPOSITORY_API StackRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool StackRasterReaderInterface::file_exists(const std::string& path) { - Poco::File pf(path); - return pf.exists(); + return fs::exists(path); } // -------------------------------------------------------------------------------------------- diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index ba15231..3dc724e 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -2,12 +2,15 @@ #include "moja/datarepository/datarepositoryexceptions.h" -#include +#include + #include #include #include +namespace fs = moja::filesystem; + namespace moja { namespace datarepository { @@ -25,8 +28,7 @@ class SQLiteConnection { sqlite3_config(SQLITE_CONFIG_URI, 1); if (path.find(":memory:") == std::string::npos) { - Poco::File file(path); - if (!file.exists()) { + if (!fs::exists(path)) { BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(path)); } } diff --git a/Source/moja.datarepository/src/rasterreader.cpp b/Source/moja.datarepository/src/rasterreader.cpp index fbfb3f9..6a406c1 100644 --- a/Source/moja.datarepository/src/rasterreader.cpp +++ b/Source/moja.datarepository/src/rasterreader.cpp @@ -5,30 +5,29 @@ #include #include +#include -#include -#include #include -#include -#include #include -#include + +#include #include -#include namespace moja { namespace datarepository { +namespace fs = moja::filesystem; + // -------------------------------------------------------------------------------------------- FlintMetaDataRasterReader::FlintMetaDataRasterReader(const std::string& path, const std::string& prefix, const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { - auto filePath = Poco::Path(path); - auto abs = filePath.absolute().toString(); - _metaPath = (boost::format("%1%%2%%3%.json") % abs % Poco::Path::separator() % prefix).str(); + + const auto mata_path = fs::absolute(path) / fmt::format("{}.json", prefix); + _metaPath = mata_path.string(); } DynamicObject FlintMetaDataRasterReader::readMetaData() const { @@ -51,7 +50,8 @@ DynamicObject FlintMetaDataRasterReader::readMetaData() const { FlintTileRasterReader::FlintTileRasterReader(const std::string& path, const Point& origin, const std::string& prefix, const TileBlockCellIndexer& indexer, const DynamicObject& settings) : TileRasterReaderInterface(path, origin, prefix, indexer, settings) { - _tilePath = (boost::format("%1%%2%%3%_%4%.blk") % path % Poco::Path::separator() % prefix % tile_id(origin)).str(); + const auto tile_path = fs::path(path) / fmt::format("{}_{}.blk", prefix, tile_id(origin)); + _tilePath = tile_path.string(); } FlintTileRasterReader::~FlintTileRasterReader() {} @@ -119,7 +119,9 @@ void FlintTileRasterReader::readFlintBlockData(const BlockIdx& blk_idx, char* bl FlintStackRasterReader::FlintStackRasterReader(const std::string& path, const Point& origin, const std::string& prefix, const TileBlockCellIndexer& indexer, const DynamicObject& settings) : StackRasterReaderInterface(path, origin, prefix, indexer, settings) { - _tilePath = (boost::format("%1%%2%%3%_%4%.blk") % path % Poco::Path::separator() % prefix % stack_id(origin)).str(); + + const auto tile_path = fs::path(path) / fmt::format("{}_{}.blk", prefix, stack_id(origin)); + _tilePath = tile_path.string(); } FlintStackRasterReader::~FlintStackRasterReader() {} diff --git a/Source/moja.flint.configuration/CMakeLists.txt b/Source/moja.flint.configuration/CMakeLists.txt index 01a32c4..4dc97c7 100644 --- a/Source/moja.flint.configuration/CMakeLists.txt +++ b/Source/moja.flint.configuration/CMakeLists.txt @@ -27,7 +27,6 @@ configure_file(../templates/exports.h ${CMAKE_CURRENT_SOURCE_DIR}/include/moja/f set(MOJA_FLINT_Configuration_headers include/moja/flint/${PACKAGE}/_${PACKAGE}_exports.h include/moja/flint/configuration/configuration.h - include/moja/flint/configuration/configurationexceptions.h include/moja/flint/configuration/configtile.h include/moja/flint/configuration/configblock.h include/moja/flint/configuration/configcell.h diff --git a/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h b/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h deleted file mode 100644 index f26d2e9..0000000 --- a/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ -#define MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ - -#include "moja/flint/configuration/_configuration_exports.h" - -#include - -#include -#include - -namespace moja { -namespace flint { -namespace configuration { - -struct CONFIGURATION_API ConfigurationException : virtual std::exception, virtual boost::exception {}; - -struct CONFIGURATION_API AssertionViolationException : virtual ConfigurationException {}; -typedef boost::error_info AssertMsg; - -struct CONFIGURATION_API FileNotFoundException : virtual ConfigurationException {}; -typedef boost::error_info FileName; - -struct CONFIGURATION_API ModuleOrderOverlapException : virtual ConfigurationException {}; -typedef boost::error_info Order; -typedef boost::error_info> ModuleNames; - -struct CONFIGURATION_API ModuleParamsException : virtual ConfigurationException {}; -typedef boost::error_info Param; - -struct CONFIGURATION_API SpinupModuleOrderOverlapException : virtual ConfigurationException {}; -typedef boost::error_info Order; -typedef boost::error_info> SpinupModuleNames; - -struct CONFIGURATION_API LandscapeDefinitionException : virtual ConfigurationException {}; -typedef boost::error_info Component; -typedef boost::error_info Constraint; - -/// -- Provider exceptions -struct CONFIGURATION_API ProviderSettingsException : virtual ConfigurationException {}; -typedef boost::error_info ProviderName; -typedef boost::error_info ProviderLibrary; - -struct CONFIGURATION_API ProviderMissingTypeException : virtual ConfigurationException {}; -typedef boost::error_info ProviderName; - -struct CONFIGURATION_API ProviderInvalidNameTypeException : virtual ConfigurationException {}; -typedef boost::error_info ProviderName; -typedef boost::error_info ProviderLibrary; -typedef boost::error_info ProviderType; - -} // namespace configuration -} // namespace flint -} // namespace moja - -#endif // MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ diff --git a/Source/moja.flint.configuration/src/configblock.cpp b/Source/moja.flint.configuration/src/configblock.cpp index 6030361..d66e083 100644 --- a/Source/moja.flint.configuration/src/configblock.cpp +++ b/Source/moja.flint.configuration/src/configblock.cpp @@ -1,7 +1,6 @@ #include "moja/flint/configuration/configblock.h" #include "moja/flint/configuration/configtile.h" -#include "moja/flint/configuration/configurationexceptions.h" #include #include @@ -14,19 +13,19 @@ namespace configuration { ConfigBlock::ConfigBlock(const ConfigTile& tile, int row, int col, int blockSizeX, int blockSizeY) : _tile(tile) { if (row < 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("row") << Constraint(">= 0")); + throw std::invalid_argument("Error in config block row < 0"); } if (col < 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("col") << Constraint(">= 0")); + throw std::invalid_argument("Error in config block col < 0"); } if (blockSizeX <= 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("blockSizeX") << Constraint("> 0")); + throw std::invalid_argument("Error in config block blockSizeX <= 0"); } if (blockSizeY <= 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("blockSizeY") << Constraint("> 0")); + throw std::invalid_argument("Error in config block blockSizeY <= 0"); } _row = row; diff --git a/Source/moja.flint.configuration/src/configcell.cpp b/Source/moja.flint.configuration/src/configcell.cpp index c9ec13f..906bc6f 100644 --- a/Source/moja.flint.configuration/src/configcell.cpp +++ b/Source/moja.flint.configuration/src/configcell.cpp @@ -1,7 +1,6 @@ #include "moja/flint/configuration/configcell.h" #include "moja/flint/configuration/configblock.h" -#include "moja/flint/configuration/configurationexceptions.h" #include #include @@ -15,11 +14,11 @@ namespace configuration { ConfigCell::ConfigCell(const ConfigBlock& block, int row, int col) : _block(block) { if (row < 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("row") << Constraint(">= 0")); + throw std::invalid_argument("Error in config cell row < 0"); } if (col < 0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("col") << Constraint(">= 0")); + throw std::invalid_argument("Error in config cell col < 0"); } _row = row; diff --git a/Source/moja.flint.configuration/src/configtile.cpp b/Source/moja.flint.configuration/src/configtile.cpp index 845f6ae..75b9c6c 100644 --- a/Source/moja.flint.configuration/src/configtile.cpp +++ b/Source/moja.flint.configuration/src/configtile.cpp @@ -1,7 +1,6 @@ #include "moja/flint/configuration/configtile.h" #include "moja/flint/configuration/configblock.h" -#include "moja/flint/configuration/configurationexceptions.h" #include @@ -16,19 +15,19 @@ namespace configuration { ConfigTile::ConfigTile(int xIndex, int yIndex, double xSize, double ySize, int xPixels, int yPixels) { if (xSize <= 0.0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("xSize") << Constraint("> 0.0")); + throw std::invalid_argument("Error in config tile xSize <= 0"); } if (ySize <= 0.0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("ySize") << Constraint("> 0.0")); + throw std::invalid_argument("Error in config tile ySize <= 0"); } if (xPixels < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("xPixels") << Constraint("> 0")); + throw std::invalid_argument("Error in config tile xPixels < 1"); } if (yPixels < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("yPixels") << Constraint("> 0")); + throw std::invalid_argument("Error in config tile yPixels < 1"); } _xIndex = xIndex; diff --git a/Source/moja.flint.configuration/src/configuration.cpp b/Source/moja.flint.configuration/src/configuration.cpp index 369667f..f2f9d65 100644 --- a/Source/moja.flint.configuration/src/configuration.cpp +++ b/Source/moja.flint.configuration/src/configuration.cpp @@ -1,6 +1,5 @@ #include "moja/flint/configuration/configuration.h" -#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/externalpool.h" #include "moja/flint/configuration/externalvariable.h" #include "moja/flint/configuration/flintdatavariable.h" @@ -262,11 +261,12 @@ void Configuration::addModule(const std::string& libraryName, const std::string& DynamicObject settings) { // It is an error for two modules to have the same order - simulation // results could be inconsistent. - auto sameOrder = std::find_if(_modules.begin(), _modules.end(), - [order](std::shared_ptr other) { return other->order() == order; }); + const auto sameOrder = std::find_if(_modules.begin(), _modules.end(), + [order](std::shared_ptr other) { return other->order() == order; }); if (sameOrder != _modules.end()) { - throw ModuleOrderOverlapException() << Order(order) << ModuleNames({libraryName, name, (*sameOrder)->name()}); + throw std::runtime_error("Error module order overlap in " + libraryName + " " + name + " = " + + (*sameOrder)->name()); } auto module = std::make_shared(libraryName, name, order, isProxy, settings); diff --git a/Source/moja.flint.configuration/src/json2configurationprovider.cpp b/Source/moja.flint.configuration/src/json2configurationprovider.cpp index 5245e8b..daf9c70 100644 --- a/Source/moja.flint.configuration/src/json2configurationprovider.cpp +++ b/Source/moja.flint.configuration/src/json2configurationprovider.cpp @@ -1,20 +1,18 @@ #include "moja/flint/configuration/json2configurationprovider.h" -#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/iterationbase.h" #include "moja/flint/configuration/iterationtileindex.h" #include "moja/flint/configuration/library.h" #include "moja/flint/configuration/localdomain.h" #include "moja/flint/configuration/spinup.h" +#include #include #include -#include #include #include #include -#include #include #include @@ -28,37 +26,36 @@ using Poco::JSON::Stringifier; #include + namespace moja { namespace flint { namespace configuration { +namespace fs = moja::filesystem; + JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath) { - for (const std::string configFie : configFilePath) { - Poco::File file(configFie); - if (!file.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); + for (const auto& config_file : configFilePath) { + if (!fs::exists(config_file)) { + throw std::runtime_error("Error cant find config file " + config_file); } } _configFilePath = configFilePath; - //_configProviderFilePath = null; _hasProviderConfigFile = false; } JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath, const std::vector& configProviderFilePath) { - for (const auto& configFie : configFilePath) { - Poco::File file(configFie); - if (!file.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); + for (const auto& config_file : configFilePath) { + if (!fs::exists(config_file)) { + throw std::runtime_error("Error cant find config file " + config_file); } } _configFilePath = std::move(configFilePath); if (!configProviderFilePath.empty()) { - for (const auto& configProviderFile : configProviderFilePath) { - Poco::File fileProvider(configProviderFile); - if (!fileProvider.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configProviderFile)); + for (const auto& config_provider_file : configProviderFilePath) { + if (!fs::exists(config_provider_file)) { + throw std::runtime_error("Error cant find provider config file " + config_provider_file); } } _configProviderFilePath = std::move(configProviderFilePath); @@ -404,7 +401,7 @@ void JSON2ConfigurationProvider::createSpinup(DynamicVar& parsedJSON, Configurat if (enabled) { for (auto param : {"sequencer_library", "sequencer", "simulateLandUnit", "landUnitBuildSuccess"}) { if (!spinupStruct.contains(param)) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param((boost::format("Spinup.%1%") % param).str())); + throw std::runtime_error("Error missing spin up parameter " + std::string(param)); } } @@ -441,18 +438,17 @@ void JSON2ConfigurationProvider::createLibraries(DynamicVar& parsedJSON, Configu } bool JSON2ConfigurationProvider::fileExists(const std::string& path) { - Poco::File pf(path); - return pf.exists(); + return fs::exists(path); } void JSON2ConfigurationProvider::createProviders(DynamicVar& parsedJSON, Configuration& config) const { auto jsonStruct2 = *parsedJSON.extract(); auto provider = jsonStruct2.getObject("Providers"); auto& data = *(provider.get()); - for (auto& item : data) { - auto d = parsePocoJSONToDynamic(item.second); + for (auto& [name, var] : data) { + auto d = parsePocoJSONToDynamic(var); if (d.isEmpty()) { - BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(item.first)); + throw std::runtime_error("Error provider settings are empty " + name); } auto xx = d.extract(); std::string libName; @@ -463,7 +459,7 @@ void JSON2ConfigurationProvider::createProviders(DynamicVar& parsedJSON, Configu const std::string providerType = d["type"].convert(); auto& settings = d.extract(); - config.addProvider(item.first, libName, providerType, settings); + config.addProvider(name, libName, providerType, settings); } } @@ -634,10 +630,10 @@ void JSON2ConfigurationProvider::createModules(DynamicVar& parsedJSON, Configura if (!enabled) continue; } if (!moduleStruct.contains("library")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); + throw std::runtime_error("Error module definition missing library"); } if (!moduleStruct.contains("order")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); + throw std::runtime_error("Error module definition missing order"); } const auto& moduleLibraryName = moduleStruct["library"].extract(); @@ -666,10 +662,10 @@ void JSON2ConfigurationProvider::createSpinupModules(DynamicVar& parsedJSON, Con } if (!moduleStruct.contains("library")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); + throw std::runtime_error("Error spin up module definition missing library"); } if (!moduleStruct.contains("order")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); + throw std::runtime_error("Error spin up module definition missing order"); } const auto moduleLibraryName = moduleStruct["library"].extract(); diff --git a/Source/moja.flint.configuration/src/provider.cpp b/Source/moja.flint.configuration/src/provider.cpp index f12c2e8..b794f78 100644 --- a/Source/moja.flint.configuration/src/provider.cpp +++ b/Source/moja.flint.configuration/src/provider.cpp @@ -1,7 +1,5 @@ #include "moja/flint/configuration/provider.h" -#include "moja/flint/configuration/configurationexceptions.h" - #include namespace moja { @@ -12,19 +10,16 @@ Provider::Provider(const std::string& name, const std::string& library, const st const DynamicObject& settings) : _name(name), _library(library), _providerType(providerType), _settings(settings) { if (name.length() == 0 || all(name, boost::algorithm::is_space())) { - BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() - << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); + throw std::invalid_argument("Error provider name is empty"); } if (library.length() == 0 || all(library, boost::algorithm::is_space())) { - BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() - << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); + throw std::invalid_argument("Error provider library is empty"); } if (providerType.length() == 0 || all(providerType, boost::algorithm::is_space())) { - BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() - << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); + throw std::invalid_argument("Error provider type is empty"); } if (settings.empty() || !settings.contains("type")) { - BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(name) << ProviderLibrary(library)); + throw std::invalid_argument("Error provider settings is empty or invalid"); } } diff --git a/Source/moja.flint.configuration/src/spinup.cpp b/Source/moja.flint.configuration/src/spinup.cpp index f60d60b..9095357 100644 --- a/Source/moja.flint.configuration/src/spinup.cpp +++ b/Source/moja.flint.configuration/src/spinup.cpp @@ -1,7 +1,6 @@ #include "moja/flint/configuration/spinup.h" #include "moja/flint/configuration/configuration.h" -#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/externalvariable.h" #include "moja/flint/configuration/flintdatavariable.h" #include "moja/flint/configuration/localdomain.h" @@ -52,8 +51,8 @@ void Spinup::addSpinupModule(const std::string& libraryName, const std::string& [order](std::shared_ptr other) { return other->order() == order; }); if (sameOrder != _modules.end()) { - throw SpinupModuleOrderOverlapException() - << Order(order) << SpinupModuleNames({libraryName, name, (*sameOrder)->name()}); + throw std::runtime_error("Error spin up module order overlap in " + libraryName + " " + name + " = " + + (*sameOrder)->name()); } auto module = std::make_shared(libraryName, name, order, createNew, settings); diff --git a/Source/moja.flint/include/moja/flint/flintexceptions.h b/Source/moja.flint/include/moja/flint/flintexceptions.h index 567f4aa..b422c39 100644 --- a/Source/moja.flint/include/moja/flint/flintexceptions.h +++ b/Source/moja.flint/include/moja/flint/flintexceptions.h @@ -5,7 +5,6 @@ #include -#include #include namespace moja { @@ -13,40 +12,6 @@ namespace flint { struct FLINT_API FLINTException : virtual std::exception, virtual boost::exception {}; -struct FLINT_API PoolNotFoundException : virtual FLINTException {}; -typedef boost::error_info PoolName; -typedef boost::error_info PoolIndex; - -struct FLINT_API VariableNotFoundException : virtual FLINTException {}; -typedef boost::error_info VariableName; - -struct FLINT_API VariableEmptyWhenValueExpectedException : virtual FLINTException {}; -typedef boost::error_info VariableName; - -struct FLINT_API SequencerNotFoundException : virtual FLINTException {}; -typedef boost::error_info Details; -typedef boost::error_info LibraryName; -typedef boost::error_info SequencerName; - -struct FLINT_API TableNotFoundException : virtual FLINTException {}; -typedef boost::error_info TableName; - -struct FLINT_API PreconditionViolatedException : virtual FLINTException {}; -typedef boost::error_info Precondition; - -struct FLINT_API LandscapeDefinitionException : virtual FLINTException {}; -typedef boost::error_info Component; -typedef boost::error_info Constraint; - -struct FLINT_API IncompleteConfigurationException : virtual FLINTException {}; -typedef boost::error_info Item; -typedef boost::error_info Details; - -struct FLINT_API DuplicateModuleDefinedException : virtual FLINTException {}; -typedef boost::error_info Details; -typedef boost::error_info LibraryName; -typedef boost::error_info ModuleName; - struct FLINT_API LocalDomainError : virtual FLINTException {}; typedef boost::error_info Details; typedef boost::error_info LibraryName; diff --git a/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h b/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h index dd5a768..e1fb244 100644 --- a/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h +++ b/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h @@ -5,8 +5,6 @@ #include "moja/flint/ioperationmanager.h" #include "moja/flint/operationresultcollection.h" -#include - namespace moja { namespace flint { class IModule; @@ -52,8 +50,8 @@ class FLINT_API OperationManagerSimpleCache : public IOperationManager { const IPool* addPool(const std::string& name, const std::string& description, const std::string& units, double scale, int order, const std::shared_ptr transform) override { - throw moja::NotImplementedException(); - }; + throw std::logic_error("addPool not implemented"); + } const IPool* addPool(const std::string& name, double initValue = 0.0) override; const IPool* addPool(const std::string& name, const std::string& description, const std::string& units, double scale, int order, double initValue = 0.0) override; diff --git a/Source/moja.flint/include/moja/flint/writesystemconfig.h b/Source/moja.flint/include/moja/flint/writesystemconfig.h index acf7921..9fc66f6 100644 --- a/Source/moja.flint/include/moja/flint/writesystemconfig.h +++ b/Source/moja.flint/include/moja/flint/writesystemconfig.h @@ -58,22 +58,6 @@ class FLINT_API WriteSystemConfig : public flint::ModuleBase { void onOutputStep() override; void onError(std::string msg) override; - // --- RAII class for file handle - class FileHandle { - typedef FILE* ptr; - - public: - explicit FileHandle(std::string const& name, std::string const& mode = std::string("r")) - : _wrapped_file(fopen(name.c_str(), mode.c_str())) {} - ~FileHandle() { - if (_wrapped_file) fclose(_wrapped_file); - } - operator ptr() const { return _wrapped_file; } - - private: - ptr _wrapped_file; - }; - private: // Mutexes std::mutex& _fileHandlingMutex; diff --git a/Source/moja.flint/include/moja/flint/writevariablegrid.h b/Source/moja.flint/include/moja/flint/writevariablegrid.h index 1761212..46bc23f 100644 --- a/Source/moja.flint/include/moja/flint/writevariablegrid.h +++ b/Source/moja.flint/include/moja/flint/writevariablegrid.h @@ -40,22 +40,6 @@ class FLINT_API WriteVariableGrid : public ModuleBase { void onOutputStep() override; void onError(std::string msg) override; - // --- RAII class for file handle - class FileHandle { - typedef FILE* ptr; - - public: - explicit FileHandle(std::string const& name, std::string const& mode = std::string("r")) - : _wrapped_file(fopen(name.c_str(), mode.c_str())) {} - ~FileHandle() { - if (_wrapped_file) fclose(_wrapped_file); - } - operator ptr() const { return _wrapped_file; } - - private: - ptr _wrapped_file; - }; - // --- Base classs for data layer class DataSettingsB { public: diff --git a/Source/moja.flint/src/aggregatorfilewriter.cpp b/Source/moja.flint/src/aggregatorfilewriter.cpp index 14a2fc0..94ab343 100644 --- a/Source/moja.flint/src/aggregatorfilewriter.cpp +++ b/Source/moja.flint/src/aggregatorfilewriter.cpp @@ -10,9 +10,9 @@ #include #include #include +#include #include -#include #include #include #include @@ -20,11 +20,12 @@ #include #include -#include using Poco::format; using Poco::NotFoundException; +namespace fs = moja::filesystem; + namespace moja { namespace flint { @@ -66,9 +67,8 @@ void AggregatorFileWriter::onSystemInit() { try { // create file here and append flux data later, this will work in threaded mode as well // file created on onSystemInit, shared Mutex around file access used to append to it. - Poco::File outputFile(_fileName); - if (outputFile.exists()) outputFile.remove(); - outputFile.createFile(); + + if (fs::exists(_fileName)) fs::remove(_fileName); if (_writeFileHeader) { Poco::FileOutputStream streamFile(_fileName); @@ -106,20 +106,6 @@ void AggregatorFileWriter::onSystemInit() { // -------------------------------------------------------------------------------------------- void AggregatorFileWriter::onSystemShutdown() { - try { - // std::vector runStatData; - - // MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":File write start"; // << Filename perhaps - } catch (Poco::AssertionViolationException& exc) { - MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":AssertionViolationException - " << exc.displayText(); - throw; - } catch (const std::exception& e) { - MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":Unknown exception: - " << e.what(); - throw; - } catch (...) { - MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":Unknown Exception"; - throw; - } } // -------------------------------------------------------------------------------------------- diff --git a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp index f213480..e3a3670 100644 --- a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp @@ -15,7 +15,6 @@ #include #include -#include #include #include @@ -154,10 +153,6 @@ void AspatialLocalDomainController::run(const LandUnitInfo& lu) { } _sequencer->Run(_notificationCenter, _landUnitController); - } catch (const Exception& e) { - MOJA_LOG_FATAL << e.displayText(); - } catch (const boost::exception& e) { - MOJA_LOG_FATAL << boost::diagnostic_information(e); } catch (const std::exception& e) { MOJA_LOG_FATAL << e.what(); } diff --git a/Source/moja.flint/src/compositetransform.cpp b/Source/moja.flint/src/compositetransform.cpp index 248d441..f73512f 100644 --- a/Source/moja.flint/src/compositetransform.cpp +++ b/Source/moja.flint/src/compositetransform.cpp @@ -6,9 +6,6 @@ #include -using moja::flint::IncompleteConfigurationException; -using moja::flint::VariableNotFoundException; - namespace moja { namespace flint { @@ -18,19 +15,17 @@ void CompositeTransform::configure(DynamicObject config, const ILandUnitControll _dataRepository = &dataRepository; if (!config.contains("vars")) { - BOOST_THROW_EXCEPTION(IncompleteConfigurationException() << Item("vars")); + throw std::runtime_error("Error composite transform settings missing vars"); } auto varNames = config["vars"]; if (varNames.size() < 1) { - BOOST_THROW_EXCEPTION(IncompleteConfigurationException() - << Item("vars") << Details("At least one variable name required")); + throw std::runtime_error("Error composite transform vars at least one variable name required"); } for (std::string varName : varNames) { if (std::find(_varNames.begin(), _varNames.end(), varName) != _varNames.end()) { - BOOST_THROW_EXCEPTION(IncompleteConfigurationException() - << Item("vars") << Details("Duplicate variable reference")); + throw std::runtime_error("Error composite transform vars duplicate variable reference"); } _varNames.push_back(varName); @@ -56,10 +51,10 @@ void CompositeTransform::controllerChanged(const ILandUnitController& controller const DynamicVar& CompositeTransform::value() const { if (_variables.empty()) { - for (auto varName : _varNames) { - auto var = _landUnitController->getVariable(varName); + for (const auto& varName : _varNames) { + const auto* var = _landUnitController->getVariable(varName); if (var == nullptr) { - BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(varName)); + throw std::runtime_error("Error composite transform variable not found " + varName); } _variables.push_back(var); diff --git a/Source/moja.flint/src/externalvariable.cpp b/Source/moja.flint/src/externalvariable.cpp index 037c685..a7cb3c7 100644 --- a/Source/moja.flint/src/externalvariable.cpp +++ b/Source/moja.flint/src/externalvariable.cpp @@ -2,18 +2,13 @@ #include "moja/flint/itransform.h" -#include - -#include - namespace moja { namespace flint { const DynamicVar& ExternalVariable::value() const { return _transform->value(); } void ExternalVariable::set_value(DynamicVar) { - auto msg = (boost::format("Can't set External Variable: %1%.") % _info.name).str(); - throw ApplicationException(msg); + throw std::runtime_error("Can't set External Variable: "+ _info.name); } void ExternalVariable::controllerChanged(const ILandUnitController& controller) const { diff --git a/Source/moja.flint/src/landunitcontroller.cpp b/Source/moja.flint/src/landunitcontroller.cpp index 7fa49f5..7f84f3d 100644 --- a/Source/moja.flint/src/landunitcontroller.cpp +++ b/Source/moja.flint/src/landunitcontroller.cpp @@ -87,7 +87,7 @@ void LandUnitController::initialisePools() const { _operationManager->initialise void LandUnitController::addVariable(std::string name, std::shared_ptr variable) { if (name.length() == 0 || all(name, boost::algorithm::is_space())) { - throw std::invalid_argument("name cannot be empty"); + throw std::invalid_argument("variable name cannot be empty"); } _variablesMap.insert(std::make_pair(name, variable.get())); _variables.push_back(variable); @@ -97,7 +97,7 @@ IVariable* LandUnitController::getVariable(const std::string& name) { const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { MOJA_LOG_FATAL << "Variable not found: " << name; - BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); + throw std::invalid_argument("Error variable not found " + name); } return v->second; } @@ -106,7 +106,7 @@ const IVariable* LandUnitController::getVariable(const std::string& name) const const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { MOJA_LOG_FATAL << "Variable not found: " << name; - BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); + throw std::invalid_argument("Error variable not found " + name); } return v->second; } diff --git a/Source/moja.flint/src/librarymanager.cpp b/Source/moja.flint/src/librarymanager.cpp index a4be831..b6a10fb 100644 --- a/Source/moja.flint/src/librarymanager.cpp +++ b/Source/moja.flint/src/librarymanager.cpp @@ -7,7 +7,6 @@ #include "moja/flint/modulebase.h" #include -#include #include #include @@ -145,7 +144,9 @@ void LibraryManager::AddLibrary(LibraryType libraryType, const std::string& inLi break; } case LibraryType::Unknown: - default: { throw LibraryLoadException("Unknown library type defined", inLibraryName); } + default: { + throw std::runtime_error("Unknown library type defined " + inLibraryName); + } } // Update hash table @@ -202,7 +203,7 @@ ModuleInterfacePtr LibraryManager::GetModule(const std::string& libraryName, con auto moduleInit = _moduleRegistry.at(moduleKey); auto moduleInterface = moduleInit(); if (!moduleInterface) { - throw LibraryLoadException("Failed to initialise the module", moduleName); + throw std::runtime_error("Failed to initialise the module " + moduleName); } libraryInfo->moduleList.push_back(moduleInterface); @@ -221,7 +222,7 @@ ModuleInterfacePtr LibraryManager::GetModule(const std::string& libraryName, con } #endif catch (...) { - throw LibraryLoadException("Module doesn't exist", moduleName); + throw std::runtime_error("Module doesn't exist " + moduleName); } } @@ -234,13 +235,13 @@ TransformInterfacePtr LibraryManager::GetTransform(const std::string& libraryNam std::shared_ptr transformInterface = transformInit(); if (!transformInterface) { - throw LibraryLoadException("Failed to initialise the transform", transformName); + throw std::runtime_error("Failed to initialise the transform " + transformName); } libraryInfo->transformList.push_back(transformInterface); return transformInterface; } catch (...) { - throw LibraryLoadException("Transform doesn't exist", transformName); + throw std::runtime_error("Transform doesn't exist " + transformName); } } @@ -254,7 +255,7 @@ FlintDataInterfacePtr LibraryManager::GetFlintData(const std::string& libraryNam std::shared_ptr flintDataInterface = flintDataInit(); if (!flintDataInterface) { - throw LibraryLoadException("Failed to initialise the flintData", flintDataName); + throw std::runtime_error("Failed to initialise the flintData " + flintDataName); } flintDataInterface->libraryName = libraryName; flintDataInterface->typeName = flintDataName; @@ -267,7 +268,7 @@ FlintDataInterfacePtr LibraryManager::GetFlintData(const std::string& libraryNam } return flintDataInterface; } catch (...) { - throw LibraryLoadException("flintData doesn't exist", flintDataName); + throw std::runtime_error("flintData doesn't exist " + flintDataName); } } @@ -282,28 +283,28 @@ ModuleInterfacePtr LibraryManager::GetManagedModule(const std::string& managedMo auto moduleKey = ModuleKey(libraryName, managedModuleName); const auto moduleInit = _moduleRegistry[moduleKey]; if (moduleInit == nullptr) { - throw LibraryLoadException("Failed to initialise the managed module", managedModuleName); + throw std::runtime_error("Failed to initialise the managed module " + managedModuleName); } std::shared_ptr libraryInfo = (*libraryInfoIt).second; if (libraryInfo->GetLibraryType() != LibraryType::Managed) { - throw LibraryLoadException("Incorrect type for managed module", managedModuleName); + throw std::runtime_error("Incorrect type for managed module " + managedModuleName); } std::shared_ptr moduleInterface; std::map modulePathMap = FindLibraryPathsExact(managedModuleName); if (modulePathMap.size() != 1) { - throw LibraryLoadException("Managed module path issue", managedModuleName); + throw std::runtime_error("Managed module path issue" + managedModuleName); } try { moduleInterface = moduleInit(); } catch (...) { - throw LibraryLoadException("Managed module issue", managedModuleName); + throw std::runtime_error("Managed module issue " + managedModuleName); } if (!moduleInterface) { - throw LibraryLoadException("Failed to initialise the module", managedModuleName); + throw std::runtime_error("Failed to initialise the module " + managedModuleName); } libraryInfo->moduleList.push_back(moduleInterface); @@ -327,12 +328,12 @@ ProviderInterfacePtr LibraryManager::GetProvider(const std::string& libraryName, auto providerInit = _providerRegistry.at(providerKey); auto providerInterface = providerInit(settings); if (!providerInterface) { - throw LibraryLoadException("Failed to initialise the provider", providerName); + throw std::runtime_error("Failed to initialise the provider " + providerName); } libraryInfo->providerList.push_back(providerInterface); return providerInterface; } catch (...) { - throw LibraryLoadException("Provider doesn't exist", providerName); + throw std::runtime_error("Provider doesn't exist " + providerName); } } @@ -434,15 +435,15 @@ bool LibraryManager::LoadInternalLibrary(std::shared_ptr li AddLibrary(LibraryType::Internal, libraryHandles->libraryName); auto libraryInfo = _libraries[libraryHandles->libraryName]; if (libraryInfo == nullptr) { - throw LibraryLoadException("Library not found", libraryHandles->libraryName); + throw std::runtime_error("Library not found " + libraryHandles->libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { MOJA_LOG_DEBUG << (boost::format("LibraryMLoadInternalLibraryanager: %s") % "calling registrations").str(); if (libraryInfo->GetLibraryType() != LibraryType::Internal) { - throw LibraryLoadException("Attempt to load library already loaded as different type", - libraryHandles->libraryName); + throw std::runtime_error("Attempt to load library already loaded as different type" + + libraryHandles->libraryName); } auto internalLibraryInfo = std::static_pointer_cast(libraryInfo); @@ -466,12 +467,12 @@ bool LibraryManager::LoadManagedLibrary() { // Grab the library info. This has the file name of the library, as well as other info. auto libraryInfo = _libraries[libraryName]; if (libraryInfo == nullptr) { - throw LibraryLoadException("Library not found", libraryName); + throw std::runtime_error("Library not found " + libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { if (libraryInfo->GetLibraryType() != LibraryType::Managed) { - throw LibraryLoadException("Attempt to load library already loaded as different type", libraryName); + throw std::runtime_error("Attempt to load library already loaded as different type " + libraryName); } auto managedLibraryInfo = std::static_pointer_cast(_libraries[libraryName]); @@ -484,11 +485,11 @@ bool LibraryManager::LoadManagedLibrary() { // Skip this check if file manager has not yet been initialized if (!libraryFileToLoad.exists()) { - throw LibraryLoadException("Library not found", managedLibraryInfo->filename); + throw std::runtime_error("Library not found " + managedLibraryInfo->filename); } if (!CheckModuleCompatibility(libraryFileToLoad.path())) { - throw LibraryLoadException("Library not compatible", libraryFileToLoad.path()); + throw std::runtime_error("Library not compatible " + libraryFileToLoad.path()); } managedLibraryInfo->handle = std::make_unique(libraryFileToLoad.path()); @@ -537,12 +538,12 @@ bool LibraryManager::LoadExternalLibrary(const std::string& libraryName, const s // Grab the library info. This has the file name of the library, as well as other info. auto libraryInfo = _libraries[libraryName]; if (libraryInfo == nullptr) { - throw LibraryLoadException("Library not found", libraryName); + throw std::runtime_error("Library not found " + libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { if (libraryInfo->GetLibraryType() != LibraryType::External) { - throw LibraryLoadException("Attempt to load library already loaded as different type", libraryName); + throw std::runtime_error("Attempt to load library already loaded as different type " + libraryName); } auto externalLibraryInfo = std::static_pointer_cast(_libraries[libraryName]); @@ -554,11 +555,11 @@ bool LibraryManager::LoadExternalLibrary(const std::string& libraryName, const s externalLibraryInfo->handle = nullptr; if (!libraryFileToLoad.exists()) { - throw LibraryLoadException("Library not found", externalLibraryInfo->filename); + throw std::runtime_error("Library not found " + externalLibraryInfo->filename); } if (!CheckModuleCompatibility(libraryFileToLoad.path())) { - throw LibraryLoadException("Library not compatible", libraryFileToLoad.path()); + throw std::runtime_error("Library not compatible " + libraryFileToLoad.path()); } externalLibraryInfo->handle = std::make_unique(libraryFileToLoad.path()); diff --git a/Source/moja.flint/src/localdomaincontrollerbase.cpp b/Source/moja.flint/src/localdomaincontrollerbase.cpp index 8c000fe..ef102ba 100644 --- a/Source/moja.flint/src/localdomaincontrollerbase.cpp +++ b/Source/moja.flint/src/localdomaincontrollerbase.cpp @@ -23,8 +23,6 @@ #include #include -using moja::flint::SequencerNotFoundException; - namespace moja { namespace flint { @@ -92,17 +90,15 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co // Load the configured modules (and handle Proxies) _moduleMap.clear(); - for (const auto& module : config.modules()) { + for (const auto* module : config.modules()) { ModuleMapKey key(module->libraryName(), module->name()); if (_moduleMap.find(key) != _moduleMap.end()) { - // duplicate Key found - throw DuplicateModuleDefinedException() << Details("Error duplicate module defined") - << LibraryName(module->libraryName()) << SequencerName(module->name()); + throw std::runtime_error("Error duplicate modules " + module->libraryName() + " " + module->name()); } _moduleMap[key] = _libraryManager.GetModule(module->libraryName(), module->name()); _moduleMap[key]->setLandUnitController(_landUnitController); if (module->isProxy()) { - auto proxy = static_cast(modules()[key]); + auto proxy = dynamic_cast(modules()[key]); auto proxyModules = module->settings()["proxyModules"]; for (auto& item : proxyModules) { DynamicObject proxyModule = item.extract(); @@ -128,9 +124,8 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co ModuleMapKey sequencerKey(config.localDomain()->sequencerLibrary(), config.localDomain()->sequencer()); _sequencer = std::dynamic_pointer_cast(_moduleMap[sequencerKey]); if (_sequencer == nullptr) { - throw SequencerNotFoundException() << Details("Error finding sequencer") - << LibraryName(config.localDomain()->sequencerLibrary()) - << SequencerName(config.localDomain()->sequencer()); + throw std::runtime_error("Error finding sequencer " + config.localDomain()->sequencerLibrary() + " " + + config.localDomain()->sequencer()); } _sequencer->configure(timing); diff --git a/Source/moja.flint/src/lookuprandomtransform.cpp b/Source/moja.flint/src/lookuprandomtransform.cpp index 6415e36..4506699 100644 --- a/Source/moja.flint/src/lookuprandomtransform.cpp +++ b/Source/moja.flint/src/lookuprandomtransform.cpp @@ -10,9 +10,6 @@ #include -using moja::flint::IncompleteConfigurationException; -using moja::flint::VariableNotFoundException; - namespace moja { namespace flint { @@ -28,7 +25,7 @@ void LookupRandomTransform::configure(DynamicObject config, const ILandUnitContr } } if (!validConfiguration) { - throw IncompleteConfigurationException() << Item(key) << Details("Expected configuration item not found"); + throw std::runtime_error("Expected configuration item not found " + std::string(key)); } } //_fromVarName = config["from"].convert(); @@ -47,14 +44,9 @@ void LookupRandomTransform::controllerChanged(const ILandUnitController& control } const DynamicVar& LookupRandomTransform::value() const { - // auto from = _landUnitController->getVariable(_fromVarName); - // if (from == nullptr) { - // throw VariableNotFoundException() << VariableName(_fromVarName); - //} - - auto to = _landUnitController->getVariable(_toVarName); + const auto* to = _landUnitController->getVariable(_toVarName); if (to == nullptr) { - throw VariableNotFoundException() << VariableName(_toVarName); + throw std::runtime_error("Variable not found " + _toVarName); } // auto& fromValue = from->value(); diff --git a/Source/moja.flint/src/lookuptransform.cpp b/Source/moja.flint/src/lookuptransform.cpp index db06ce7..c0ea5c8 100644 --- a/Source/moja.flint/src/lookuptransform.cpp +++ b/Source/moja.flint/src/lookuptransform.cpp @@ -7,9 +7,6 @@ #include #include -using moja::flint::IncompleteConfigurationException; -using moja::flint::VariableNotFoundException; - namespace moja { namespace flint { @@ -25,7 +22,7 @@ void LookupTransform::configure(DynamicObject config, const ILandUnitController& } if (!validConfiguration) { - throw IncompleteConfigurationException() << Item(key) << Details("Expected configuration item not found"); + throw std::runtime_error("Expected configuration item not found " + std::string(key)); } } @@ -41,18 +38,18 @@ void LookupTransform::controllerChanged(const ILandUnitController& controller) { }; const DynamicVar& LookupTransform::value() const { - auto from = _landUnitController->getVariable(_fromVarName); + const auto* from = _landUnitController->getVariable(_fromVarName); if (from == nullptr) { - throw VariableNotFoundException() << VariableName(_fromVarName); + throw std::runtime_error("From variable not found " + _fromVarName); } - auto to = _landUnitController->getVariable(_toVarName); + const auto* to = _landUnitController->getVariable(_toVarName); if (to == nullptr) { - throw VariableNotFoundException() << VariableName(_toVarName); + throw std::runtime_error("To variable not found " + _fromVarName); } - auto& fromValue = from->value(); - auto& toValue = to->value(); + const auto& fromValue = from->value(); + const auto& toValue = to->value(); if (fromValue.isEmpty() || toValue.isEmpty()) { setCachedValue(DynamicVar()); return _cachedValue; diff --git a/Source/moja.flint/src/operationmanagersimple.cpp b/Source/moja.flint/src/operationmanagersimple.cpp index 306dde7..4fff89b 100644 --- a/Source/moja.flint/src/operationmanagersimple.cpp +++ b/Source/moja.flint/src/operationmanagersimple.cpp @@ -9,7 +9,6 @@ #include "moja/flint/operationstocksimple.h" #include "moja/flint/poolsimple.h" -#include #include #include @@ -229,7 +228,7 @@ const IPool* OperationManagerSimple::addPool(const std::string& name, const std: } if (_poolValues.size() == _poolValues.capacity()) { - throw ApplicationException( + throw std::runtime_error( "maximum pool definitions exceeded. Only 255 pools allowed"); // to protect the references held by PoolSimple // wrappers } @@ -260,17 +259,17 @@ const IPool* OperationManagerSimple::addPool(PoolMetaData& metadata, double init const IPool* OperationManagerSimple::getPool(const std::string& name) const { try { - auto r = _poolNameObjectMap.at(name); + auto& r = _poolNameObjectMap.at(name); return r.get(); } catch (...) { - throw PoolNotFoundException() << PoolName(name); + throw std::invalid_argument("Error pool not found " + name); } } // -------------------------------------------------------------------------------------------- const IPool* OperationManagerSimple::getPool(int index) const { - if (index >= _poolObjects.size() || index < 0) throw PoolNotFoundException() << PoolName("Bad index"); + if (index >= _poolObjects.size() || index < 0) throw std::invalid_argument("Error in get pool index out of range"); auto& r = _poolObjects[index]; return r.get(); } diff --git a/Source/moja.flint/src/operationmanagersimplecache.cpp b/Source/moja.flint/src/operationmanagersimplecache.cpp index 8a5dca5..42ae1e1 100644 --- a/Source/moja.flint/src/operationmanagersimplecache.cpp +++ b/Source/moja.flint/src/operationmanagersimplecache.cpp @@ -219,15 +219,15 @@ const IPool* OperationManagerSimpleCache::addPool(PoolMetaData& metadata, double const IPool* OperationManagerSimpleCache::getPool(const std::string& name) const { try { - auto r = _poolNameObjectMap.at(name); + auto& r = _poolNameObjectMap.at(name); return r.get(); } catch (...) { - throw PoolNotFoundException() << PoolName(name); + throw std::invalid_argument("Error pool not found " + name); } } const IPool* OperationManagerSimpleCache::getPool(int index) const { - if (index >= _poolObjects.size() || index < 0) throw PoolNotFoundException() << PoolName("Bad index"); + if (index >= _poolObjects.size() || index < 0) throw std::invalid_argument("Error in get pool index out of range"); auto& r = _poolObjects[index]; return r.get(); } diff --git a/Source/moja.flint/src/outputerstream.cpp b/Source/moja.flint/src/outputerstream.cpp index 4f081bd..3d4218e 100644 --- a/Source/moja.flint/src/outputerstream.cpp +++ b/Source/moja.flint/src/outputerstream.cpp @@ -8,8 +8,7 @@ #include #include - -#include +#include #include @@ -21,6 +20,8 @@ #define DL_CHR "," #define STOCK_PRECISION 15 +namespace fs = moja::filesystem; + namespace moja { namespace flint { @@ -203,10 +204,9 @@ void OutputerStream::outputShutdown(std::ostream& stream) { // -------------------------------------------------------------------------------------------- void OutputerStream::onSystemInit() { - Poco::File outputFile(_fileName); - if (outputFile.exists()) outputFile.remove(); - outputFile.createFile(); - + fs::path outputFile(_fileName); + if (fs::exists(_fileName)) fs::remove(_fileName); + _streamFile.open(_fileName, std::ios::out); _output.addStream(_streamFile); if (_outputToScreen) _output.addStream(std::cout); diff --git a/Source/moja.flint/src/outputerstreamflux.cpp b/Source/moja.flint/src/outputerstreamflux.cpp index 3b895a8..4988af2 100644 --- a/Source/moja.flint/src/outputerstreamflux.cpp +++ b/Source/moja.flint/src/outputerstreamflux.cpp @@ -10,8 +10,7 @@ #include #include #include - -#include +#include #include // std::setprecision #include @@ -20,6 +19,8 @@ #define DL_CHR "," #define STOCK_PRECISION 15 +namespace fs = moja::filesystem; + namespace moja { namespace flint { @@ -109,10 +110,7 @@ void OutputerStreamFlux::outputShutdown(std::ostream& stream) const { // -------------------------------------------------------------------------------------------- void OutputerStreamFlux::onSystemInit() { - Poco::File outputFile(_fileName); - if (outputFile.exists()) outputFile.remove(); - outputFile.createFile(); - + if (fs::exists(_fileName)) fs::remove(_fileName); _streamFile.open(_fileName, std::ios::out); _output.addStream(_streamFile); if (_outputToScreen) _output.addStream(std::cout); diff --git a/Source/moja.flint/src/poolsimplecache.cpp b/Source/moja.flint/src/poolsimplecache.cpp index aac2317..0d2906d 100644 --- a/Source/moja.flint/src/poolsimplecache.cpp +++ b/Source/moja.flint/src/poolsimplecache.cpp @@ -36,7 +36,7 @@ int PoolSimpleCache::idx() const { return _idx; } double PoolSimpleCache::value() const { if (!_initialised) { - throw PreconditionViolatedException() << Precondition("PoolSimpleCache has not been initialised"); + throw std::runtime_error("Error Pool Simple Cache has not been initialized"); } return (*_pools)[_idx]; } diff --git a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp index 5a61a3e..8ac0641 100644 --- a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp +++ b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include @@ -176,55 +175,6 @@ void StatsUnitRecord::mojaLog(const std::string& levelStr, int localDomainId, da #define RETRY_ATTEMPTS 10000 #define RETRY_SLEEP std::chrono::milliseconds(200) -// void StatsUnitRecord::sqliteCreateTable(bool dropExisting, int localDomainId, std::string databaseName, std::string -// tableName) { auto retry = false; auto maxRetries = RETRY_ATTEMPTS; do { try { retry = false; -// SQLite::Connector::registerConnector(); -// Session session("SQLite", databaseName); -// -// if (dropExisting) -// session << "DROP TABLE IF EXISTS " << tableName, now; -// session << "CREATE TABLE IF NOT EXISTS " << tableName << " (id UNSIGNED BIG INT NOT NULL, runId -//VARCHAR(255), elapsedTimeTotal UNSIGNED BIG INT NOT NULL, elapsedTimeFramework UNSIGNED BIG INT NOT NULL, -//elapsedTimeSpinup UNSIGNED BIG INT NOT NULL, elapsedTimeProcessed UNSIGNED BIG INT NOT NULL, unitsTotal UNSIGNED BIG -//INT NOT NULL, unitsProcessed UNSIGNED BIG INT NOT NULL, unitsNotProcessed UNSIGNED BIG INT NOT NULL, unitsWithErrors -//UNSIGNED BIG INT NOT NULL, tileIdx INTEGER, blockIdx INTEGER)", now; -// -// SQLite::Connector::unregisterConnector(); -// } -// catch (SQLite::DBLockedException&) { -// MOJA_LOG_DEBUG << localDomainId << ":DBLockedException - " << maxRetries << " retries -//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << -//localDomainId << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; -// } -// } -// } while (retry); -//} -// -// void StatsUnitRecord::sqliteWrite(std::shared_ptr>& collection, int -// localDomainId, std::string databaseName, std::string tableName) { auto retry = false; auto maxRetries = -//RETRY_ATTEMPTS; do { try { retry = false; SQLite::Connector::registerConnector(); Session session("SQLite", -//databaseName); -// -// // -- collection -// if (collection->size() != 0) { -// MOJA_LOG_DEBUG << localDomainId << ":SQLite " << tableName << " - inserted " << -//collection->size() << " records"; session.begin(); session << "INSERT INTO " << tableName << " VALUES(?, ?, ?, ?, ?, -//?, ?, ?, ?, ?, ?, ?)", bind(collection->getPersistableCollection()), now; session.commit(); -// } -// -// SQLite::Connector::unregisterConnector(); -// } -// catch (SQLite::DBLockedException&) { -// MOJA_LOG_DEBUG << localDomainId << ":DBLockedException - " << maxRetries << " retries -//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << -//localDomainId << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; -// } -// } -// } while (retry); -//} - -// -- - // -------------------------------------------------------------------------------------------- class SpatialTiledLocalDomainController::InternalThreadBlocks { @@ -791,48 +741,20 @@ bool SpatialTiledLocalDomainController::runCellSpinUp(std::shared_ptr_unitsProcessed++; return true; } catch (const flint::SimulationError& e) { - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - const int* errorCode = boost::get_error_info(e); - _spatiallocationinfo->_errorCode = *errorCode; - _spatiallocationinfo->_library = libraryName; - _spatiallocationinfo->_module = moduleName; - _spatiallocationinfo->_message = details; - _spinupNotificationCenter.postNotification(moja::signals::Error, details); + _spatiallocationinfo->_errorCode = *boost::get_error_info(e); + _spatiallocationinfo->_library = *boost::get_error_info(e); + _spatiallocationinfo->_module = *boost::get_error_info(e); + _spatiallocationinfo->_message = *boost::get_error_info
(e); + _spinupNotificationCenter.postNotification(moja::signals::Error, *boost::get_error_info
(e)); return true; } catch (const flint::LocalDomainError& e) { - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - const int* errorCode = boost::get_error_info(e); - _spatiallocationinfo->_errorCode = *errorCode; - _spatiallocationinfo->_library = libraryName; - _spatiallocationinfo->_module = moduleName; - _spatiallocationinfo->_message = details; - _spinupNotificationCenter.postNotification(moja::signals::Error, details); - } catch (flint::VariableNotFoundException& e) { - std::string str = - ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); - } catch (flint::VariableEmptyWhenValueExpectedException& e) { - std::string str = - ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); - blockStatsUnit->_unitsNotProcessed++; - blockStatsUnit->_unitsWithError++; - return true; - } catch (const Poco::Exception& e) { - std::string str = ((boost::format("In Spinup: %1%") % e.displayText()).str()); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); - } catch (const boost::exception& e) { - std::string str = ((boost::format("In Spinup: %1%") % boost::diagnostic_information(e)).str()); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); - } catch (const Exception& e) { - std::string str = e.displayText(); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); + _spatiallocationinfo->_errorCode = *boost::get_error_info(e); + _spatiallocationinfo->_library = *boost::get_error_info(e); + _spatiallocationinfo->_module = *boost::get_error_info(e); + _spatiallocationinfo->_message = *boost::get_error_info
(e); + _spinupNotificationCenter.postNotification(moja::signals::Error, *boost::get_error_info
(e)); } catch (const std::exception& e) { - std::string str = e.what(); + const std::string str = e.what(); _spinupNotificationCenter.postNotification(moja::signals::Error, str); } catch (...) { _spinupNotificationCenter.postNotification(moja::signals::Error, "unknown exception"); @@ -955,28 +877,8 @@ bool SpatialTiledLocalDomainController::runCell(std::shared_ptr _spatiallocationinfo->_module = moduleName; _spatiallocationinfo->_message = details; _notificationCenter.postNotification(moja::signals::Error, details); - } catch (flint::VariableNotFoundException& e) { - std::string str = ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); - _notificationCenter.postNotification(moja::signals::Error, str); - } catch (flint::VariableEmptyWhenValueExpectedException& e) { - std::string str = ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); - _notificationCenter.postNotification(moja::signals::Error, str); - blockStatsUnit->_unitsNotProcessed++; - blockStatsUnit->_unitsWithError++; - return true; - } catch (const Poco::Exception& e) { - std::string str = ((boost::format("In Main: %1%") % e.displayText()).str()); - _notificationCenter.postNotification(moja::signals::Error, str); - } catch (const boost::exception& e) { - std::string str = boost::diagnostic_information(e); - _notificationCenter.postNotification(moja::signals::Error, str); - } catch (const Exception& e) { - std::string str = e.displayText(); - blockStatsUnit->_unitsNotProcessed++; - blockStatsUnit->_unitsWithError++; - _notificationCenter.postNotification(moja::signals::Error, str); } catch (const std::exception& e) { - std::string str = e.what(); + const std::string str = e.what(); _notificationCenter.postNotification(moja::signals::Error, str); } catch (...) { _notificationCenter.postNotification(moja::signals::Error, "unknown exception"); @@ -1193,50 +1095,6 @@ void SpatialTiledLocalDomainController::run() { } _notificationCenter.postNotification(signals::LocalDomainProcessingUnitShutdown); // End Processing unit } - } catch (const Exception& e) { - _spatiallocationinfo->_errorCode = 1; - MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting..." - << "Module: " << _spatiallocationinfo->_module << ", " - << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx - << "," << _spatiallocationinfo->_cellIdx << ", " - << "msg:" << e.displayText(); - exit(1); - } catch (const boost::exception& e) { - _spatiallocationinfo->_errorCode = 1; - MOJA_LOG_FATAL << "boost::exception caught at LocalDomain level, exiting..." - << "Module: " << _spatiallocationinfo->_module << ", " - << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx - << "," << _spatiallocationinfo->_cellIdx << ", " - << "msg:" << boost::diagnostic_information(e); - exit(1); - } catch (const Poco::FileException& e) { - _spatiallocationinfo->_errorCode = 1; - MOJA_LOG_FATAL << "Poco::FileException caught at LocalDomain level, exiting..." - << "Module: " << _spatiallocationinfo->_module << ", " - << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx - << "," << _spatiallocationinfo->_cellIdx << ", " - << "msg:" << e.displayText(); - ; - exit(1); - } - // catch (const Poco::Data::SQLite::SQLiteException& e) { - // _spatiallocationinfo->_errorCode = 1; - // MOJA_LOG_FATAL - // << "Poco::Data::SQLite::SQLiteException caught at LocalDomain level, exiting..." - // << "Module: " << _spatiallocationinfo->_module << ", " - // << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx << - //"," << _spatiallocationinfo->_cellIdx << ", " - // << "msg:" << e.displayText(); - // exit(1); - //} - catch (const Poco::Exception& e) { - _spatiallocationinfo->_errorCode = 1; - MOJA_LOG_FATAL << "Poco::Exception caught at LocalDomain level, exiting..." - << "Module: " << _spatiallocationinfo->_module << ", " - << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx - << "," << _spatiallocationinfo->_cellIdx << ", " - << "msg: " << e.displayText(); - exit(1); } catch (const std::exception& e) { _spatiallocationinfo->_errorCode = 1; MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." diff --git a/Source/moja.flint/src/spinuplandunitcontroller.cpp b/Source/moja.flint/src/spinuplandunitcontroller.cpp index 8258efc..25eda1e 100644 --- a/Source/moja.flint/src/spinuplandunitcontroller.cpp +++ b/Source/moja.flint/src/spinuplandunitcontroller.cpp @@ -87,7 +87,7 @@ void SpinupLandUnitController::addVariable(std::string name, std::shared_ptrsecond.get(); } @@ -95,7 +95,7 @@ IVariable* SpinupLandUnitController::getVariable(const std::string& name) { const IVariable* SpinupLandUnitController::getVariable(const std::string& name) const { const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { - BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); + std::runtime_error("Error in spin up landunit controller variable not found " + name); } return v->second.get(); } diff --git a/Source/moja.flint/src/sqlquerytransform.cpp b/Source/moja.flint/src/sqlquerytransform.cpp index 759a7f2..77ac6e6 100644 --- a/Source/moja.flint/src/sqlquerytransform.cpp +++ b/Source/moja.flint/src/sqlquerytransform.cpp @@ -9,9 +9,7 @@ #include #include -#include - -#include +#include #include @@ -68,8 +66,9 @@ void SQLQueryTransform::configure(DynamicObject config, const ILandUnitControlle } std::string SQLQueryTransform::readSQLFile(const std::string& path) { - if (!Poco::File(path).exists()) { - throw FileNotFoundException(path); + namespace fs = moja::filesystem; + if (!fs::exists(path)) { + throw std::runtime_error("SQL file not found " + path); } std::ifstream file(path); @@ -169,7 +168,7 @@ std::string SQLQueryTransform::formatVariableValues(const IVariable& var, Dynami if (_allowEmptyVarValues) { return "NULL"; } else { - BOOST_THROW_EXCEPTION(VariableEmptyWhenValueExpectedException() << VariableName(var.info().name)); + throw std::runtime_error("SQL value NULL when value expected"); } } diff --git a/Source/moja.flint/src/writesystemconfig.cpp b/Source/moja.flint/src/writesystemconfig.cpp index 849e727..14ce4bf 100644 --- a/Source/moja.flint/src/writesystemconfig.cpp +++ b/Source/moja.flint/src/writesystemconfig.cpp @@ -20,18 +20,36 @@ #include #include #include +#include -#include -#include +#include -#include +#include #include #include +namespace fs = moja::filesystem; + namespace moja { namespace flint { + // --- RAII class for file handle +class FileHandle { + typedef FILE* ptr; + + public: + explicit FileHandle(const moja::filesystem::path& name, std::string const& mode = std::string("r")) + : _wrapped_file(fopen(name.string().c_str(), mode.c_str())) {} + ~FileHandle() { + if (_wrapped_file) fclose(_wrapped_file); + } + operator ptr() const { return _wrapped_file; } + + private: + ptr _wrapped_file; +}; + // -------------------------------------------------------------------------------------------- void WriteSystemConfig::configure(const DynamicObject& config) { @@ -76,25 +94,16 @@ void WriteSystemConfig::subscribe(NotificationCenter& notificationCenter) { // -------------------------------------------------------------------------------------------- void WriteSystemConfig::onSystemInit() { - Poco::File workingFolder(_outputPath); - if (!workingFolder.exists()) { - try { - workingFolder.createDirectories(); - } catch (Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed - in 1.7.8 */ - } + fs::path workingFolder(_outputPath); + if (!fs::exists(workingFolder)) { + fs::create_directories(workingFolder); } - if (workingFolder.exists() && !workingFolder.isDirectory()) { - MOJA_LOG_ERROR << "Error creating spatial tiled point configurations output folder: " << _outputPath; + if (fs::exists(workingFolder) && !fs::is_directory(workingFolder)) { + MOJA_LOG_ERROR << "Error creating spatial tiled point configurations output folder: " << workingFolder.string(); } - auto outputFolderPath = (boost::format("%1%%2%%3%") % workingFolder.path() % Poco::Path::separator() % _name).str(); - Poco::File outputFolder(outputFolderPath); - if (!outputFolder.exists()) { - try { - outputFolder.createDirectories(); - } catch (Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed - in 1.7.8 */ - } + auto outputFolderPath = workingFolder / _name; + if (!fs::exists(outputFolderPath)) { + fs::create_directories(outputFolderPath); } } @@ -256,7 +265,7 @@ void outputDynamicToStream(std::ofstream& fout, const DynamicVar& object, int le if (object.type() == typeid(DateTime)) { DateTime dt = object.extract(); std::string simpleDateStr = - (boost::format("{ \"$date\": \"%1%/%2%/%3%\" }") % dt.year() % dt.month() % dt.day()).str(); + fmt::format("{ \"$date\": \"{}/{}/{}\" }", dt.year(), dt.month(), dt.day()); fout << simpleDateStr; // fout << "\"" << escape_json(simpleDateStr) << "\""; } else if (object.type() == typeid(Int16)) { @@ -313,18 +322,13 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { auto timestep = timing->step(); auto timesubstep = timing->subStep(); - Poco::File workingFolder(_outputPath); auto configFilename = - (boost::format("%1%%2%%3%%4%%5%_%6%_%7%_%8%_%9%_%10%.json") % workingFolder.path() % Poco::Path::separator() % - _name % Poco::Path::separator() % - boost::io::group(std::setfill('0'), std::setw(5), _spatialLocationInfo ? _spatialLocationInfo->_tileIdx : 0) % - boost::io::group(std::setfill('0'), std::setw(3), _spatialLocationInfo ? _spatialLocationInfo->_blockIdx : 0) % - boost::io::group(std::setfill('0'), std::setw(6), _spatialLocationInfo ? _spatialLocationInfo->_cellIdx : 0) % - notificationStr % timestep % timesubstep) - .str(); - - Poco::File configFile(configFilename); - if (configFile.exists()) configFile.remove(false); // delete existing config file + fs::path(_outputPath) / _name / + fmt::format("{:05}_{:03}_{:06}_{}_{}_{}.json", _spatialLocationInfo ? _spatialLocationInfo->_tileIdx : 0, + _spatialLocationInfo ? _spatialLocationInfo->_blockIdx : 0, + _spatialLocationInfo ? _spatialLocationInfo->_cellIdx : 0, notificationStr, timestep, timesubstep); + + if (fs::exists(configFilename)) fs::remove(configFilename); // delete existing config file // configFile.createFile() FileHandle pFile(configFilename, "wb"); @@ -353,15 +357,12 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { DynamicObject localdomain; auto localDomainConfig = config->localDomain(); localdomain["type"] = configuration::LocalDomain::localDomainTypeToStr(localDomainConfig->type()); - localdomain["start_date_init"] = (boost::format("%1%/%2%/%3%") % config->startDate().year() % - config->startDate().month() % config->startDate().day()) - .str(); - localdomain["start_date"] = (boost::format("%1%/%2%/%3%") % timing->curStartDate().year() % - timing->curStartDate().month() % timing->curStartDate().day()) - .str(); + localdomain["start_date_init"] = fmt::format("{}/{}/{}", config->startDate().year(), + config->startDate().month(), config->startDate().day()); + localdomain["start_date"] = fmt::format("{}/{}/{}", timing->curStartDate().year(), + timing->curStartDate().month(), timing->curStartDate().day()); localdomain["end_date"] = - (boost::format("%1%/%2%/%3%") % config->endDate().year() % config->endDate().month() % config->endDate().day()) - .str(); + fmt::format("{}/{}/{}", config->endDate().year(), config->endDate().month(), config->endDate().day()); localdomain["sequencer_library"] = localDomainConfig->sequencerLibrary(); localdomain["sequencer"] = localDomainConfig->sequencer(); localdomain["simulateLandUnit"] = localDomainConfig->simulateLandUnit(); diff --git a/Source/moja.flint/src/writevariablegrid.cpp b/Source/moja.flint/src/writevariablegrid.cpp index 2396040..3a644bd 100644 --- a/Source/moja.flint/src/writevariablegrid.cpp +++ b/Source/moja.flint/src/writevariablegrid.cpp @@ -9,10 +9,9 @@ #include #include +#include -#include -#include - +#include #include #include #include @@ -21,9 +20,27 @@ #include #include +namespace fs = moja::filesystem; + namespace moja { namespace flint { + // --- RAII class for file handle +class FileHandle { + typedef FILE* ptr; + + public: + explicit FileHandle(const fs::path& name, std::string const& mode = std::string("r")) + : _wrapped_file(fopen(name.string().c_str(), mode.c_str())) {} + ~FileHandle() { + if (_wrapped_file) fclose(_wrapped_file); + } + operator ptr() const { return _wrapped_file; } + + private: + ptr _wrapped_file; +}; + void WriteVariableGrid::configure(const DynamicObject& config) { _globalOutputPath = config.contains("output_path") ? config["output_path"].convert() : ""; @@ -298,32 +315,12 @@ template void WriteVariableGrid::DataSettingsT::doSystemInit(flint::ILandUnitDataWrapper* _landUnitData) { flint::VariableAndPoolStringBuilder databaseNameBuilder(_landUnitData, _outputPath); _outputPath = databaseNameBuilder.result(); + const fs::path working_folder(_outputPath); - std::string variableFolder; if (_forceVariableFolderName) { - variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); + fs::create_directories(working_folder / _name); } else { - variableFolder = ""; - } - - Poco::File workingFolder(_outputPath); - const auto spatialOutputFolderPath = (boost::format("%1%%2%") % workingFolder.path() % variableFolder - // % Poco::Path::separator() - // % _name - ) - .str(); - - try { - workingFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ - } - - Poco::File spatialOutputFolder(spatialOutputFolderPath); - try { - spatialOutputFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + fs::create_directories(working_folder); } } @@ -350,43 +347,25 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainInit(flint::ILandUnitData template void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitInit( std::shared_ptr spatialLocationInfo) { - Poco::File workingFolder(_outputPath); + fs::path working_folder(_outputPath); std::string variableFolder; if (_forceVariableFolderName) { - variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); - } else { - variableFolder = ""; + working_folder /= _name; } + fs::path tile_folder; if (_useIndexesForFolderName) { - _tileFolderPath = (boost::format("%1%%2%%3%%4%") % - workingFolder.path() - // % Poco::Path::separator() - // % _name - % variableFolder % Poco::Path::separator() % - boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx)) - .str(); + tile_folder = (working_folder /= fmt::format("{:06}", spatialLocationInfo->_tileIdx)); } else { - _tileFolderPath = - (boost::format("%1%%2%%3%%4%%5%_%6%%7%") % - workingFolder.path() - // % Poco::Path::separator() - // % _name - % variableFolder % Poco::Path::separator() % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % - (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat))) - .str(); - } - - Poco::File tileFolder(_tileFolderPath); - std::unique_lock lock(_fileHandlingMutex); - try { - tileFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ - } + tile_folder = (working_folder /= fmt::format("{}{:03}{}{:03}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lon), + spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lat))); + } + _tileFolderPath = tile_folder.string(); + std::unique_lock lock(_fileHandlingMutex); + fs::create_directories(tile_folder); } // -------------------------------------------------------------------------------------------- @@ -407,36 +386,27 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( // for (const auto& timestepData : _data) { typename std::unordered_map>::iterator itPrev; + for (auto it = _data.begin(); it != _data.end(); ++it) { - const auto& timestepData = (*it); + const auto& [step, data] = (*it); - Poco::File tileFolder(_tileFolderPath); + fs::path tileFolder(_tileFolderPath); std::string folderLocStr; if (_useIndexesForFolderName) { - folderLocStr = (boost::format("%1%_%2%") % - boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx) % - boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) - .str(); + folderLocStr = fmt::format("{}_{:06}_{:02}_{}", _name, spatialLocationInfo->_tileIdx, + spatialLocationInfo->_blockIdx, step); } else { - folderLocStr = - (boost::format("%1%%2%_%3%%4%_%5%") % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % - (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat)) % - boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) - .str(); + folderLocStr = fmt::format( + "{}_{}{:03}_{}{:03}_{:02}_{}", _name, spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lon), spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lat), spatialLocationInfo->_blockIdx, step); } - auto filenameGrd = (boost::format("%1%%2%%3%_%4%_%5%.grd") % tileFolder.path() % Poco::Path::separator() % _name % - folderLocStr % timestepData.first) - .str(); - auto filenameHdr = (boost::format("%1%%2%%3%_%4%_%5%.hdr") % tileFolder.path() % Poco::Path::separator() % _name % - folderLocStr % timestepData.first) - .str(); + auto filenameGrd = tileFolder / fmt::format("{}.grd", folderLocStr); + auto filenameHdr = tileFolder / fmt::format("{}.hdr", folderLocStr); - Poco::File blockGrd(filenameGrd); - if (blockGrd.exists()) { - blockGrd.remove(false); // delete existing file + if (fs::exists(filenameGrd)) { + fs::remove(filenameGrd); // delete existing file } int cellRows = spatialLocationInfo->_cellRows; @@ -446,21 +416,21 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( FileHandle pFile(filenameGrd, "wb"); if (_subtractPrevValue) { if (it == _data.begin()) // prev value is 0 - fwrite(timestepData.second.data(), sizeof(T), numCells, pFile); + fwrite(data.data(), sizeof(T), numCells, pFile); else { // I know i have a prevIt - const auto& timestepDataPrev = (*itPrev); + const auto& [step_prev, data_prev] = (*itPrev); std::vector newData; - auto it1 = timestepData.second.begin(); - auto it2 = timestepDataPrev.second.begin(); - for (; it1 != timestepData.second.end() && it2 != timestepDataPrev.second.end(); ++it1, ++it2) { + auto it1 = data.begin(); + auto it2 = data_prev.begin(); + for (; it1 != data.end() && it2 != data_prev.end(); ++it1, ++it2) { newData.push_back(*it1 - *it2); } fwrite(newData.data(), sizeof(T), numCells, pFile); } } else { - fwrite(timestepData.second.data(), sizeof(T), numCells, pFile); + fwrite(data.data(), sizeof(T), numCells, pFile); } std::ofstream out(filenameHdr); diff --git a/Source/moja.modules.gdal/src/rasterreadergdal.cpp b/Source/moja.modules.gdal/src/rasterreadergdal.cpp index aeee1dc..21c250a 100644 --- a/Source/moja.modules.gdal/src/rasterreadergdal.cpp +++ b/Source/moja.modules.gdal/src/rasterreadergdal.cpp @@ -16,7 +16,6 @@ #include #include -#include namespace moja { namespace modules { @@ -42,10 +41,8 @@ MetaDataRasterReaderGDAL::MetaDataRasterReaderGDAL(const std::string& path, cons const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { try { - auto filePath = Poco::Path(path); - auto parent = filePath.parent().toString(); - auto abs = filePath.parent().absolute().toString(); - _path = (boost::format("%1%%2%.json") % abs % filePath.getBaseName()).str(); + const auto filePath = fs::absolute(path).replace_extension(".json"); + _path = filePath.string(); _metaDataRequired = true; if (settings.contains("metadata_required")) { _metaDataRequired = settings["metadata_required"].extract(); diff --git a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp index a8f432e..015ae0d 100644 --- a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp +++ b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp @@ -1,7 +1,5 @@ #include "moja/modules/gdal/writevariablegeotiff.h" -#include "moja/exception.h" - #include #include #include @@ -11,9 +9,9 @@ #include #include +#include -#include -#include +#include #include #include @@ -24,6 +22,8 @@ #include #include +namespace fs = moja::filesystem; + namespace moja { namespace modules { namespace gdal { @@ -303,32 +303,11 @@ void WriteVariableGeotiff::DataSettingsT::doSystemInit(flint::ILandUnitDataWr flint::VariableAndPoolStringBuilder databaseNameBuilder(_landUnitData, _outputPath); _outputPath = databaseNameBuilder.result(); - std::string variableFolder; + fs::path spatialOutputFolderPath(_outputPath); if (_forceVariableFolderName) { - variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); - } else { - variableFolder = ""; - } - - Poco::File workingFolder(_outputPath); - const auto spatialOutputFolderPath = (boost::format("%1%%2%") % workingFolder.path() % variableFolder - // % Poco::Path::separator() - // % _name - ) - .str(); - - try { - workingFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ - } - - Poco::File spatialOutputFolder(spatialOutputFolderPath); - try { - spatialOutputFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + spatialOutputFolderPath /= _name; } + fs::create_directories(spatialOutputFolderPath); } // -------------------------------------------------------------------------------------------- @@ -354,43 +333,24 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainInit(flint::ILandUnitD template void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitInit( std::shared_ptr spatialLocationInfo) { - Poco::File workingFolder(_outputPath); - std::string variableFolder; + fs::path workingFolder(_outputPath); if (_forceVariableFolderName) { - variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); - } else { - variableFolder = ""; + workingFolder /= _name; } + fs::path tileFolder; if (_useIndexesForFolderName) { - _tileFolderPath = (boost::format("%1%%2%%3%%4%") % - workingFolder.path() - // % Poco::Path::separator() - // % _name - % variableFolder % Poco::Path::separator() % - boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx)) - .str(); + tileFolder = workingFolder / fmt::format("{:06}", spatialLocationInfo->_tileIdx); } else { - _tileFolderPath = - (boost::format("%1%%2%%3%%4%%5%_%6%%7%") % - workingFolder.path() - // % Poco::Path::separator() - // % _name - % variableFolder % Poco::Path::separator() % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % - (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat))) - .str(); - } - - Poco::File tileFolder(_tileFolderPath); - std::unique_lock lock(_fileHandlingMutex); - try { - tileFolder.createDirectories(); - } catch ( - Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + tileFolder = workingFolder / fmt::format("{}{:03}_{}{:03}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lon), + spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lat)); } + + std::unique_lock lock(_fileHandlingMutex); + fs::create_directories(tileFolder); } // -------------------------------------------------------------------------------------------- @@ -448,7 +408,7 @@ struct dataset_closer { GDALDataset* gdal_dataset_; }; -static std::shared_ptr create_gdalraster(const Poco::File& path, int rows, int cols, +static std::shared_ptr create_gdalraster(const fs::path& path, int rows, int cols, GDALDataType datatype, double* transform) { if (GDALGetDriverCount() == 0) { GDALAllRegister(); @@ -460,11 +420,9 @@ static std::shared_ptr create_gdalraster(const Poco::File& path, options = CSLSetNameValue(options, "TILED", "YES"); options = CSLSetNameValue(options, "COMPRESS", "DEFLATE"); - auto dataset = driver->Create(path.path().c_str(), cols, rows, 1, datatype, options); + auto dataset = driver->Create(path.string().c_str(), cols, rows, 1, datatype, options); if (dataset == nullptr) { - std::ostringstream oss; - oss << "Could not create raster file: " << path.path() << std::endl; - throw ApplicationException(oss.str()); + throw std::runtime_error("Could not create raster file: " + path.string()); } dataset->SetGeoTransform(transform); OGRSpatialReference srs; @@ -480,23 +438,17 @@ static std::shared_ptr create_gdalraster(const Poco::File& path, template void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown( std::shared_ptr spatialLocationInfo) { - std::unique_lock lock(_fileHandlingMutex); + std::unique_lock lock(_fileHandlingMutex); - Poco::File tileFolder(_tileFolderPath); + fs::path tileFolder(_tileFolderPath); std::string folderLocStr; if (_useIndexesForFolderName) { - folderLocStr = - (boost::format("%1%_%2%") % boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx) % - boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) - .str(); + folderLocStr = fmt::format("{:06}_{:02}", spatialLocationInfo->_tileIdx, spatialLocationInfo->_blockIdx); } else { - folderLocStr = - (boost::format("%1%%2%_%3%%4%_%5%") % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % - (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % - boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat)) % - boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) - .str(); + folderLocStr = fmt::format("{}{:03}_{}{:03}_{:02}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lon), + spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", + std::abs(spatialLocationInfo->_tileLatLon.lat), spatialLocationInfo->_blockIdx); } int cellRows = spatialLocationInfo->_cellRows; @@ -510,42 +462,38 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown typename std::unordered_map>::iterator itPrev; for (auto it = _data.begin(); it != _data.end(); ++it) { - auto& timestepData = (*it); - - auto filename = (boost::format("%1%%2%%3%_%4%_%5%.tif") % tileFolder.path() % Poco::Path::separator() % _name % - folderLocStr % timestepData.first) - .str(); + const auto& [step, data] = (*it); - Poco::File block_path(filename); - if (block_path.exists()) { - block_path.remove(false); // delete existing file + auto block_path = tileFolder /= fmt::format("{}_{}_{}.tif", _name, folderLocStr, step); + if (fs::exists(block_path)) { + fs::remove(block_path); // delete existing file } auto band = create_gdalraster(block_path, cellRows, cellCols, gdal_type(_dataType), adfGeoTransform); auto err = band->SetNoDataValue(_nodataValue); if (err != CE_None) { std::ostringstream oss; - oss << "Could not set raster file nodata: " << block_path.path() << " (" << _nodataValue << ")" << std::endl; - throw ApplicationException(oss.str()); + oss << "Could not set raster file nodata: " << block_path.string() << " (" << _nodataValue << ")" << std::endl; + throw std::runtime_error(oss.str()); } if (_subtractPrevValue) { if (it == _data.begin()) { // prev value is 0 - auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, timestepData.second.data(), cellCols, + auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, (void*)data.data(), cellCols, cellRows, gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.path() << std::endl; - throw ApplicationException(oss.str()); + oss << "Could not write to raster file: " << block_path.string() << std::endl; + throw std::runtime_error(oss.str()); } } else { // I know i have a prevIt - const auto& timestepDataPrev = (*itPrev); + const auto& [step_prev, data_prev] = (*itPrev); std::vector newData; - auto it1 = timestepData.second.begin(); - auto it2 = timestepDataPrev.second.begin(); - for (; it1 != timestepData.second.end() && it2 != timestepDataPrev.second.end(); ++it1, ++it2) { + auto it1 = data.begin(); + auto it2 = data_prev.begin(); + for (; it1 != data.end() && it2 != data_prev.end(); ++it1, ++it2) { newData.push_back(*it1 - *it2); } @@ -553,17 +501,17 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.path() << std::endl; - throw ApplicationException(oss.str()); + oss << "Could not write to raster file: " << block_path.string() << std::endl; + throw std::runtime_error(oss.str()); } } } else { - auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, timestepData.second.data(), cellCols, + auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, (void*)data.data(), cellCols, cellRows, gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.path() << std::endl; - throw ApplicationException(oss.str()); + oss << "Could not write to raster file: " << block_path.string() << std::endl; + throw std::runtime_error(oss.str()); } } itPrev = it; From 91965af60235574e53a2c0f73c41459540d37547 Mon Sep 17 00:00:00 2001 From: Mal Date: Fri, 18 Jun 2021 10:57:11 +1000 Subject: [PATCH 38/40] Finished cleaning up exceptions --- Source/moja.cli/src/moja.cpp | 4 +- Source/moja.core/CMakeLists.txt | 1 + Source/moja.core/include/moja/status.h | 62 ++++++ Source/moja.datarepository/CMakeLists.txt | 1 - .../datarepository/datarepositoryexceptions.h | 60 ------ .../include/moja/datarepository/stack.h | 1 - .../datarepository/tileblockcellindexer.h | 24 +-- .../src/aspatialtileinfo.cpp | 5 +- .../src/aspatialtileinfocollection.cpp | 5 +- .../src/datarepository.cpp | 14 +- .../moja.datarepository/src/landunitinfo.cpp | 6 +- .../src/providerrelationalsqlite.cpp | 10 +- .../src/providerspatialrastertiled.cpp | 10 +- .../moja.datarepository/src/rasterreader.cpp | 7 +- .../flint/aspatiallocaldomaincontroller.h | 10 +- .../include/moja/flint/flintexceptions.h | 77 +++++--- .../moja/flint/ilocaldomaincontroller.h | 35 ++-- .../moja/flint/localdomaincontrollerbase.h | 14 +- .../flint/spatialtiledlocaldomaincontroller.h | 10 +- .../threadedaspatiallocaldomaincontroller.h | 11 +- .../src/aspatiallocaldomaincontroller.cpp | 64 ++++--- .../src/localdomaincontrollerbase.cpp | 104 +++-------- .../moja.flint/src/operationmanagersimple.cpp | 18 +- .../src/spatialtiledlocaldomaincontroller.cpp | 176 +++++------------- Source/moja.flint/src/sqlquerytransform.cpp | 1 + .../threadedaspatiallocaldomaincontroller.cpp | 12 +- Source/moja.flint/src/writevariablegrid.cpp | 2 +- .../src/rasterreadergdal.cpp | 32 +--- .../src/providerrelationallibpqpostgresql.cpp | 7 +- 29 files changed, 315 insertions(+), 468 deletions(-) create mode 100644 Source/moja.core/include/moja/status.h delete mode 100644 Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h diff --git a/Source/moja.cli/src/moja.cpp b/Source/moja.cli/src/moja.cpp index 73ad5c8..c280cd1 100644 --- a/Source/moja.cli/src/moja.cpp +++ b/Source/moja.cli/src/moja.cpp @@ -288,11 +288,11 @@ int main(int argc, char* argv[]) { << ", Version: " << ldc->landUnitController().operationManager()->version() << ", Config: " << ldc->landUnitController().operationManager()->config(); - ldc->_notificationCenter.postNotification(moja::signals::SystemInit); + ldc->notification_center().postNotification(moja::signals::SystemInit); ldc->startup(); ldc->run(); ldc->shutdown(); - ldc->_notificationCenter.postNotification(moja::signals::SystemShutdown); + ldc->notification_center().postNotification(moja::signals::SystemShutdown); } catch (const std::exception& e) { MOJA_LOG_FATAL << e.what(); return EXIT_FAILURE; diff --git a/Source/moja.core/CMakeLists.txt b/Source/moja.core/CMakeLists.txt index e63cb9e..ff9aa69 100644 --- a/Source/moja.core/CMakeLists.txt +++ b/Source/moja.core/CMakeLists.txt @@ -94,6 +94,7 @@ set(MOJA_Core_headers include/moja/pocojsonutils.h include/moja/signals.h include/moja/stopwatch.h + include/moja/status.h include/moja/timespan.h include/moja/types.h include/moja/version.h diff --git a/Source/moja.core/include/moja/status.h b/Source/moja.core/include/moja/status.h new file mode 100644 index 0000000..d6b3a08 --- /dev/null +++ b/Source/moja.core/include/moja/status.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +namespace moja { + +enum class status_code { + /// Not an error; returned on success. + Ok = 0, + + Error = 1, + Cancelled = 2, + Unknown = 3, +}; + +std::string StatusCodeToString(status_code code); +std::ostream& operator<<(std::ostream& os, status_code code); + +/** + * Reports error code and details. + * + * This class is modeled after `grpc::Status`. + */ +class status { + public: + status() = default; + + explicit status(status_code status_code, std::string message = "") + : code_(status_code), message_(std::move(message)) {} + + bool ok() const { return code_ == status_code::Ok; } + + status_code code() const { return code_; } + const std::string& message() const { return message_; } + + private: + status_code code_{status_code::Ok}; + std::string message_; +}; + +inline std::ostream& operator<<(std::ostream& os, status const& rhs) { + return os << rhs.message() << " [" << StatusCodeToString(rhs.code()) << "]"; +} + +inline bool operator==(status const& lhs, status const& rhs) { + return lhs.code() == rhs.code() && lhs.message() == rhs.message(); +} + +inline bool operator!=(status const& lhs, status const& rhs) { return !(lhs == rhs); } + +class runtime_status_error : public std::runtime_error { + public: + explicit runtime_status_error(status status); + + status const& status() const { return status_; } + + private: + moja::status status_; +}; + +} // namespace moja diff --git a/Source/moja.datarepository/CMakeLists.txt b/Source/moja.datarepository/CMakeLists.txt index ff090ac..23863af 100644 --- a/Source/moja.datarepository/CMakeLists.txt +++ b/Source/moja.datarepository/CMakeLists.txt @@ -75,7 +75,6 @@ set(MOJA_DataRepository_headers include/moja/${PACKAGE}/aspatialtileinfocollection.h include/moja/${PACKAGE}/cube.h include/moja/${PACKAGE}/datarepository.h - include/moja/${PACKAGE}/datarepositoryexceptions.h include/moja/${PACKAGE}/iproviderinterface.h include/moja/${PACKAGE}/iprovidernosqlinterface.h include/moja/${PACKAGE}/iproviderrelationalinterface.h diff --git a/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h b/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h deleted file mode 100644 index bc56cf1..0000000 --- a/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ -#define MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ - -#include "moja/datarepository/_datarepository_exports.h" - -#include - -#include - -namespace moja { -namespace datarepository { - -struct DATAREPOSITORY_API DataRepositoryException : virtual std::exception, virtual boost::exception {}; - -struct DATAREPOSITORY_API AssertionViolationException : virtual DataRepositoryException {}; -typedef boost::error_info AssertMsg; - -struct DATAREPOSITORY_API ConnectionFailedException : virtual DataRepositoryException {}; -typedef boost::error_info ConnectionError; - -struct DATAREPOSITORY_API FileNotFoundException : virtual DataRepositoryException {}; -typedef boost::error_info FileName; - -struct DATAREPOSITORY_API FileReadException : virtual DataRepositoryException {}; -typedef boost::error_info FileName; -typedef boost::error_info Message; - -struct DATAREPOSITORY_API LayerNotFoundException : virtual DataRepositoryException {}; -typedef boost::error_info LayerName; - -struct DATAREPOSITORY_API NotImplementedException : virtual DataRepositoryException {}; -typedef boost::error_info Message; - -struct DATAREPOSITORY_API QueryException : virtual DataRepositoryException {}; -typedef boost::error_info SQL; -typedef boost::error_info SQLError; - -struct DATAREPOSITORY_API LandscapeDefinitionException : virtual DataRepositoryException {}; -typedef boost::error_info Component; -typedef boost::error_info Constraint; - -struct DATAREPOSITORY_API TileBlockCellIndexerInvalidParameterException : virtual DataRepositoryException {}; -typedef boost::error_info Component; -typedef boost::error_info Constraint; - -// -- Provider exceptions -struct DATAREPOSITORY_API ProviderUnsupportedException : virtual DataRepositoryException {}; -typedef boost::error_info ProviderType; - -struct DATAREPOSITORY_API ProviderNotFoundException : virtual DataRepositoryException {}; -typedef boost::error_info ProviderType; - -struct DATAREPOSITORY_API ProviderAlreadyExistsException : virtual DataRepositoryException {}; -typedef boost::error_info ProviderName; -typedef boost::error_info ProviderType; - -} // namespace datarepository -} // namespace moja - -#endif // MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ diff --git a/Source/moja.datarepository/include/moja/datarepository/stack.h b/Source/moja.datarepository/include/moja/datarepository/stack.h index be310cd..828addf 100644 --- a/Source/moja.datarepository/include/moja/datarepository/stack.h +++ b/Source/moja.datarepository/include/moja/datarepository/stack.h @@ -2,7 +2,6 @@ #define MOJA_DATAREPOSITORY_STACK_H_ #include "moja/datarepository/cube.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include diff --git a/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h b/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h index ed11db7..73bea04 100644 --- a/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h +++ b/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h @@ -2,16 +2,16 @@ #define MOJA_DATAREPOSITORY_TILEBLOCKCELLINDEXER_H_ #include "moja/datarepository/_datarepository_exports.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include #include -#include #include #include +#include #include #include +#include namespace moja { namespace datarepository { @@ -183,16 +183,14 @@ struct DATAREPOSITORY_API BlockIdx { if (t_idx > indexer.tileDesc.indexLimit - 1) { std::stringstream ss; - ss << "tile.X:" << tile.X << "tile.Y:" << tile.Y; - BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() - << Component(ss.str()) << Constraint("tileindex > indexLimit")); + ss << "tile index > index limit " << "tile.X:" << tile.X << " tile.Y:" << tile.Y; + throw std::range_error(ss.str()); } if (b_idx > indexer.blockDesc.indexLimit - 1) { std::stringstream ss; - ss << "block.X:" << block.X << "block.Y:" << block.Y; - BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() - << Component(ss.str()) << Constraint("blockindex > indexLimit")); + ss << "block index > index limit block.X:" << block.X << "block.Y:" << block.Y; + throw std::range_error(ss.str()); } tileIdx = t_idx; @@ -203,16 +201,14 @@ struct DATAREPOSITORY_API BlockIdx { : tileCols(indexer.tileDesc.cols), blockCols(indexer.blockDesc.cols) { if (tileIdx > indexer.tileDesc.indexLimit - 1) { std::stringstream ss; - ss << "tileIdx:" << tileIdx; - BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() - << Component(ss.str()) << Constraint("tileindex > indexLimit")); + ss << "tile index > index limit tileIdx:" << tileIdx; + throw std::invalid_argument(ss.str()); } if (blockIdx > indexer.blockDesc.indexLimit - 1) { std::stringstream ss; - ss << "blockIdx:" << blockIdx; - BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() - << Component(ss.str()) << Constraint("blockindex > indexLimit")); + ss << "block index > index limit blockIdx:" << blockIdx; + throw std::invalid_argument(ss.str()); } this->tileIdx = tileIdx; diff --git a/Source/moja.datarepository/src/aspatialtileinfo.cpp b/Source/moja.datarepository/src/aspatialtileinfo.cpp index e664cdb..a21242f 100644 --- a/Source/moja.datarepository/src/aspatialtileinfo.cpp +++ b/Source/moja.datarepository/src/aspatialtileinfo.cpp @@ -1,9 +1,6 @@ #include "moja/datarepository/aspatialtileinfo.h" #include "moja/datarepository/aspatialtileinfocollection.h" -#include "moja/datarepository/datarepositoryexceptions.h" - -using moja::datarepository::LandscapeDefinitionException; namespace moja { namespace datarepository { @@ -11,7 +8,7 @@ namespace datarepository { AspatialTileInfo::AspatialTileInfo(const AspatialTileInfoCollection& tileInfoCollection, int id) : _tileInfoCollection(&tileInfoCollection) { if (id < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("id") << Constraint("> 0")); + std::invalid_argument("Error tile id less than zero: " + std::to_string(id)); } _id = id; diff --git a/Source/moja.datarepository/src/aspatialtileinfocollection.cpp b/Source/moja.datarepository/src/aspatialtileinfocollection.cpp index a124e92..702d48e 100644 --- a/Source/moja.datarepository/src/aspatialtileinfocollection.cpp +++ b/Source/moja.datarepository/src/aspatialtileinfocollection.cpp @@ -1,7 +1,6 @@ #include "moja/datarepository/aspatialtileinfocollection.h" #include "moja/datarepository/aspatialtileinfo.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/iproviderrelationalinterface.h" #include @@ -16,11 +15,11 @@ namespace datarepository { AspatialTileInfoCollection::AspatialTileInfoCollection(const IProviderRelationalInterface& provider, int maxTileSize, int tileCacheSize) { if (maxTileSize < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("maxTileSize") << Constraint("> 0")); + throw std::invalid_argument("Error maxTileSize less that 1 " + std::to_string(maxTileSize)); } if (tileCacheSize < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("tileCacheSize") << Constraint("> 0")); + throw std::invalid_argument("Error tileCacheSize less that 1 " + std::to_string(tileCacheSize)); } _provider = &provider; diff --git a/Source/moja.datarepository/src/datarepository.cpp b/Source/moja.datarepository/src/datarepository.cpp index 55a77ef..1717fc8 100644 --- a/Source/moja.datarepository/src/datarepository.cpp +++ b/Source/moja.datarepository/src/datarepository.cpp @@ -1,13 +1,10 @@ #include "moja/datarepository/datarepository.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/providerrelationalsqlite.h" #include "moja/datarepository/providerspatialrastertiled.h" #include -#include - namespace moja { namespace datarepository { @@ -22,14 +19,14 @@ std::shared_ptr DataRepository::createProvider(const std::st const std::string& providerType, const DynamicObject& settings) const { const auto providerKey = ProviderKey(library, providerType); - auto it = _providerRegistry->find(providerKey); + const auto it = _providerRegistry->find(providerKey); if (it == _providerRegistry->end()) { - BOOST_THROW_EXCEPTION(moja::datarepository::ProviderUnsupportedException() << ProviderType(providerType)); + throw std::runtime_error("Error unsupported provider " + library + ":" + providerType); } auto providerInit = _providerRegistry->at(providerKey); auto providerInterface = providerInit(settings); if (providerInterface == nullptr) { - BOOST_THROW_EXCEPTION(moja::datarepository::ProviderUnsupportedException() << ProviderType(providerType)); + throw std::runtime_error("Error provider init failed " + library + ":" + providerType); } return providerInterface; } @@ -40,8 +37,7 @@ void DataRepository::addProvider(const std::string& name, const std::string& pro if (p == _providers.end()) { _providers.insert(std::pair>(name, provider)); } else { - BOOST_THROW_EXCEPTION(moja::datarepository::ProviderAlreadyExistsException() - << ProviderName(name) << ProviderType(providerType)); + throw std::runtime_error("Error provider already exists " + name + ":" + providerType); } } @@ -50,7 +46,7 @@ std::shared_ptr DataRepository::getProvider(const std::strin if (p != _providers.end()) { return p->second; } - BOOST_THROW_EXCEPTION(moja::datarepository::ProviderNotFoundException() << ProviderName(name)); + throw std::runtime_error("Error provider not found " + name); } } // namespace datarepository diff --git a/Source/moja.datarepository/src/landunitinfo.cpp b/Source/moja.datarepository/src/landunitinfo.cpp index 36d076a..cd78888 100644 --- a/Source/moja.datarepository/src/landunitinfo.cpp +++ b/Source/moja.datarepository/src/landunitinfo.cpp @@ -1,22 +1,20 @@ #include "moja/datarepository/landunitinfo.h" #include "moja/datarepository/aspatialtileinfo.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include -using moja::datarepository::LandscapeDefinitionException; namespace moja { namespace datarepository { LandUnitInfo::LandUnitInfo(const AspatialTileInfo& tile, Int64 id, double area) { if (id < 1) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("id") << Constraint("> 0")); + throw std::invalid_argument("Error id less that 1 " + std::to_string(id)); } if (area <= 0.0) { - BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("area") << Constraint("> 0")); + throw std::invalid_argument("Error area less than or equal to 0.0 " + std::to_string(area)); } _tile = &tile; diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index 3dc724e..4a0ed49 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -1,7 +1,5 @@ #include "moja/datarepository/providerrelationalsqlite.h" -#include "moja/datarepository/datarepositoryexceptions.h" - #include #include @@ -29,12 +27,12 @@ class SQLiteConnection { if (path.find(":memory:") == std::string::npos) { if (!fs::exists(path)) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(path)); + throw std::runtime_error("Error SQLite database file not found " + path); } } if (sqlite3_open_v2(path.c_str(), &_conn, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, 0) != SQLITE_OK) { - BOOST_THROW_EXCEPTION(ConnectionFailedException() << ConnectionError(sqlite3_errmsg(_conn))); + throw std::runtime_error("Error SQLite database connection failed: " + std::string(sqlite3_errmsg(_conn))); } sqlite3_busy_timeout(_conn, 60000); @@ -54,7 +52,7 @@ class SQLiteStatement { SQLiteStatement(const SQLiteConnection& conn, const std::string& query) { if (sqlite3_prepare_v2(conn, query.c_str(), int(query.size()), &_stmt, nullptr) != SQLITE_OK) { - BOOST_THROW_EXCEPTION(QueryException() << SQL(query) << SQLError(sqlite3_errmsg(conn))); + throw std::runtime_error("Error SQLite prepare failed: " + std::string(sqlite3_errmsg(conn))); } } @@ -124,7 +122,7 @@ class ProviderRelationalSQLite::impl { return std::vector(); break; default: - BOOST_THROW_EXCEPTION(QueryException() << SQL(query) << SQLError(sqlite3_errmsg(_conn))); + throw std::runtime_error("Error SQLite query failed: " + std::string(sqlite3_errmsg(_conn))); } std::vector result; diff --git a/Source/moja.datarepository/src/providerspatialrastertiled.cpp b/Source/moja.datarepository/src/providerspatialrastertiled.cpp index 124c5bb..1b4af52 100644 --- a/Source/moja.datarepository/src/providerspatialrastertiled.cpp +++ b/Source/moja.datarepository/src/providerspatialrastertiled.cpp @@ -108,7 +108,7 @@ ProviderSpatialRasterTiled::ProviderSpatialRasterTiled( layerblockLonSize, layercellLatSize, layercellLonSize), layer["nodata"], layer, attributeTable))); } else { - BOOST_THROW_EXCEPTION(ProviderUnsupportedException() << ProviderType(layerData)); + throw std::runtime_error("Unsuported layer data type: " + layerData); } } else if (layerType == "StackLayer") { // if (rasterReaderFactorySet.stackReaderFactory() == nullptr) { @@ -160,7 +160,7 @@ ProviderSpatialRasterTiled::ProviderSpatialRasterTiled( layerblockLonSize, layercellLatSize, layercellLonSize), layer["nodata"], layer))); } else { - BOOST_THROW_EXCEPTION(ProviderUnsupportedException() << ProviderType(layerData)); + throw std::runtime_error("Unsuported layer data type: " + layerData); } } else { // TODO: should throw exception @@ -176,14 +176,12 @@ DynamicVar ProviderSpatialRasterTiled::GetValue(const std::string& name, double } const IProviderLayer* ProviderSpatialRasterTiled::getLayer(const std::string& name) const { - if (_layers.find(name) == _layers.end()) BOOST_THROW_EXCEPTION(LayerNotFoundException() << LayerName(name)); + if (_layers.find(name) == _layers.end()) throw std::runtime_error("Layer not found " + name); return _layers.at(name).get(); } inline const TileBlockCellIndexer& ProviderSpatialRasterTiled::indexer(const std::string& layerName) { - if (_layers.find(layerName) == _layers.end()) - BOOST_THROW_EXCEPTION(LayerNotFoundException() << LayerName(layerName)); - auto layer = _layers.at(layerName); + const auto* layer = getLayer(layerName); return layer->indexer(); } diff --git a/Source/moja.datarepository/src/rasterreader.cpp b/Source/moja.datarepository/src/rasterreader.cpp index 6a406c1..cb6d04d 100644 --- a/Source/moja.datarepository/src/rasterreader.cpp +++ b/Source/moja.datarepository/src/rasterreader.cpp @@ -1,6 +1,5 @@ #include "moja/datarepository/rasterreader.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/tileblockcellindexer.h" #include @@ -41,7 +40,7 @@ DynamicObject FlintMetaDataRasterReader::readMetaData() const { auto layerMetadata = parsePocoJSONToDynamic(metadata).extract(); return layerMetadata; } else { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(_metaPath)); + throw std::runtime_error("Error metadata file not found " + _metaPath); } } @@ -102,7 +101,7 @@ void FlintTileRasterReader::readFlintBlockData(const BlockIdx& blk_idx, char* bl fileStream.open(_tilePath, std::ios::binary); std::streampos blockStart = blk_idx.blockIdx * block_size; if (fileStream.fail()) { - BOOST_THROW_EXCEPTION(FileReadException() << FileName(_tilePath) << Message(strerror(errno))); + throw std::runtime_error("Error failed to open block data " + _tilePath + std::string(strerror(errno))); } fileStream.seekg(blockStart); @@ -173,7 +172,7 @@ void FlintStackRasterReader::readFlintBlockData(const BlockIdx& blk_idx, int nSe fileStream.open(_tilePath, std::ios::binary); std::streampos blockStart = blk_idx.blockIdx * block_size; if (fileStream.fail()) { - BOOST_THROW_EXCEPTION(FileReadException() << FileName(_tilePath) << Message(strerror(errno))); + throw std::runtime_error("Error failed to open block data " + _tilePath + std::string(strerror(errno))); } fileStream.seekg(blockStart); diff --git a/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h index 7b130a0..2962d7b 100644 --- a/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h @@ -20,12 +20,12 @@ class FLINT_API AspatialLocalDomainController final : public LocalDomainControll AspatialLocalDomainController() : _spinupLandUnitController(_landUnitController), _runSpinUp(false) {} ~AspatialLocalDomainController() = default; - void configure(const configuration::Configuration& config) override; - void run() override; - void startup() override; - void shutdown() override; + status configure(const configuration::Configuration& config) override; + status run() override; void run(moja::datarepository::AspatialTileInfo& tile); + status startup() override; + status shutdown() override; private: std::unique_ptr _landscape; @@ -36,7 +36,7 @@ class FLINT_API AspatialLocalDomainController final : public LocalDomainControll LocalDomainControllerBase::ModuleMap _spinupModules; IVariable* _luId; - void run(const moja::datarepository::LandUnitInfo& lu); + status run(const moja::datarepository::LandUnitInfo& lu); }; } // namespace flint diff --git a/Source/moja.flint/include/moja/flint/flintexceptions.h b/Source/moja.flint/include/moja/flint/flintexceptions.h index b422c39..ded06f9 100644 --- a/Source/moja.flint/include/moja/flint/flintexceptions.h +++ b/Source/moja.flint/include/moja/flint/flintexceptions.h @@ -1,30 +1,55 @@ -#ifndef MOJA_FLINT_FLINTEXCEPTIONS_H_ -#define MOJA_FLINT_FLINTEXCEPTIONS_H_ +#pragma once #include "moja/flint/_flint_exports.h" -#include - #include - -namespace moja { -namespace flint { - -struct FLINT_API FLINTException : virtual std::exception, virtual boost::exception {}; - -struct FLINT_API LocalDomainError : virtual FLINTException {}; -typedef boost::error_info Details; -typedef boost::error_info LibraryName; -typedef boost::error_info ModuleName; -typedef boost::error_info ErrorCode; - -struct FLINT_API SimulationError : virtual FLINTException {}; -typedef boost::error_info Details; -typedef boost::error_info LibraryName; -typedef boost::error_info ModuleName; -typedef boost::error_info ErrorCode; - -} // namespace flint -} // namespace moja - -#endif // MOJA_FLINT_FLINTEXCEPTIONS_H_ +#include +#include + +namespace moja::flint { + +class FLINT_API flint_exception : public std::runtime_error { + public: + flint_exception(const std::string& message) : std::runtime_error(message) {} + flint_exception(const char* message) : std::runtime_error(message) {} +}; + +class FLINT_API local_domain_error : public flint_exception { + public: + local_domain_error(const std::string& message, std::string library_name = "", std::string module_name = "", + int error_code = -1) + : flint_exception(message), + library_name_(std::move(library_name)), + module_name_(std::move(module_name)), + error_code_(error_code) {} + + [[nodiscard]] const std::string& library_name() const noexcept { return library_name_; } + [[nodiscard]] const std::string& module_name() const noexcept { return module_name_; } + [[nodiscard]] int error_code() const noexcept { return error_code_; } + + private: + std::string library_name_; + std::string module_name_; + int error_code_; +}; + +class FLINT_API simulation_error : public flint_exception { + public: + simulation_error(const std::string& message, std::string library_name = "", std::string module_name = "", + int error_code = -1) + : flint_exception(message), + library_name_(std::move(library_name)), + module_name_(std::move(module_name)), + error_code_(error_code) {} + + [[nodiscard]] const std::string& library_name() const noexcept { return library_name_; } + [[nodiscard]] const std::string& module_name() const noexcept { return module_name_; } + [[nodiscard]] int error_code() const noexcept { return error_code_; } + + private: + std::string library_name_; + std::string module_name_; + int error_code_; +}; + +} // namespace moja::flint diff --git a/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h index a3b1020..703ecbc 100644 --- a/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h @@ -1,10 +1,9 @@ -#ifndef MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ -#define MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ +#pragma once #include +#include -namespace moja { -namespace flint { +namespace moja::flint { class IVariable; class IModule; class ILandUnitController; @@ -15,18 +14,18 @@ class SequencerModuleBase; class ILocalDomainController { public: - typedef std::pair ModuleMapKey; - typedef std::map> ModuleMap; + using ModuleMapKey = std::pair; + using ModuleMap = std::map>; - virtual ~ILocalDomainController(void) = default; + virtual ~ILocalDomainController() = default; // configure the simulation - virtual void configure(const configuration::Configuration& config) = 0; + virtual status configure(const configuration::Configuration& config) = 0; // execute the simulation - virtual void run() = 0; - virtual void startup() = 0; - virtual void shutdown() = 0; + virtual status run() = 0; + virtual status startup() = 0; + virtual status shutdown() = 0; virtual int localDomainId() = 0; virtual void set_localDomainId(const int value) = 0; @@ -35,17 +34,9 @@ class ILocalDomainController { virtual std::map modules() const = 0; - // every simulation needs a notification center - NotificationCenter _notificationCenter; + virtual NotificationCenter& notification_center() = 0; + virtual const NotificationCenter& notification_center() const = 0; - // every simulation needs a sequencer - std::shared_ptr _sequencer; - - // Variable used to check if landUnit should be simulated - const flint::IVariable* _simulateLandUnit; - flint::IVariable* _landUnitBuildSuccess; }; -} // namespace flint -} // namespace moja -#endif // MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ +} // namespace moja::flint diff --git a/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h b/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h index 29cb035..2a980a6 100644 --- a/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h +++ b/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h @@ -20,10 +20,10 @@ class FLINT_API LocalDomainControllerBase : public ILocalDomainController { explicit LocalDomainControllerBase(std::shared_ptr libraryHandles); virtual ~LocalDomainControllerBase(void) = default; - void configure(const configuration::Configuration& config) override; - void run() override; - void startup() override; - void shutdown() override; + status configure(const configuration::Configuration& config) override; + status run() override; + status startup() override; + status shutdown() override; ILandUnitController& landUnitController() override { return _landUnitController; }; virtual void initialiseData() { _landUnitController.initialiseData(false); } @@ -32,8 +32,14 @@ class FLINT_API LocalDomainControllerBase : public ILocalDomainController { void set_localDomainId(const int value) override { _localDomainId = value; } std::map modules() const override; + NotificationCenter& notification_center() override { return _notificationCenter; } + const NotificationCenter& notification_center() const override { return _notificationCenter; } protected: + NotificationCenter _notificationCenter; + std::shared_ptr _sequencer; + const IVariable* _simulateLandUnit; + IVariable* _landUnitBuildSuccess; int _localDomainId; const configuration::Configuration* _config; LandUnitController _landUnitController; diff --git a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h index ae8cf41..933b7d1 100644 --- a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h @@ -10,6 +10,8 @@ #include #include +#include + #include #include @@ -119,12 +121,12 @@ class FLINT_API SpatialTiledLocalDomainController : public LocalDomainController bool isThread = false); ~SpatialTiledLocalDomainController(void) = default; - void configure(const flint::configuration::Configuration& config) override; - void run() override; + status configure(const flint::configuration::Configuration& config) override; + status run() override; - void startup() override; + status startup() override; - void shutdown() override; + status shutdown() override; private: class InternalThreadBlocks; diff --git a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h index 2b4f6f7..7ac9810 100644 --- a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h @@ -41,14 +41,11 @@ class AspatialLocalDomainThread { class FLINT_API ThreadedAspatialLocalDomainController final : public flint::LocalDomainControllerBase { public: - ThreadedAspatialLocalDomainController(void); - ~ThreadedAspatialLocalDomainController() = default; + ThreadedAspatialLocalDomainController(); + ~ThreadedAspatialLocalDomainController() override = default; - virtual void configure(const flint::configuration::Configuration& config) override; - virtual void run() override; - - virtual void startup() override {} - virtual void shutdown() override {} + status configure(const configuration::Configuration& config) override; + status run() override; std::vector> tasks() const { return _tasks; } diff --git a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp index e3a3670..7dd8b85 100644 --- a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp @@ -28,7 +28,7 @@ using moja::flint::configuration::LocalDomainType; namespace moja { namespace flint { -void AspatialLocalDomainController::configure(const configuration::Configuration& config) { +status AspatialLocalDomainController::configure(const configuration::Configuration& config) { LocalDomainControllerBase::configure(config); // Build landscape. @@ -130,9 +130,11 @@ void AspatialLocalDomainController::configure(const configuration::Configuration _spinupLandUnitController.initialiseData(false); _luId = _landUnitController.getVariable("LandUnitId"); } + return status(status_code::Ok); } -void AspatialLocalDomainController::run(const LandUnitInfo& lu) { +status AspatialLocalDomainController::run(const LandUnitInfo& lu) { + status run_result; try { _luId->set_value(lu.id()); _landUnitController.initialiseData(true); @@ -144,20 +146,23 @@ void AspatialLocalDomainController::run(const LandUnitInfo& lu) { } if (!_simulateLandUnit->value()) { - return; + return run_result; } _notificationCenter.postNotification(moja::signals::PreTimingSequence); if (!_landUnitBuildSuccess->value()) { - return; + return run_result; } - _sequencer->Run(_notificationCenter, _landUnitController); + if(!_sequencer->Run(_notificationCenter, _landUnitController)) { + run_result = status(status_code::Error, "Sequencer failed"); + } } catch (const std::exception& e) { MOJA_LOG_FATAL << e.what(); } _landUnitController.clearAllOperationResults(); + return run_result; } void AspatialLocalDomainController::run(AspatialTileInfo& tile) { @@ -174,41 +179,52 @@ void AspatialLocalDomainController::run(AspatialTileInfo& tile) { MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); } -void AspatialLocalDomainController::startup() { - LocalDomainControllerBase::startup(); - if (_runSpinUp) { +status AspatialLocalDomainController::startup() { + auto base_status = LocalDomainControllerBase::startup(); + if (base_status.ok() && _runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainInit); + return status(status_code::Ok); } + return base_status; } -void AspatialLocalDomainController::shutdown() { - LocalDomainControllerBase::shutdown(); +status AspatialLocalDomainController::shutdown() { + auto base_status = LocalDomainControllerBase::shutdown(); if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainShutdown); + return status(status_code::Ok); } + return base_status; } -void AspatialLocalDomainController::run() { - startup(); - auto startTime = DateTime::now(); +status AspatialLocalDomainController::run() { - auto total = _landscape->getTotalLUCount(); - auto count = 0; - for (auto lu : *_landscape) { - count++; - MOJA_LOG_INFO << std::setfill(' ') << std::setw(10) << count << " of " << std::setfill(' ') << std::setw(10) - << total; - run(lu); + status result(status_code::Ok); + auto startTime = DateTime::now(); + if (const auto startup_status = startup(); startup_status.ok()) { + auto total = _landscape->getTotalLUCount(); + auto count = 0; + for (auto& lu : *_landscape) { + count++; + MOJA_LOG_INFO << std::setfill(' ') << std::setw(10) << count << " of " << std::setfill(' ') << std::setw(10) + << total; + if (auto run_status = run(lu); !run_status.ok()) { + MOJA_LOG_ERROR << run_status.message(); + } + } + if (auto shutdown_status = shutdown(); !shutdown_status.ok()) { + MOJA_LOG_ERROR << shutdown_status.message(); + } + } else { + MOJA_LOG_ERROR << startup_status.message(); } - shutdown(); - - //_notificationCenter.postNotification(moja::signals::SystemShutdown); - + auto endTime = DateTime::now(); auto ldSpan = endTime - startTime; MOJA_LOG_INFO << "LocalDomain: Start Time : " << startTime; MOJA_LOG_INFO << "LocalDomain: Finish Time : " << endTime; MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); + return result; } } // namespace flint diff --git a/Source/moja.flint/src/localdomaincontrollerbase.cpp b/Source/moja.flint/src/localdomaincontrollerbase.cpp index ef102ba..aa85c80 100644 --- a/Source/moja.flint/src/localdomaincontrollerbase.cpp +++ b/Source/moja.flint/src/localdomaincontrollerbase.cpp @@ -29,20 +29,20 @@ namespace flint { // -------------------------------------------------------------------------------------------- LocalDomainControllerBase::LocalDomainControllerBase() - : _localDomainId(0), _config(nullptr), _landUnitController(), _dataRepository(), _libraryManager() { + : _localDomainId(0), _config(nullptr) { _dataRepository.setProviderRegistry(_libraryManager.getProviderRegistry()); } // -------------------------------------------------------------------------------------------- LocalDomainControllerBase::LocalDomainControllerBase(std::shared_ptr libraryHandles) - : _localDomainId(0), _config(nullptr), _landUnitController(), _dataRepository(), _libraryManager(libraryHandles) { + : _localDomainId(0), _config(nullptr), _libraryManager(libraryHandles) { _dataRepository.setProviderRegistry(_libraryManager.getProviderRegistry()); } // -------------------------------------------------------------------------------------------- -void LocalDomainControllerBase::configure(const configuration::Configuration& config) { +status LocalDomainControllerBase::configure(const configuration::Configuration& config) { // Keep pointer to config _config = &config; @@ -56,7 +56,7 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co _landUnitController.configure(config); // Load the configured Libraries - for (auto lib : config.libraries()) { + for (const auto* lib : config.libraries()) { switch (lib->type()) { case configuration::LibraryType::Internal: // Not required, all internals modules are loaded automatically @@ -93,7 +93,7 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co for (const auto* module : config.modules()) { ModuleMapKey key(module->libraryName(), module->name()); if (_moduleMap.find(key) != _moduleMap.end()) { - throw std::runtime_error("Error duplicate modules " + module->libraryName() + " " + module->name()); + return status(status_code::Error, "Error duplicate modules " + module->libraryName() + " " + module->name()); } _moduleMap[key] = _libraryManager.GetModule(module->libraryName(), module->name()); _moduleMap[key]->setLandUnitController(_landUnitController); @@ -124,8 +124,8 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co ModuleMapKey sequencerKey(config.localDomain()->sequencerLibrary(), config.localDomain()->sequencer()); _sequencer = std::dynamic_pointer_cast(_moduleMap[sequencerKey]); if (_sequencer == nullptr) { - throw std::runtime_error("Error finding sequencer " + config.localDomain()->sequencerLibrary() + " " + - config.localDomain()->sequencer()); + return status(status_code::Error, "Error finding sequencer " + config.localDomain()->sequencerLibrary() + " " + + config.localDomain()->sequencer()); } _sequencer->configure(timing); @@ -141,73 +141,7 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co std::map, TransformInterfacePtr> transforms; std::map, FlintDataInterfacePtr> flintData; -#if 0 - for (const auto var : config.variables2()) { - if (var->variableType() == configuration::VariableType::Internal) { - auto variable = static_cast(var); - _landUnitController.addVariable(variable->name(), std::make_shared(variable->value(), VariableInfo{ variable->name() })); - } - else if (var->variableType() == configuration::VariableType::External) { - auto variable = static_cast(var); - - const auto& transformConfig = variable->transform(); - const auto libraryName = transformConfig.libraryName(); - const auto variableName = variable->name(); - auto key = std::make_pair(libraryName, variableName); - transforms[key] = _libraryManager.GetTransform(libraryName, transformConfig.typeName()); - _landUnitController.addVariable(variable->name(), std::make_shared(transforms[key], VariableInfo{ variable->name() })); - } - else if (var->variableType() == configuration::VariableType::FlintData) { - auto variable = static_cast(var); - - const auto& flintDataConfig = variable->flintdata(); - const auto libraryName = flintDataConfig.libraryName(); - const auto variableName = variable->name(); - auto key = std::make_pair(libraryName, variableName); - flintData[key] = _libraryManager.GetFlintData(libraryName, flintDataConfig.typeName()); - _landUnitController.addVariable(variable->name(), std::make_shared(flintData[key], libraryName, flintDataConfig.typeName(), VariableInfo{ variable->name() })); - } - } - - //// Now go back over and check for nested things - //for (auto var : config.variables2()) { - // if (var->variableType() == configuration::VariableType::Internal) { - // auto variable = static_cast(var); - // auto value = variable->value(); - // variable->set_value(checkForNestedVariables(variable->name(), value)); - // } - //} - - // New version of Variables - // Now Configure external, internal and flintdata - - // config external (transforms) - for (const auto var : config.variables2()) { - if (var->variableType() == configuration::VariableType::External) { - auto variable = static_cast(var); - - const auto& transformConfig = variable->transform(); - const auto libraryName = transformConfig.libraryName(); - const auto variableName = variable->name(); - auto key = std::make_pair(libraryName, variableName); - transforms[key]->configure(transformConfig.settings(), _landUnitController, _dataRepository); - } - } - - // config flintdata - for (const auto var : config.variables2()) { - if (var->variableType() == configuration::VariableType::FlintData) { - auto variable = static_cast(var); - - const auto& flintDataConfig = variable->flintdata(); - const auto libraryName = flintDataConfig.libraryName(); - const auto variableName = variable->name(); - auto key = std::make_pair(libraryName, variableName); - flintData[key]->configure(flintDataConfig.settings(), _landUnitController, _dataRepository); - } - } - -#else + // Configure Variables for (const auto variable : config.variables()) { auto value = variable->value(); @@ -284,8 +218,6 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co transforms[key]->configure(transformConfig.settings(), _landUnitController, _dataRepository); } -#endif - // Configure the simulateLandUnit & landUnitBuildSuccess variables here _simulateLandUnit = _landUnitController.getVariable(config.localDomain()->simulateLandUnit()); _landUnitBuildSuccess = _landUnitController.getVariable(config.localDomain()->landUnitBuildSuccess()); @@ -294,31 +226,39 @@ void LocalDomainControllerBase::configure(const configuration::Configuration& co // variables (some special handling for vectors & structs _landUnitController.initialiseData(false); /// TODO: check if this is required here. initialiseData is also called /// in SpatialTiledLocalDomainController::runCell + return status(status_code::Ok); } // -------------------------------------------------------------------------------------------- -void LocalDomainControllerBase::run() { +status LocalDomainControllerBase::run() { _notificationCenter.postNotification(moja::signals::PreTimingSequence); _notificationCenter.postNotification(signals::LocalDomainProcessingUnitInit); - _sequencer->Run(_notificationCenter, _landUnitController); + const bool sequencer_result = _sequencer->Run(_notificationCenter, _landUnitController); _notificationCenter.postNotification(signals::LocalDomainProcessingUnitShutdown); + return sequencer_result ? status(status_code::Ok) : status(status_code::Error, "Sequencer failed"); } // -------------------------------------------------------------------------------------------- -void LocalDomainControllerBase::startup() { _notificationCenter.postNotification(moja::signals::LocalDomainInit); } +status LocalDomainControllerBase::startup() { + _notificationCenter.postNotification(moja::signals::LocalDomainInit); + return status(status_code::Ok); +} // -------------------------------------------------------------------------------------------- -void LocalDomainControllerBase::shutdown() { _notificationCenter.postNotification(moja::signals::LocalDomainShutdown); } +status LocalDomainControllerBase::shutdown() { + _notificationCenter.postNotification(moja::signals::LocalDomainShutdown); + return status(status_code::Ok); +} // -------------------------------------------------------------------------------------------- std::map LocalDomainControllerBase::modules() const { std::map results; - for (auto module : _moduleMap) { - results.insert(std::pair(module.first, module.second.get())); + for (auto& [key, module_ptr] : _moduleMap) { + results.emplace(key, module_ptr.get()); } return results; } diff --git a/Source/moja.flint/src/operationmanagersimple.cpp b/Source/moja.flint/src/operationmanagersimple.cpp index 4fff89b..ffc38cb 100644 --- a/Source/moja.flint/src/operationmanagersimple.cpp +++ b/Source/moja.flint/src/operationmanagersimple.cpp @@ -13,13 +13,11 @@ #include #include -#include +#include namespace moja { namespace flint { -#define OPERATION_MANAGER_CACHE_SIZE 100 - // -------------------------------------------------------------------------------------------- OperationManagerSimple::OperationManagerSimple(Timing& timing, const DynamicObject& config) : _timing(timing) { @@ -117,15 +115,11 @@ void OperationManagerSimple::applyOperations() { if (!_allowNegativeTransfers && FloatCmp::lessThan(val, 0.0)) { auto amount = flux.transferType() == OperationTransferType::Proportional ? val * 100.0 : val; - BOOST_THROW_EXCEPTION(SimulationError() - << Details((boost::format("Negative transfer by %1%: %2% %3% %4% to %5%") % - flux.metaData()->moduleName % amount % - (flux.transferType() == OperationTransferType::Proportional ? "% of" - : "from") % - getPool(srcIx)->name() % getPool(dstIx)->name()) - .str()) - << LibraryName("moja.flint") << ModuleName("OperationManagerSimple") - << ErrorCode(0)); + throw simulation_error( + fmt::format("Negative transfer by {}: {} {} {} to {}", flux.metaData()->moduleName, amount, + (flux.transferType() == OperationTransferType::Proportional ? "% of" : "from"), + getPool(srcIx)->name(), getPool(dstIx)->name()), + "moja.flint", "OperationManagerSimple", 0); } if (_warnNegativeTransfers && FloatCmp::lessThan(val, 0.0)) { diff --git a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp index 8ac0641..e8e0e16 100644 --- a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp +++ b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp @@ -172,9 +172,6 @@ void StatsUnitRecord::mojaLog(const std::string& levelStr, int localDomainId, da << "LU/second: " << std::setfill(' ') << std::setw(8) << luPerSecond << "]"; } -#define RETRY_ATTEMPTS 10000 -#define RETRY_SLEEP std::chrono::milliseconds(200) - // -------------------------------------------------------------------------------------------- class SpatialTiledLocalDomainController::InternalThreadBlocks { @@ -388,7 +385,7 @@ SpatialTiledLocalDomainController::SpatialTiledLocalDomainController( // -------------------------------------------------------------------------------------------- -void SpatialTiledLocalDomainController::configure(const configuration::Configuration& config) { +status SpatialTiledLocalDomainController::configure(const configuration::Configuration& config) { LocalDomainControllerBase::configure(config); _doLogging = config.localDomain()->doLogging(); _numThreads = config.localDomain()->numThreads(); @@ -614,6 +611,7 @@ void SpatialTiledLocalDomainController::configure(const configuration::Configura _tasks.push_back(task); } } + return status(status_code::Ok); } // -------------------------------------------------------------------------------------------- @@ -740,22 +738,21 @@ bool SpatialTiledLocalDomainController::runCellSpinUp(std::shared_ptr_stopWatchProcessed.stop(); blockStatsUnit->_unitsProcessed++; return true; - } catch (const flint::SimulationError& e) { - _spatiallocationinfo->_errorCode = *boost::get_error_info(e); - _spatiallocationinfo->_library = *boost::get_error_info(e); - _spatiallocationinfo->_module = *boost::get_error_info(e); - _spatiallocationinfo->_message = *boost::get_error_info
(e); - _spinupNotificationCenter.postNotification(moja::signals::Error, *boost::get_error_info
(e)); + } catch (const flint::simulation_error& e) { + _spatiallocationinfo->_errorCode = e.error_code(); + _spatiallocationinfo->_library = e.library_name(); + _spatiallocationinfo->_module = e.module_name(); + _spatiallocationinfo->_message = e.what(); + _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); return true; - } catch (const flint::LocalDomainError& e) { - _spatiallocationinfo->_errorCode = *boost::get_error_info(e); - _spatiallocationinfo->_library = *boost::get_error_info(e); - _spatiallocationinfo->_module = *boost::get_error_info(e); - _spatiallocationinfo->_message = *boost::get_error_info
(e); - _spinupNotificationCenter.postNotification(moja::signals::Error, *boost::get_error_info
(e)); + } catch (const flint::local_domain_error& e) { + _spatiallocationinfo->_errorCode = e.error_code(); + _spatiallocationinfo->_library = e.library_name(); + _spatiallocationinfo->_module = e.module_name(); + _spatiallocationinfo->_message = e.what(); + _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); } catch (const std::exception& e) { - const std::string str = e.what(); - _spinupNotificationCenter.postNotification(moja::signals::Error, str); + _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); } catch (...) { _spinupNotificationCenter.postNotification(moja::signals::Error, "unknown exception"); } @@ -851,35 +848,26 @@ bool SpatialTiledLocalDomainController::runCell(std::shared_ptr } // This error is recoverable, retunr true for success - catch (const flint::SimulationError& e) { + catch (const flint::simulation_error& e) { // This error is recoverable, drop cell and continue simulation - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - const int* errorCode = boost::get_error_info(e); - _spatiallocationinfo->_errorCode = *errorCode; - _spatiallocationinfo->_library = libraryName; - _spatiallocationinfo->_module = moduleName; - _spatiallocationinfo->_message = details; - _notificationCenter.postNotification(moja::signals::Error, details); + _spatiallocationinfo->_errorCode = e.error_code(); + _spatiallocationinfo->_library = e.library_name(); + _spatiallocationinfo->_module = e.module_name(); + _spatiallocationinfo->_message = e.what(); + _notificationCenter.postNotification(moja::signals::Error, e.what()); blockStatsUnit->_unitsNotProcessed++; blockStatsUnit->_unitsWithError++; return true; } // All other catches are failures for the localdomain, return false! - catch (const flint::LocalDomainError& e) { - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - const int* errorCode = boost::get_error_info(e); - _spatiallocationinfo->_errorCode = *errorCode; - _spatiallocationinfo->_library = libraryName; - _spatiallocationinfo->_module = moduleName; - _spatiallocationinfo->_message = details; - _notificationCenter.postNotification(moja::signals::Error, details); + catch (const flint::local_domain_error& e) { + _spatiallocationinfo->_errorCode = e.error_code(); + _spatiallocationinfo->_library = e.library_name(); + _spatiallocationinfo->_module = e.module_name(); + _spatiallocationinfo->_message = e.what(); + _notificationCenter.postNotification(moja::signals::Error, e.what()); } catch (const std::exception& e) { - const std::string str = e.what(); - _notificationCenter.postNotification(moja::signals::Error, str); + _notificationCenter.postNotification(moja::signals::Error, e.what()); } catch (...) { _notificationCenter.postNotification(moja::signals::Error, "unknown exception"); } @@ -890,44 +878,7 @@ bool SpatialTiledLocalDomainController::runCell(std::shared_ptr // -------------------------------------------------------------------------------------------- -// void SpatialTiledLocalDomainController::writeRunSummaryToSQLite(bool insertRunList, -// std::vector& runSummaryData) { auto retry = false; auto maxRetries = RETRY_ATTEMPTS; do { try { -// retry = false; -// SQLite::Connector::registerConnector(); -// Session session("SQLite", _sqliteDatabaseName); -// if (_createStatsTablesSQLIte) { -// -// if (insertRunList) -// session << "DROP TABLE IF EXISTS run_list", now; -// session << "DROP TABLE IF EXISTS run_properties", now; -// } -// if (insertRunList) -// session << "CREATE TABLE IF NOT EXISTS run_list (id INTEGER PRIMARY KEY AUTOINCREMENT, run_id -//VARCHAR(64) NOT NULL, run_desc VARCHAR(255))", now; session << "CREATE TABLE IF NOT EXISTS run_properties (id INTEGER -//PRIMARY KEY AUTOINCREMENT, run_id_fk VARCHAR(64), property_name VARCHAR(255), property_info VARCHAR(255), -//property_value VARCHAR(255))", now; -// -// if (insertRunList) { -// auto insertStr = (boost::format("INSERT INTO run_list (run_id, run_desc) VALUES('%1%','%2%')") -//% _runId % _runDesc).str(); session.begin(); session << insertStr, now; session.commit(); -// } -// -// session.begin(); -// session << "INSERT INTO run_properties (run_id_fk, property_name, property_info, property_value) -//VALUES(?, ?, ?, ?)", bind(runSummaryData), now; session.commit(); SQLite::Connector::unregisterConnector(); -// } -// catch (SQLite::DBLockedException&) { -// MOJA_LOG_DEBUG << localDomainId() << ":DBLockedException - " << maxRetries << " retries -//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << -//localDomainId() << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; -// } -// } -// } while (retry); -//} - -// -------------------------------------------------------------------------------------------- - -void SpatialTiledLocalDomainController::run() { +status SpatialTiledLocalDomainController::run() { using namespace std::chrono; using clock = system_clock; using timepoint = time_point; @@ -939,14 +890,7 @@ void SpatialTiledLocalDomainController::run() { std::string runSummaryInfoStr; if (_writeStatsToSQLIte && (!_threadedSystem || (_threadedSystem && _isMaster))) { - /// Write first part of Stats to SQLite - /// Stats are finalized after simulation - // StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, localDomainId(), _sqliteDatabaseName, - // "run_stats_global"); StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, localDomainId(), - // _sqliteDatabaseName, "run_stats_tile"); StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, - // localDomainId(), _sqliteDatabaseName, "run_stats_block"); - - //(boost::format("Thread %1%") % _localDomainId).str(); + runSummaryInfoStr = !_threadedSystem ? "Process" : "Master"; if (!_threadedSystem && _landUnitController.hasVariable("spatialLocationInfo")) { auto _spatiallocationinfo = std::static_pointer_cast( @@ -1001,7 +945,7 @@ void SpatialTiledLocalDomainController::run() { } _spatiallocationinfo->_engGlobal.seed(_spatiallocationinfo->_randomSeedGlobal); - while (_blockIdxList.size() > 0) { + while (!_blockIdxList.empty()) { if (!_threadedSystem) { _blockIdxListPosition = int(_blockIdxList.size()); } @@ -1058,7 +1002,7 @@ void SpatialTiledLocalDomainController::run() { break; // This will exit on error return from runCell } } else { - std::pair key(blockIdx.tileIdx, blockIdx.blockIdx); + const auto key = std::make_pair(blockIdx.tileIdx, blockIdx.blockIdx); if (_blockCellIdxList.find(key) != _blockCellIdxList.end()) { auto& blockCells = _blockCellIdxList[key]; for (const auto& cell : blockCells) { @@ -1102,7 +1046,7 @@ void SpatialTiledLocalDomainController::run() { << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx << "," << _spatiallocationinfo->_cellIdx << ", " << "msg:" << e.what(); - exit(1); + return status(status_code::Error, std::string("Error in local domain") + e.what()); } catch (...) { _spatiallocationinfo->_errorCode = 1; MOJA_LOG_FATAL << "exception caught at LocalDomain level, exiting..." @@ -1111,7 +1055,7 @@ void SpatialTiledLocalDomainController::run() { << "," << _spatiallocationinfo->_cellIdx << ", " << "msg:" << "unknown exception"; - exit(1); + return status(status_code::Error, std::string("Error in local domain unknown exception")); } } @@ -1133,23 +1077,10 @@ void SpatialTiledLocalDomainController::run() { datarepository::TileIdx tileIdxObject(tileRec->_tileIdx, _provider->indexer()); tileRec->mojaLog("Tile", localDomainId(), &tileIdxObject, nullptr, true); } - // for (auto& i : _tileStatsDimension->getPersistableCollection()) { - // StatsUnitRecord tileStatsRecord(i.get<1>(), seconds(i.get<2>()), seconds(i.get<3>()), - //seconds(i.get<4>()), seconds(i.get<5>()), i.get<6>(), i.get<7>(), i.get<8>(), i.get<9>(), i.get<10>(), - //i.get<11>(), StatsDurationType::Seconds); datarepository::TileIdx tileIdxObject(i.get<10>(), - //_provider->indexer()); tileStatsRecord.mojaLog("Tile", localDomainId(), &tileIdxObject, nullptr, true); - //} MOJA_LOG_INFO << std::setfill(' ') << std::setw(3) << localDomainId() << ":" << "Summary of processing for full run"; - // for (auto& i : _globalStatsDimension->getPersistableCollection()) { - // StatsUnitRecord tileStatsRecord(i.get<1>(), seconds(i.get<2>()), seconds(i.get<3>()), - //seconds(i.get<4>()), seconds(i.get<5>()), i.get<6>(), i.get<7>(), i.get<8>(), i.get<9>(), i.get<10>(), - //i.get<11>(), StatsDurationType::Seconds); totalUnits += tileStatsRecord._unitsTotal; totalUnitsProcessed += - //tileStatsRecord._unitsProcessed; tileStatsRecord.mojaLog("Global", localDomainId(), nullptr, nullptr, true); - //} - for (auto& rec : _globalStatsDimension->records()) { StatsUnitRecord* tileRec = static_cast(rec.get()); totalUnits += tileRec->_unitsTotal; @@ -1193,11 +1124,6 @@ void SpatialTiledLocalDomainController::run() { auto durMS = duration_cast(runTime).count(); auto durS = duration_cast(runTime).count(); - // StatsUnitRecord::sqliteWrite(_globalStatsDimension, localDomainId(), _sqliteDatabaseName, "run_stats_global"); - // StatsUnitRecord::sqliteWrite(_tileStatsDimension, localDomainId(), _sqliteDatabaseName, "run_stats_tile"); - // StatsUnitRecord::sqliteWrite(_blockStatsDimension, localDomainId(), _sqliteDatabaseName, - // "run_stats_block"); - std::vector runSummaryData; runSummaryData.push_back(RunSummaryDataRecord(_runId, "run_system_finish", runSummaryInfoStr, (boost::format("%1%") % moja::put_time(&t2, "%c %Z")).str())); @@ -1237,11 +1163,11 @@ void SpatialTiledLocalDomainController::run() { runSummaryData.push_back(RunSummaryDataRecord(_runId, "run_thread_%1%_time_milliseconds", runSummaryInfoStr, (boost::format("%1%") % durMS).str())); - // writeRunSummaryToSQLite(false, runSummaryData); } + return status(status_code::Ok); } -void SpatialTiledLocalDomainController::startup() { +status SpatialTiledLocalDomainController::startup() { try { if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::SystemInit); @@ -1253,21 +1179,15 @@ void SpatialTiledLocalDomainController::startup() { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainInit); } } - } catch (const flint::SimulationError& e) { - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting." - << " | Library: " << libraryName << " | Module: " << moduleName << " | Details: " << details; - exit(1); } catch (const std::exception& e) { - MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." + MOJA_LOG_FATAL << "std::exception caught at LocalDomain level " << "msg:" << e.what(); - exit(1); + return status(status_code::Error, std::string("Exception caught at LocalDomain level ") + e.what()); } + return status(status_code::Ok); } -void SpatialTiledLocalDomainController::shutdown() { +status SpatialTiledLocalDomainController::shutdown() { try { // When running threaded we don't want main process to fire LocalDomainShutdown if (!_threadedSystem || (_threadedSystem && _isThread)) { @@ -1279,22 +1199,12 @@ void SpatialTiledLocalDomainController::shutdown() { if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::SystemShutdown); } - } catch (const flint::SimulationError& e) { - std::string details = *(boost::get_error_info
(e)); - std::string libraryName = *(boost::get_error_info(e)); - std::string moduleName = *(boost::get_error_info(e)); - MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting." - << " | Library: " << libraryName << " | Module: " << moduleName << " | Details: " << details; - exit(1); } catch (const std::exception& e) { - MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." - << "msg:" << e.what(); - exit(1); + MOJA_LOG_FATAL << "std::exception caught at LocalDomain level msg:" << e.what(); + return status(status_code::Error, std::string("Exception caught at LocalDomain level ") + e.what()); } + return status(status_code::Ok); } -#undef RETRY_ATTEMPTS -#undef RETRY_SLEEP - } // namespace flint } // namespace moja diff --git a/Source/moja.flint/src/sqlquerytransform.cpp b/Source/moja.flint/src/sqlquerytransform.cpp index 77ac6e6..4c14b93 100644 --- a/Source/moja.flint/src/sqlquerytransform.cpp +++ b/Source/moja.flint/src/sqlquerytransform.cpp @@ -14,6 +14,7 @@ #include #include +#include using moja::datarepository::IProviderRelationalInterface; diff --git a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp index 3d24c58..43778a7 100644 --- a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp @@ -48,7 +48,7 @@ void AspatialLocalDomainThread::operator()() { while (keepRunning) { // Pop a block index from the queue std::unique_lock lock(_tileListMutex); - if (_tileList.size() > 0) { + if (!_tileList.empty()) { auto tile = _tileList.front(); _tileList.pop(); MOJA_LOG_INFO << std::setfill(' ') << std::setw(3) << _threadId @@ -68,7 +68,7 @@ void AspatialLocalDomainThread::operator()() { ThreadedAspatialLocalDomainController::ThreadedAspatialLocalDomainController() : LocalDomainControllerBase() {} -void ThreadedAspatialLocalDomainController::configure(const configuration::Configuration& config) { +status ThreadedAspatialLocalDomainController::configure(const configuration::Configuration& config) { // Call base class configure LocalDomainControllerBase::configure(config); @@ -92,9 +92,10 @@ void ThreadedAspatialLocalDomainController::configure(const configuration::Confi _tasks.push_back(task); } + return status(status_code::Ok); } -void ThreadedAspatialLocalDomainController::run() { +status ThreadedAspatialLocalDomainController::run() { auto startTime = DateTime::now(); for (const auto& task : _tasks) { @@ -104,14 +105,13 @@ void ThreadedAspatialLocalDomainController::run() { for (auto& thread : _threads) { thread.join(); } - - //_notificationCenter.postNotification(moja::signals::SystemShutdown); - + auto endTime = DateTime::now(); auto ldSpan = endTime - startTime; MOJA_LOG_INFO << "LocalDomain: Start Time : " << startTime; MOJA_LOG_INFO << "LocalDomain: Finish Time : " << endTime; MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); + return status(status_code::Ok); } } // namespace flint diff --git a/Source/moja.flint/src/writevariablegrid.cpp b/Source/moja.flint/src/writevariablegrid.cpp index 3a644bd..fd2422f 100644 --- a/Source/moja.flint/src/writevariablegrid.cpp +++ b/Source/moja.flint/src/writevariablegrid.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include namespace fs = moja::filesystem; diff --git a/Source/moja.modules.gdal/src/rasterreadergdal.cpp b/Source/moja.modules.gdal/src/rasterreadergdal.cpp index 21c250a..8280705 100644 --- a/Source/moja.modules.gdal/src/rasterreadergdal.cpp +++ b/Source/moja.modules.gdal/src/rasterreadergdal.cpp @@ -1,6 +1,5 @@ #include "moja/modules/gdal/rasterreadergdal.h" -#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/logging.h" #include "gdalcpp.h" @@ -40,17 +39,11 @@ std::tuple seqChunk2geog(int tile_idx, int block_idx) { MetaDataRasterReaderGDAL::MetaDataRasterReaderGDAL(const std::string& path, const std::string& prefix, const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { - try { - const auto filePath = fs::absolute(path).replace_extension(".json"); - _path = filePath.string(); - _metaDataRequired = true; - if (settings.contains("metadata_required")) { - _metaDataRequired = settings["metadata_required"].extract(); - } - } catch (...) { - BOOST_THROW_EXCEPTION(flint::LocalDomainError() - << flint::Details("GDAL Error in constructor") << flint::LibraryName("moja.modules.gdal") - << flint::ModuleName(BOOST_CURRENT_FUNCTION) << flint::ErrorCode(1)); + const auto filePath = fs::absolute(path).replace_extension(".json"); + _path = filePath.string(); + _metaDataRequired = true; + if (settings.contains("metadata_required")) { + _metaDataRequired = settings["metadata_required"].extract(); } } @@ -66,7 +59,7 @@ DynamicObject MetaDataRasterReaderGDAL::readMetaData() const { return layerMetadata; } else { if (_metaDataRequired) { - BOOST_THROW_EXCEPTION(datarepository::FileNotFoundException() << datarepository::FileName(_path)); + throw std::runtime_error("Error metadata file not found " + _path); } else { return DynamicObject(); } @@ -171,18 +164,11 @@ void TileRasterReaderGDAL::readBlockData(const datarepository::BlockIdx& blkIdx, } else { MOJA_LOG_ERROR << "RunId (" << _runId << ") - " << "GDAL - read error, target (" << _path << ")"; - const auto str = fmt::format("GDAL read error: {}", _path); - BOOST_THROW_EXCEPTION(flint::LocalDomainError() - << flint::Details(str) << flint::LibraryName("moja.modules.gdal") - << flint::ModuleName(BOOST_CURRENT_FUNCTION) << flint::ErrorCode(1)); + + throw std::runtime_error("GDAL error reading block data " + _path); } } catch (const gdalcpp::gdal_error& err) { - BOOST_THROW_EXCEPTION(datarepository::FileReadException() - << datarepository::FileName(_path) << datarepository::Message(err.what())); - } catch (...) { - MOJA_LOG_ERROR << "RunId (" << _runId << ") - " - << "GDAL - exception (" << _path << ")"; - throw; + throw std::runtime_error("GDAL error reading block data " + _path + " " + err.what()); } } } // namespace gdal diff --git a/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp b/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp index 5f3710a..dc8ca09 100644 --- a/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp +++ b/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp @@ -1,7 +1,5 @@ #include "moja/modules/libpq/providerrelationallibpqpostgresql.h" -#include - #include #include @@ -46,7 +44,7 @@ class PostgresConnection { PQerrorMessage(_conn) % connInfo) .str(); MOJA_LOG_ERROR << "PostgresConnection: " << msg; - BOOST_THROW_EXCEPTION(datarepository::ConnectionFailedException() << datarepository::ConnectionError(msg)); + throw std::runtime_error("Error Postgres connection failed: " + msg); } PQsetNoticeReceiver(_conn, noticeReceiver, nullptr); PQsetNoticeProcessor(_conn, noticeProcessor, nullptr); @@ -329,8 +327,7 @@ class ProviderRelationalLibpqPostgreSQL::impl { PQerrorMessage(_conn) % _dbConnString) .str(); MOJA_LOG_ERROR << "ProviderRelationalLibpqPostgreSQL: " << msg; - BOOST_THROW_EXCEPTION(datarepository::QueryException() - << datarepository::SQL(query) << datarepository::SQLError(msg)); + throw std::runtime_error("Error " + msg); break; } case PGRES_COPY_BOTH: From aca7c0fe8465236d7ea724b207ae52a81956e61e Mon Sep 17 00:00:00 2001 From: Mal Date: Fri, 18 Jun 2021 11:03:39 +1000 Subject: [PATCH 39/40] Revert "Merge pull request #104 from MullionGroup/feature/exceptions_simplify" This reverts commit 1fd79798737b21c529f948f2854c5e55225235a5. --- .devcontainer/Dockerfile | 125 +------ .devcontainer/devcontainer.json | 81 +---- .devcontainer/docker-compose.yml | 96 ----- .vscode/settings.json | 13 +- Source/CMakeLists.txt | 16 +- Source/moja.cli/src/moja.cpp | 46 ++- Source/moja.core/CMakeLists.txt | 9 +- .../moja.core/include/moja/coreexceptions.h | 29 ++ Source/moja.core/include/moja/exception.h | 243 +++++++++++++ Source/moja.core/include/moja/filesystem.h | 14 - Source/moja.core/include/moja/status.h | 62 ---- Source/moja.core/src/datetime.cpp | 2 + Source/moja.core/src/environment_unix.cpp | 14 +- Source/moja.core/src/environment_win32.cpp | 24 +- Source/moja.core/src/environment_win32u.cpp | 26 +- Source/moja.core/src/exception.cpp | 114 ++++++ Source/moja.core/src/filesystem.cpp | 11 - Source/moja.core/src/logging.cpp | 41 ++- Source/moja.core/src/pocojsonutils.cpp | 79 +++-- Source/moja.datarepository/CMakeLists.txt | 1 + .../datarepository/datarepositoryexceptions.h | 60 ++++ .../datarepository/rasterreaderinterface.h | 15 +- .../include/moja/datarepository/stack.h | 1 + .../datarepository/tileblockcellindexer.h | 24 +- .../src/aspatialtileinfo.cpp | 5 +- .../src/aspatialtileinfocollection.cpp | 5 +- .../src/datarepository.cpp | 14 +- .../moja.datarepository/src/landunitinfo.cpp | 6 +- .../src/providerrelationalsqlite.cpp | 16 +- .../src/providerspatialrastertiled.cpp | 10 +- .../moja.datarepository/src/rasterreader.cpp | 31 +- .../moja.flint.configuration/CMakeLists.txt | 1 + .../configuration/configurationexceptions.h | 55 +++ .../src/configblock.cpp | 9 +- .../src/configcell.cpp | 5 +- .../src/configtile.cpp | 9 +- .../src/configuration.cpp | 8 +- .../src/json2configurationprovider.cpp | 50 +-- .../moja.flint.configuration/src/provider.cpp | 13 +- .../moja.flint.configuration/src/spinup.cpp | 5 +- Source/moja.flint/CMakeLists.txt | 25 +- .../include/moja/flint/aggregatorfilewriter.h | 4 +- .../flint/aspatiallocaldomaincontroller.h | 10 +- .../include/moja/flint/flintexceptions.h | 110 +++--- .../moja/flint/ilocaldomaincontroller.h | 35 +- .../moja/flint/localdomaincontrollerbase.h | 14 +- .../moja/flint/operationmanagersimplecache.h | 6 +- .../include/moja/flint/outputerstream.h | 3 +- .../include/moja/flint/outputerstreamflux.h | 3 +- .../moja/flint/recordaccumulatoruuid.h | 6 +- .../moja/flint/recordaccumulatorwithmutex.h | 22 +- .../flint/spatialtiledlocaldomaincontroller.h | 12 +- .../threadedaspatiallocaldomaincontroller.h | 19 +- .../include/moja/flint/writesystemconfig.h | 22 +- .../include/moja/flint/writevariablegrid.h | 31 +- .../moja.flint/src/aggregatorfilewriter.cpp | 26 +- .../src/aspatiallocaldomaincontroller.cpp | 69 ++-- Source/moja.flint/src/compositetransform.cpp | 17 +- Source/moja.flint/src/externalvariable.cpp | 7 +- Source/moja.flint/src/landunitcontroller.cpp | 11 +- Source/moja.flint/src/libraryfactory.cpp | 6 +- Source/moja.flint/src/librarymanager.cpp | 53 ++- .../src/localdomaincontrollerbase.cpp | 113 ++++-- .../moja.flint/src/lookuprandomtransform.cpp | 14 +- Source/moja.flint/src/lookuptransform.cpp | 17 +- .../moja.flint/src/operationmanagersimple.cpp | 27 +- .../src/operationmanagersimplecache.cpp | 6 +- Source/moja.flint/src/outputerstream.cpp | 82 +++-- Source/moja.flint/src/outputerstreamflux.cpp | 72 ++-- Source/moja.flint/src/poolsimplecache.cpp | 2 +- .../src/spatialtiledlocaldomaincontroller.cpp | 327 +++++++++++++++--- .../src/spinuplandunitcontroller.cpp | 5 +- Source/moja.flint/src/sqlquerytransform.cpp | 12 +- .../threadedaspatiallocaldomaincontroller.cpp | 18 +- Source/moja.flint/src/writesystemconfig.cpp | 81 +++-- Source/moja.flint/src/writevariablegrid.cpp | 150 ++++---- Source/moja.flint/tests/CMakeLists.txt | 2 + .../moja.flint/tests/src/matrixtestseigen.cpp | 301 ++++++++++++++++ .../moja.flint/tests/src/matrixtestsublas.cpp | 159 +++++++++ .../tests/src/operationmanagertestutils.cpp | 18 + .../tests/src/operationmanagerublastests.cpp | 155 +++++++++ .../moja/modules/gdal/writevariablegeotiff.h | 11 +- .../moja.modules.gdal/src/libraryfactory.cpp | 2 +- .../src/rasterreadergdal.cpp | 35 +- .../src/writevariablegeotiff.cpp | 145 +++++--- .../src/providerrelationallibpqpostgresql.cpp | 7 +- 86 files changed, 2528 insertions(+), 1132 deletions(-) delete mode 100644 .devcontainer/docker-compose.yml create mode 100644 Source/moja.core/include/moja/coreexceptions.h create mode 100644 Source/moja.core/include/moja/exception.h delete mode 100644 Source/moja.core/include/moja/filesystem.h delete mode 100644 Source/moja.core/include/moja/status.h create mode 100644 Source/moja.core/src/exception.cpp delete mode 100644 Source/moja.core/src/filesystem.cpp create mode 100644 Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h create mode 100644 Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h create mode 100644 Source/moja.flint/tests/src/matrixtestseigen.cpp create mode 100644 Source/moja.flint/tests/src/matrixtestsublas.cpp create mode 100644 Source/moja.flint/tests/src/operationmanagerublastests.cpp diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 44b3650..bb02d19 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,127 +1,6 @@ -#FROM mulliongroup/moja.mulliongroup.baseimage:ubuntu-20.04 -FROM gcr.io/flintpro-212105/flintpro/moja.mulliongroup.baseimage_vscode:ubuntu-20.04 as builder +FROM mojaglobal/vscode-baseimage:bionic ENV DEBIAN_FRONTEND=noninteractive -ARG CACHE_DATE=2021-05-27.1 -ARG FLINT_BRANCH=develop - -ENV ROOTDIR /usr - -RUN apt-get update \ - && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends gdb apt-utils dialog gettext sqlformat postgresql-client 2>&1 - -#RUN apt-get install -y apt-utils dialog gettext sqlformat - -# -# GET moja.global - Mullion Fork. Build Debug and Release -# -# RUN git clone --recursive --depth 1 -b ${FLINT_BRANCH} https://github.com/MullionGroup/FLINT.git flint \ -# && mkdir -p flint/Source/build && cd flint/Source/build \ -# && cmake -DCMAKE_BUILD_TYPE=DEBUG \ -# -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ -# -DENABLE_TESTS:BOOL=OFF \ -# -DENABLE_MOJA.MODULES.GDAL=ON \ -# -DDEBUG=ON \ -# -DBoost_USE_STATIC_LIBS=OFF \ -# -DBUILD_SHARED_LIBS=ON \ -# -DCMAKE_TOOLCHAIN_FILE=$ROOTDIR/src/vcpkg/scripts/buildsystems/vcpkg.cmake \ -# .. \ -# && make --quiet -j$(nproc) \ -# && make --quiet install \ -# && cd $ROOTDIR/src - -# RUN mkdir -p flint/Source/build_release && cd flint/Source/build_release \ -# && cmake -DCMAKE_BUILD_TYPE=RELEASE \ -# -DCMAKE_INSTALL_PREFIX=$ROOTDIR \ -# -DENABLE_TESTS:BOOL=OFF \ -# -DENABLE_MOJA.MODULES.GDAL=ON \ -# -DBoost_USE_STATIC_LIBS=OFF \ -# -DBUILD_SHARED_LIBS=ON \ -# -DCMAKE_TOOLCHAIN_FILE=$ROOTDIR/src/vcpkg/scripts/buildsystems/vcpkg.cmake \ -# .. \ -# && make --quiet -j$(nproc) \ -# && make --quiet install \ -# && cd $ROOTDIR/src - -# PROJ dependencies -RUN apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libsqlite3-0 libtiff5 libcurl4 \ - curl unzip ca-certificates - -# GDAL dependencies -RUN apt-get update -y; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - python3-numpy libpython3.8 \ - libexpat1 \ - libxerces-c3.2 \ - libwebp6 \ - libzstd1 bash libpq5 libssl1.1 libopenjp2-7 - -RUN apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -y python-is-python3 python3-pip - -WORKDIR /usr/src/vcpkg -RUN ./vcpkg install --triplet x64-linux boost-format boost-thread boost-filesystem boost-iostreams boost-program-options boost-random boost-crc boost-test libzip bzip2 liblzma zlib zstd snappy avro-cpp[snappy] - -# set environment variables -ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt -ENV GDAL_DATA /usr/share/gdal -ENV GDAL_HTTP_VERSION 2 - -ENV PATH $ROOTDIR/bin:$PATH -ENV LD_LIBRARY_PATH $ROOTDIR/lib:$ROOTDIR/src/vcpkg/installed/x64-linux/lib:$LD_LIBRARY_PATH -ENV PYTHONPATH /usr/lib:$PYTHONPATH - -RUN ln -s $ROOTDIR/lib/libmoja.modules.* $ROOTDIR/bin - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 - -RUN ldconfig - -### # TEST TEST -### FROM mcr.microsoft.com/vscode/devcontainers/base:ubuntu-20.04 as devtools -### -### # PROJ dependencies -### RUN apt-get update; \ -### DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ -### libsqlite3-0 libtiff5 libcurl4 \ -### curl unzip ca-certificates -### -### # GDAL dependencies -### RUN apt-get update -y; \ -### DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ -### python3-numpy libpython3.8 \ -### libexpat1 \ -### libxerces-c3.2 \ -### libwebp6 \ -### libzstd1 bash libpq5 libssl1.1 libopenjp2-7 -### -### RUN apt-get update; \ -### DEBIAN_FRONTEND=noninteractive apt-get install -y python-is-python3 python3-pip -### -### # set environment variables -### ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt -### ENV GDAL_DATA /usr/share/gdal -### ENV GDAL_HTTP_VERSION 2 -### -### ENV PATH /usr/bin:$PATH -### ENV LD_LIBRARY_PATH /usr/lib:$LD_LIBRARY_PATH -### ENV PYTHONPATH /usr/lib:$PYTHONPATH -### -### # ================================================================================================================== -### # Moja and Mullion libraries and Binaries -### # ================================================================================================================== -### -### COPY --from=builder /usr/lib/ /usr/lib/ -### COPY --from=builder /usr/bin/ /usr/bin/ -### COPY --from=builder /usr/share/ /usr/share/ -### COPY --from=builder /usr/include/gdal_version.h /usr/include -### -### RUN ldconfig -### # RUN ln -s /usr/lib/libmoja.modules.*.so /usr/bin/ - +# Switch back to dialog for any ad-hoc use of apt-get ENV DEBIAN_FRONTEND=dialog diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 09a5784..331d76b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,73 +1,30 @@ -// See https://aka.ms/vscode-remote/devcontainer.json for format details. +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.117.1/containers/cpp { - // See https://aka.ms/vscode-remote/devcontainer.json for format details. "name": "C++", - // "dockerFile": "Dockerfile", - "dockerComposeFile": "docker-compose.yml", - "service": "app", - "shutdownAction": "stopCompose", + "dockerFile": "Dockerfile", + "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - // "runArgs": [ - // "--cap-add=SYS_PTRACE", - // "--security-opt", - // "seccomp=unconfined", - // "--network", - // "rasterizer_net_backend" - // ], - "build": { - // "dockerfile": "docker-compose.yml", - "dockerComposeFile": "docker-compose.yml", - "service": "app", - "args": { - "NUM_CPU": "8", - "FLINT_BRANCH": "develop", - "MYARGFROMENVVAR": "${localEnv:VARIABLE_NAME}" - } + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" }, - "containerEnv": { - // "GOOGLE_APPLICATION_CREDENTIALS": "/workspaces/moja.mulliongroup_develop/sensitive/datalayer.json" - }, - "mounts": [ - // "source=${localWorkspaceFolder}/Run_Envs/bashhistory,target=/commandhistory,type=bind,consistency=cached" - ], - - // Default path to open when attaching to a new container. - // "workspaceFolder": "/workspaces/moja.mulliongroup_develop", - // "workspaceFolder": "${containerWorkspaceFolder}", - "workspaceFolder": "/workspaces", - - // Sets the run context to one level up instead of the .devcontainer folder. - // "context": "..", - - // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - // "dockerFile": "../Build/DockerFile.moja.global.ubuntu", - // "dockerFile": "../Build/DockerFile.moja.global.ubuntu.18.04", - - // The optional 'runArgs' property can be used to specify additional runtime arguments. - // "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], - // "runArgs": [ - // // Uncomment the next line if you want to use Docker from the container. See the docker-in-docker definition for details. - // // "-v","/var/run/docker.sock:/var/run/docker.sock", - - // // Uncomment the next line if you will be using a ptrace-based debugger like C++, Go, and Rust. - // "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" - // ], - - // Uncomment the next line if you want to publish any ports. - // "appPort": [], - - // Uncomment the next line if you want to add in default container specific settings.json values - // "settings": { "workbench.colorTheme": "Quiet Light" }, - // Uncomment the next line to run commands after the container is created. - // "postCreateCommand": "uname -a", - - // Add the IDs of any extensions you want installed in the array below. + // Add the IDs of extensions you want installed when the container is created. "extensions": [ "ms-vscode.cpptools", "austin.code-gnu-global", "twxs.cmake", - "ms-vscode.cmake-tools", - "mutantdino.resourcemonitor" + "ms-vscode.cmake-tools" ] + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" + } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml deleted file mode 100644 index 7a396a2..0000000 --- a/.devcontainer/docker-compose.yml +++ /dev/null @@ -1,96 +0,0 @@ -version: '3' - -services: - app: - # Using a Dockerfile is optional, but included for completeness. - security_opt: - - seccomp:unconfined - cap_add: - - SYS_PTRACE - build: - context: . - dockerfile: Dockerfile - # [Optional] You can use build args to set options. e.g. 'VARIANT' below affects the image in the Dockerfile - args: - NUM_CPU: 8 - FLINT_BRANCH: develop - # FLINT_BRANCH: feature-collection-updates - - volumes: - # This is where VS Code should expect to find your project's source code and the value of "workspaceFolder" in .devcontainer/devcontainer.json - # - ..:/workspaces:cached - - ..:/workspaces:delegated - - # Uncomment the next line to use Docker from inside the container. See https://aka.ms/vscode-remote/samples/docker-from-docker-compose for details. - # - /var/run/docker.sock:/var/run/docker.sock - - environment: - - GOOGLE_APPLICATION_CREDENTIALS=/workspaces/sensitive/datalayer.json - - # Overrides default command so things don't shut down after the process ends. - command: /bin/sh -c "while sleep 1000; do :; done" - - # Runs app on the same network as the service container, allows "forwardPorts" in devcontainer.json function. - # network_mode: service:rasterizer_net_backend - # network_mode: service:another-service - networks: - - net_backend - - # Use "forwardPorts" in **devcontainer.json** to forward an app port locally. - # (Adding the "ports" property to this file will not forward from a Codespace.) - - # Uncomment the next line to use a non-root user for all processes - See https://aka.ms/vscode-remote/containers/non-root for details. - # user: vscode - - # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. - # cap_add: - # - SYS_PTRACE - # security_opt: - # - seccomp:unconfined - -# flintpro-rasterizer: -# #image: gcr.io/flintpro-212105/flintpro-rasterizer:v20201015.03 -# # Thsi version is the new client side rasterizer -# image: gcr.io/flintpro-212105/flintpro-rasterizer:v20210525.01 -# hostname: flintpro-rasterizer -# ports: -# - 50051:50051 -# depends_on: -# - cloudsql -# command: '/run/rasterizer_server -p proxy:9090' -# restart: unless-stopped -# networks: -# - net_backend - -# proxy: -# image: gcr.io/flintpro-212105/proxy:v20200507.1 -# hostname: proxy -# # your own configurations for that app -# ports: -# - 9090:9090 -# depends_on: -# - cloudsql -# command: ["./wait-for-it.sh", "cloudsql:5432", "--", "/go/bin/featureservice-proxy", "-grpc-port", "9090", "-lookupConfigFile", "./proxy-config/lookup_config.json"] -# volumes: -# - ../Run_Envs/rasterizer/proxy-config:/proxy-config -# restart: unless-stopped -# networks: -# - net_backend - -# cloudsql: -# image: gcr.io/cloudsql-docker/gce-proxy:1.22.0 -# networks: -# - net_backend -# command: /cloud_sql_proxy -instances=flintpro-212105:us-central1:flintpro-qa-3=tcp:0.0.0.0:5432,flintpro-212105:us-central1:flintpro-prod=tcp:0.0.0.0:5433 -credential_file=/config/datalayer.json -# volumes: -# - ../sensitive:/config -# # - /mnt/c/Development/MullionGroup/cloud_sql:/config -# # - c:\Development\MullionGroup\cloud_sql:/config -# ports: -# - 5432:5432 -# restart: unless-stopped - -networks: - net_backend: - - # As in the "app" service, use "forwardPorts" in **devcontainer.json** to forward an app port locally. \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index dfde3a4..3e3cbd7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,18 +1,11 @@ { "cmake.configureArgs": [ "-DCMAKE_BUILD_TYPE=DEBUG", - "-DDEBUG=ON", "-DCMAKE_INSTALL_PREFIX=/usr/local", "-DENABLE_TESTS:BOOL=OFF", - "-DENABLE_MOJA.MODULES.GDAL=OFF", + "-DENABLE_MOJA.MODULES.GDAL=ON", "-DENABLE_MOJA.MODULES.LIBPQ=ON", "-DBoost_USE_STATIC_LIBS=OFF", - "-DBUILD_SHARED_LIBS=ON", - "-DCMAKE_TOOLCHAIN_FILE=/usr/src/vcpkg/scripts/buildsystems/vcpkg.cmake" - ], - "cmake.sourceDirectory": "${workspaceFolder}/Source", - "cmake.buildDirectory": "${workspaceFolder}/Source/vscodebuild/${buildKit}/${buildType}", - "files.associations": { - "iterator": "cpp" - } + "-DBUILD_SHARED_LIBS=ON" + ] } \ No newline at end of file diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 7f025ca..55d09d1 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.10.0) -project(moja VERSION 1.0.6 LANGUAGES C CXX) +project(moja VERSION 1.0.6 LANGUAGES CXX) #turn on using solution folders set_property( GLOBAL PROPERTY USE_FOLDERS ON) @@ -28,11 +28,11 @@ set(Moja_COMPONENTS "") # Allow enabling and disabling components option(ENABLE_MOJA.MODULES.ZIPPER "moja.modules.zipper" OFF) -option(ENABLE_MOJA.MODULES.POCO "moja.modules.poco" OFF) -option(ENABLE_MOJA.MODULES.LIBPQ "moja.modules.libpq" ON) -option(ENABLE_MOJA.MODULES.GDAL "moja.modules.gdal" ON) -option(ENABLE_MOJA.CLI "moja.cli" ON) -option(ENABLE_MOJA.SYSTEMTEST "moja.systemtest" OFF) +option(ENABLE_MOJA.MODULES.POCO "moja.modules.poco" OFF) +option(ENABLE_MOJA.MODULES.LIBPQ "moja.modules.libpq" OFF) +option(ENABLE_MOJA.MODULES.GDAL "moja.modules.gdal" OFF) +option(ENABLE_MOJA.CLI "moja.cli" ON) +option(ENABLE_MOJA.SYSTEMTEST "moja.systemtest" OFF) if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) set(CMAKE_INSTALL_PREFIX "C:/Development/Software/${PROJECT_NAME}" CACHE PATH "..." FORCE) @@ -127,10 +127,6 @@ if (CMAKE_SYSTEM MATCHES "Linux" ) else() message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++17 support. Please use a different C++ compiler.") endif() - - if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - set(CXX_FILESYSTEM_LIBRARIES "stdc++fs") - endif() endif(CMAKE_SYSTEM MATCHES "Linux" ) add_subdirectory(moja.core) diff --git a/Source/moja.cli/src/moja.cpp b/Source/moja.cli/src/moja.cpp index c280cd1..e148291 100644 --- a/Source/moja.cli/src/moja.cpp +++ b/Source/moja.cli/src/moja.cpp @@ -9,12 +9,15 @@ #include "moja/flint/moduleproxybase.h" #include "moja/flint/ioperationmanager.h" -#include -#include -#include +#include "moja/signals.h" +#include "moja/exception.h" +#include "moja/logging.h" +#include #include +#include + #include #include #include @@ -29,10 +32,9 @@ namespace conf = mf::configuration; using mf::configuration::LocalDomainType; using mf::ILocalDomainController; -namespace fs = moja::filesystem; - bool checkFilePath(const std::string& filePath) { - if (!fs::exists(filePath)) { + Poco::File file(filePath); + if (!file.exists()) { std::cerr << "File not found: " << filePath; return false; } @@ -288,18 +290,28 @@ int main(int argc, char* argv[]) { << ", Version: " << ldc->landUnitController().operationManager()->version() << ", Config: " << ldc->landUnitController().operationManager()->config(); - ldc->notification_center().postNotification(moja::signals::SystemInit); + ldc->_notificationCenter.postNotification(moja::signals::SystemInit); ldc->startup(); ldc->run(); ldc->shutdown(); - ldc->notification_center().postNotification(moja::signals::SystemShutdown); - } catch (const std::exception& e) { - MOJA_LOG_FATAL << e.what(); - return EXIT_FAILURE; - } catch (...) { - MOJA_LOG_FATAL << "Unknown exception"; - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; + ldc->_notificationCenter.postNotification(moja::signals::SystemShutdown); + } + catch (const moja::Exception& e) { + MOJA_LOG_FATAL << e.displayText(); + return EXIT_FAILURE; + } catch (const boost::exception& e) { + MOJA_LOG_FATAL << boost::diagnostic_information(e); + return EXIT_FAILURE; + } catch (const Poco::Exception& e) { + MOJA_LOG_FATAL << e.message(); + return EXIT_FAILURE; + } catch (const std::exception& e) { + MOJA_LOG_FATAL << e.what(); + return EXIT_FAILURE; + } catch (...) { + MOJA_LOG_FATAL << "Unknown exception"; + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; } diff --git a/Source/moja.core/CMakeLists.txt b/Source/moja.core/CMakeLists.txt index ff9aa69..d024677 100644 --- a/Source/moja.core/CMakeLists.txt +++ b/Source/moja.core/CMakeLists.txt @@ -6,7 +6,6 @@ include(${CMAKE_MODULE_PATH}/generate_product_version.cmake) find_package(Poco REQUIRED Foundation JSON) find_package(Boost COMPONENTS log log_setup REQUIRED) -find_package(fmt REQUIRED) if (MSVC) generate_product_version(ProductVersionFiles @@ -84,7 +83,7 @@ set(MOJA_Core_headers include/moja/dispatcher.h include/moja/dynamic.h include/moja/environment.h - include/moja/filesystem.h + include/moja/exception.h include/moja/floatcmp.h include/moja/hash.h include/moja/logging.h @@ -94,18 +93,18 @@ set(MOJA_Core_headers include/moja/pocojsonutils.h include/moja/signals.h include/moja/stopwatch.h - include/moja/status.h include/moja/timespan.h include/moja/types.h include/moja/version.h include/moja/publisher.h include/moja/utility.h + include/moja/coreexceptions.h ) set(MOJA_Core_sources src/datetime.cpp src/environment.cpp - src/filesystem.cpp + src/exception.cpp src/floatcmp.cpp src/logging.cpp src/mathex.cpp @@ -149,8 +148,6 @@ target_compile_definitions(${LIBNAME} target_link_libraries(${LIBNAME} PUBLIC Boost::log Boost::log_setup Poco::Foundation Poco::JSON - ${CXX_FILESYSTEM_LIBRARIES} - fmt::fmt-header-only PRIVATE ${SYSLIBS} ) diff --git a/Source/moja.core/include/moja/coreexceptions.h b/Source/moja.core/include/moja/coreexceptions.h new file mode 100644 index 0000000..3bb63aa --- /dev/null +++ b/Source/moja.core/include/moja/coreexceptions.h @@ -0,0 +1,29 @@ +#ifndef MOJA_CORE_COREEXCEPTIONS_H_ +#define MOJA_CORE_COREEXCEPTIONS_H_ + +#include "_core_exports.h" + +#include + +#include +#include + +namespace moja { + +struct CORE_API CoreException : virtual std::exception, virtual boost::exception {}; + +struct CORE_API CoreAssertionViolationException : virtual CoreException {}; +typedef boost::error_info AssertMsg; + +struct CORE_API CoreFileNotFoundException : virtual CoreException {}; +typedef boost::error_info FileName; + +struct CORE_API CoreNotImplementedException : virtual CoreException {}; +typedef boost::error_info Message; + +struct CORE_API CoreUnhandledType : virtual CoreException {}; +typedef boost::error_info TypeErrorMsg; + +} // namespace moja + +#endif // MOJA_CORE_COREEXCEPTIONS_H_ \ No newline at end of file diff --git a/Source/moja.core/include/moja/exception.h b/Source/moja.core/include/moja/exception.h new file mode 100644 index 0000000..4e1e654 --- /dev/null +++ b/Source/moja.core/include/moja/exception.h @@ -0,0 +1,243 @@ +#ifndef MOJA_CORE_EXCEPTION_H_ +#define MOJA_CORE_EXCEPTION_H_ + +#include "_core_exports.h" + +#include + +namespace moja { + +/** + * + * This is the base class for all exceptions defined in the moja Core class library. + * + */ +class CORE_API Exception : public std::exception { + public: + /** + * Creates an exception. + * + * The message. + * (Optional) the code. + */ + Exception(const std::string& msg, int code = 0); + + /** + * Creates an exception. + * + * The message. + * The extended message. + * (Optional) the code. + */ + Exception(const std::string& msg, const std::string& arg, int code = 0); + + /** + * Creates an exception and stores a clone of the nested exception. + * + * The message. + * The nested. + * (Optional) the code. + */ + Exception(const std::string& msg, const Exception& nested, int code = 0); + + /** + * Copy constructor. + */ + Exception(const Exception& exc); + + /** Destroys the exception and deletes the nested exception. */ + ~Exception() throw(); + + /** + * Assignment operator. + */ + Exception& operator=(const Exception& exc); + + /** + * Returns a static string describing the exception. + */ + virtual const char* name() const throw(); + + /** + * Returns the name of the exception class. + */ + virtual const char* className() const throw(); + + /** + * + * Returns a static string describing the exception. + * + * Same as name(), but for compatibility with std::exception. + * + */ + virtual const char* what() const throw(); + + /** + * + * Returns a pointer to the nested exception, or null if no nested exception exists. + * + */ + const Exception* nested() const; + + /** + * Returns the message text. + */ + const std::string& message() const; + + /** + * Returns the exception code if defined. + */ + int code() const; + + /** + * Returns a string consisting of the message name and the message text. + */ + std::string displayText() const; + + /** + * + * Creates an exact copy of the exception. + * + * The copy can later be thrown again by invoking rethrow() on it. + * + */ + virtual Exception* clone() const; + + /** + * + * (Re)Throws the exception. + * + * This is useful for temporarily storing a copy of an exception (see clone()), then throwing it + * again. + * + */ + virtual void rethrow() const; + + protected: + /** + * Standard constructor. + * + * (Optional) the code. + */ + Exception(int code = 0); + + /** + * Sets the message for the exception. + * + * The message. + */ + void message(const std::string& msg); + + /** + * Sets the extended message for the exception. + * + * The extended message. + */ + void extendedMessage(const std::string& arg); + + private: + std::string _msg; + Exception* _pNested; + int _code; +}; + +inline const Exception* Exception::nested() const { return _pNested; } + +inline const std::string& Exception::message() const { return _msg; } + +inline void Exception::message(const std::string& msg) { _msg = msg; } + +inline int Exception::code() const { return _code; } + +// Macros for quickly declaring and implementing exception classes. +// Unfortunately, we cannot use a template here because character +// pointers (which we need for specifying the exception name) +// are not allowed as template arguments. +#define MOJA_DECLARE_EXCEPTION_CODE(API, CLS, BASE, CODE) \ + class API CLS : public BASE { \ + public: \ + CLS(int code = CODE); \ + CLS(const std::string& msg, int code = CODE); \ + CLS(const std::string& msg, const std::string& arg, int code = CODE); \ + CLS(const std::string& msg, const moja::Exception& exc, int code = CODE); \ + CLS(const CLS& exc); \ + ~CLS() throw(); \ + CLS& operator=(const CLS& exc); \ + const char* name() const throw(); \ + const char* className() const throw(); \ + moja::Exception* clone() const; \ + void rethrow() const; \ + }; + +#define MOJA_DECLARE_EXCEPTION(API, CLS, BASE) MOJA_DECLARE_EXCEPTION_CODE(API, CLS, BASE, 0) + +#define MOJA_IMPLEMENT_EXCEPTION(CLS, BASE, NAME) \ + CLS::CLS(int code) : BASE(code) {} \ + CLS::CLS(const std::string& msg, int code) : BASE(msg, code) {} \ + CLS::CLS(const std::string& msg, const std::string& arg, int code) : BASE(msg, arg, code) {} \ + CLS::CLS(const std::string& msg, const moja::Exception& exc, int code) : BASE(msg, exc, code) {} \ + CLS::CLS(const CLS& exc) : BASE(exc) {} \ + CLS::~CLS() throw() {} \ + CLS& CLS::operator=(const CLS& exc) { \ + BASE::operator=(exc); \ + return *this; \ + } \ + const char* CLS::name() const throw() { return NAME; } \ + const char* CLS::className() const throw() { return typeid(*this).name(); } \ + moja::Exception* CLS::clone() const { return new CLS(*this); } \ + void CLS::rethrow() const { throw *this; } + +// Standard exception classes. +MOJA_DECLARE_EXCEPTION(CORE_API, LogicException, Exception) +MOJA_DECLARE_EXCEPTION(CORE_API, AssertionViolationException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, NullPointerException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, NullValueException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, BugcheckException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, InvalidArgumentException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, NotImplementedException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, RangeException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, IllegalStateException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, InvalidAccessException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, SignalException, LogicException) +MOJA_DECLARE_EXCEPTION(CORE_API, UnhandledException, LogicException) + +MOJA_DECLARE_EXCEPTION(CORE_API, RuntimeException, Exception) +MOJA_DECLARE_EXCEPTION(CORE_API, NotFoundException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, ExistsException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, TimeoutException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, SystemException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, RegularExpressionException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, LibraryLoadException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, LibraryAlreadyLoadedException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, LibrarySymbolLoadException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, NoThreadAvailableException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, PropertyNotSupportedException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, PoolOverflowException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, NoPermissionException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, OutOfMemoryException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, DataException, RuntimeException) + +MOJA_DECLARE_EXCEPTION(CORE_API, DataFormatException, DataException) +MOJA_DECLARE_EXCEPTION(CORE_API, SyntaxException, DataException) +MOJA_DECLARE_EXCEPTION(CORE_API, CircularReferenceException, DataException) +MOJA_DECLARE_EXCEPTION(CORE_API, PathSyntaxException, SyntaxException) +MOJA_DECLARE_EXCEPTION(CORE_API, IOException, RuntimeException) +MOJA_DECLARE_EXCEPTION(CORE_API, ProtocolException, IOException) +MOJA_DECLARE_EXCEPTION(CORE_API, FileException, IOException) +MOJA_DECLARE_EXCEPTION(CORE_API, FileExistsException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, FileNotFoundException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, PathNotFoundException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, FileReadOnlyException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, FileAccessDeniedException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, CreateFileException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, OpenFileException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, WriteFileException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, ReadFileException, FileException) +MOJA_DECLARE_EXCEPTION(CORE_API, UnknownURISchemeException, RuntimeException) + +MOJA_DECLARE_EXCEPTION(CORE_API, ApplicationException, Exception) +MOJA_DECLARE_EXCEPTION(CORE_API, BadCastException, RuntimeException) + +} // namespace moja + +#endif // MOJA_CORE_EXCEPTION_H_ \ No newline at end of file diff --git a/Source/moja.core/include/moja/filesystem.h b/Source/moja.core/include/moja/filesystem.h deleted file mode 100644 index b549d20..0000000 --- a/Source/moja.core/include/moja/filesystem.h +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "_core_exports.h" - -#include - -namespace moja::filesystem { - -using namespace std::filesystem; - -inline path CORE_API expand(const std::string& path); -inline path CORE_API expand(const path& path); - -} // namespace moja::filesystem diff --git a/Source/moja.core/include/moja/status.h b/Source/moja.core/include/moja/status.h deleted file mode 100644 index d6b3a08..0000000 --- a/Source/moja.core/include/moja/status.h +++ /dev/null @@ -1,62 +0,0 @@ -#pragma once - -#include -#include - -namespace moja { - -enum class status_code { - /// Not an error; returned on success. - Ok = 0, - - Error = 1, - Cancelled = 2, - Unknown = 3, -}; - -std::string StatusCodeToString(status_code code); -std::ostream& operator<<(std::ostream& os, status_code code); - -/** - * Reports error code and details. - * - * This class is modeled after `grpc::Status`. - */ -class status { - public: - status() = default; - - explicit status(status_code status_code, std::string message = "") - : code_(status_code), message_(std::move(message)) {} - - bool ok() const { return code_ == status_code::Ok; } - - status_code code() const { return code_; } - const std::string& message() const { return message_; } - - private: - status_code code_{status_code::Ok}; - std::string message_; -}; - -inline std::ostream& operator<<(std::ostream& os, status const& rhs) { - return os << rhs.message() << " [" << StatusCodeToString(rhs.code()) << "]"; -} - -inline bool operator==(status const& lhs, status const& rhs) { - return lhs.code() == rhs.code() && lhs.message() == rhs.message(); -} - -inline bool operator!=(status const& lhs, status const& rhs) { return !(lhs == rhs); } - -class runtime_status_error : public std::runtime_error { - public: - explicit runtime_status_error(status status); - - status const& status() const { return status_; } - - private: - moja::status status_; -}; - -} // namespace moja diff --git a/Source/moja.core/src/datetime.cpp b/Source/moja.core/src/datetime.cpp index 07f24d5..a290b67 100644 --- a/Source/moja.core/src/datetime.cpp +++ b/Source/moja.core/src/datetime.cpp @@ -1,5 +1,7 @@ #include "moja/datetime.h" +#include "moja/exception.h" + using namespace date; using namespace std::chrono; diff --git a/Source/moja.core/src/environment_unix.cpp b/Source/moja.core/src/environment_unix.cpp index 471bc87..b4b5cba 100644 --- a/Source/moja.core/src/environment_unix.cpp +++ b/Source/moja.core/src/environment_unix.cpp @@ -1,28 +1,28 @@ #include "moja/environment_unix.h" +#include + #include #include -#include #include namespace moja { std::string EnvironmentImpl::startProcessFolderImpl() { - namespace fs = std::filesystem; - static bool bHaveResult = false; - static char charBuffer[1024] = ""; + static char path[1024] = ""; if (!bHaveResult) { - if (readlink("/proc/self/exe", charBuffer, sizeof(charBuffer) - 1) == -1) { + if (readlink("/proc/self/exe", path, sizeof(path) - 1) == -1) { int ErrNo = errno; // unreachable return ""; } bHaveResult = true; } - fs::path path(charBuffer); - return path.remove_filename().string(); + auto folder = Poco::Path(path).parent(); + return folder.toString(); + ; } } // namespace moja diff --git a/Source/moja.core/src/environment_win32.cpp b/Source/moja.core/src/environment_win32.cpp index ad4725d..adc67fc 100644 --- a/Source/moja.core/src/environment_win32.cpp +++ b/Source/moja.core/src/environment_win32.cpp @@ -1,23 +1,23 @@ #include "moja/environment_win32.h" +#include +#include + #include -#include #include +#pragma comment(lib, "psapi.lib") + namespace moja { std::string EnvironmentImpl::startProcessFolderImpl() { - namespace fs = std::filesystem; - - std::vector charBuffer; - auto size = MAX_PATH; - do { - size *= 2; - charBuffer.resize(size); - } while (GetModuleFileNameA(NULL, charBuffer.data(), size) == size); - - fs::path path(charBuffer.data()); // Contains the full path including .exe - return path.remove_filename().string(); + static char path[512] = ""; + if (!path[0]) { + // Get directory this executable was launched from. + GetModuleFileNameA(NULL, path, sizeof(path) - 1); + } + auto folder = Poco::Path(path).parent(); + return folder.toString(); } } // namespace moja diff --git a/Source/moja.core/src/environment_win32u.cpp b/Source/moja.core/src/environment_win32u.cpp index d9c6967..e0ebdbf 100644 --- a/Source/moja.core/src/environment_win32u.cpp +++ b/Source/moja.core/src/environment_win32u.cpp @@ -1,23 +1,23 @@ #include "moja/environment_win32u.h" +#include +#include +#include + #include -#include #include namespace moja { -std::string EnvironmentImpl::startProcessFolderImpl() { - namespace fs = std::filesystem; - - std::vector charBuffer; - auto size = MAX_PATH; - do { - size *= 2; - charBuffer.resize(size); - } while (GetModuleFileNameW(NULL, charBuffer.data(), size) == size); - - fs::path path(charBuffer.data()); // Contains the full path including .exe - return path.remove_filename().string(); +static std::string EnvironmentImpl::startProcessFolderImpl() { + static wchar_t path[512] = ""; + if (!path[0]) { + // Get directory this executable was launched from. + GetModuleFileNameW(NULL, path, sizeof(path) - 1); + } + Poco::UnicodeConverter::toUTF8(path, result); + auto folder = Poco::Path(path).parent(); + return folder.toString(); } } // namespace moja diff --git a/Source/moja.core/src/exception.cpp b/Source/moja.core/src/exception.cpp new file mode 100644 index 0000000..ea7aa3b --- /dev/null +++ b/Source/moja.core/src/exception.cpp @@ -0,0 +1,114 @@ +#include "moja/exception.h" + +#include + +namespace moja { + +Exception::Exception(int code) : _pNested(0), _code(code) {} + +Exception::Exception(const std::string& msg, int code) : _msg(msg), _pNested(0), _code(code) {} + +Exception::Exception(const std::string& msg, const std::string& arg, int code) : _msg(msg), _pNested(0), _code(code) { + if (!arg.empty()) { + _msg.append(": "); + _msg.append(arg); + } +} + +Exception::Exception(const std::string& msg, const Exception& nested, int code) + : _msg(msg), _pNested(nested.clone()), _code(code) {} + +Exception::Exception(const Exception& exc) : std::exception(exc), _msg(exc._msg), _code(exc._code) { + _pNested = exc._pNested ? exc._pNested->clone() : 0; +} + +Exception::~Exception() throw() { delete _pNested; } + +Exception& Exception::operator=(const Exception& exc) { + if (&exc != this) { + delete _pNested; + _msg = exc._msg; + _pNested = exc._pNested ? exc._pNested->clone() : 0; + _code = exc._code; + } + + return *this; +} + +const char* Exception::name() const throw() { return "Exception"; } + +const char* Exception::className() const throw() { return typeid(*this).name(); } + +const char* Exception::what() const throw() { return name(); } + +std::string Exception::displayText() const { + std::string txt = name(); + if (!_msg.empty()) { + txt.append(": "); + txt.append(_msg); + } + return txt; +} + +void Exception::extendedMessage(const std::string& arg) { + if (!arg.empty()) { + if (!_msg.empty()) _msg.append(": "); + _msg.append(arg); + } +} + +Exception* Exception::clone() const { return new Exception(*this); } + +void Exception::rethrow() const { throw *this; } + +MOJA_IMPLEMENT_EXCEPTION(LogicException, Exception, "Logic exception") +MOJA_IMPLEMENT_EXCEPTION(AssertionViolationException, LogicException, "Assertion violation") +MOJA_IMPLEMENT_EXCEPTION(NullPointerException, LogicException, "Null pointer") +MOJA_IMPLEMENT_EXCEPTION(NullValueException, LogicException, "Null value") +MOJA_IMPLEMENT_EXCEPTION(BugcheckException, LogicException, "Bugcheck") +MOJA_IMPLEMENT_EXCEPTION(InvalidArgumentException, LogicException, "Invalid argument") +MOJA_IMPLEMENT_EXCEPTION(NotImplementedException, LogicException, "Not implemented") +MOJA_IMPLEMENT_EXCEPTION(RangeException, LogicException, "Out of range") +MOJA_IMPLEMENT_EXCEPTION(IllegalStateException, LogicException, "Illegal state") +MOJA_IMPLEMENT_EXCEPTION(InvalidAccessException, LogicException, "Invalid access") +MOJA_IMPLEMENT_EXCEPTION(SignalException, LogicException, "Signal received") +MOJA_IMPLEMENT_EXCEPTION(UnhandledException, LogicException, "Unhandled exception") + +MOJA_IMPLEMENT_EXCEPTION(RuntimeException, Exception, "Runtime exception") +MOJA_IMPLEMENT_EXCEPTION(NotFoundException, RuntimeException, "Not found") +MOJA_IMPLEMENT_EXCEPTION(ExistsException, RuntimeException, "Exists") +MOJA_IMPLEMENT_EXCEPTION(TimeoutException, RuntimeException, "Timeout") +MOJA_IMPLEMENT_EXCEPTION(SystemException, RuntimeException, "System exception") +MOJA_IMPLEMENT_EXCEPTION(RegularExpressionException, RuntimeException, "Error in regular expression") +MOJA_IMPLEMENT_EXCEPTION(LibraryLoadException, RuntimeException, "Cannot load library") +MOJA_IMPLEMENT_EXCEPTION(LibraryAlreadyLoadedException, RuntimeException, "Library already loaded") +MOJA_IMPLEMENT_EXCEPTION(LibrarySymbolLoadException, RuntimeException, "Cannot load library symbol") +MOJA_IMPLEMENT_EXCEPTION(NoThreadAvailableException, RuntimeException, "No thread available") +MOJA_IMPLEMENT_EXCEPTION(PropertyNotSupportedException, RuntimeException, "Property not supported") +MOJA_IMPLEMENT_EXCEPTION(PoolOverflowException, RuntimeException, "Pool overflow") +MOJA_IMPLEMENT_EXCEPTION(NoPermissionException, RuntimeException, "No permission") +MOJA_IMPLEMENT_EXCEPTION(OutOfMemoryException, RuntimeException, "Out of memory") +MOJA_IMPLEMENT_EXCEPTION(DataException, RuntimeException, "Data error") + +MOJA_IMPLEMENT_EXCEPTION(DataFormatException, DataException, "Bad data format") +MOJA_IMPLEMENT_EXCEPTION(SyntaxException, DataException, "Syntax error") +MOJA_IMPLEMENT_EXCEPTION(CircularReferenceException, DataException, "Circular reference") +MOJA_IMPLEMENT_EXCEPTION(PathSyntaxException, SyntaxException, "Bad path syntax") +MOJA_IMPLEMENT_EXCEPTION(IOException, RuntimeException, "I/O error") +MOJA_IMPLEMENT_EXCEPTION(ProtocolException, IOException, "Protocol error") +MOJA_IMPLEMENT_EXCEPTION(FileException, IOException, "File access error") +MOJA_IMPLEMENT_EXCEPTION(FileExistsException, FileException, "File exists") +MOJA_IMPLEMENT_EXCEPTION(FileNotFoundException, FileException, "File not found") +MOJA_IMPLEMENT_EXCEPTION(PathNotFoundException, FileException, "Path not found") +MOJA_IMPLEMENT_EXCEPTION(FileReadOnlyException, FileException, "File is read-only") +MOJA_IMPLEMENT_EXCEPTION(FileAccessDeniedException, FileException, "Access to file denied") +MOJA_IMPLEMENT_EXCEPTION(CreateFileException, FileException, "Cannot create file") +MOJA_IMPLEMENT_EXCEPTION(OpenFileException, FileException, "Cannot open file") +MOJA_IMPLEMENT_EXCEPTION(WriteFileException, FileException, "Cannot write file") +MOJA_IMPLEMENT_EXCEPTION(ReadFileException, FileException, "Cannot read file") +MOJA_IMPLEMENT_EXCEPTION(UnknownURISchemeException, RuntimeException, "Unknown URI scheme") + +MOJA_IMPLEMENT_EXCEPTION(ApplicationException, Exception, "Application exception") +MOJA_IMPLEMENT_EXCEPTION(BadCastException, RuntimeException, "Bad cast exception") + +} // namespace moja diff --git a/Source/moja.core/src/filesystem.cpp b/Source/moja.core/src/filesystem.cpp deleted file mode 100644 index 2d25e68..0000000 --- a/Source/moja.core/src/filesystem.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include "moja/filesystem.h" - -#include - -namespace moja::filesystem { - -path expand(const std::string& path) { return Poco::Path::expand(path); } - -path expand(const path& path) { return expand(path.string()); } - -} // namespace moja::filesystem \ No newline at end of file diff --git a/Source/moja.core/src/logging.cpp b/Source/moja.core/src/logging.cpp index 697b5ca..ce20133 100644 --- a/Source/moja.core/src/logging.cpp +++ b/Source/moja.core/src/logging.cpp @@ -1,7 +1,9 @@ #include "moja/logging.h" #include "moja/environment.h" -#include "moja/filesystem.h" + +#include +#include #include #include @@ -15,14 +17,13 @@ namespace moja { const std::string Logging::_defaultFileName = "logging.conf"; bool Logging::_explicitConfigurationFileSet = false; bool Logging::_explicitConfigurationTextSet = false; -std::string Logging::_explicitConfigurationFile; -std::string Logging::_explicitConfigurationText; +std::string Logging::_explicitConfigurationFile = ""; +std::string Logging::_explicitConfigurationText = ""; std::string Logging::_loggingConfigurationFile = "unknown"; void Logging::init() { namespace logging = boost::log; namespace attrs = boost::log::attributes; - namespace fs = moja::filesystem; static auto initialized = false; if (initialized) { @@ -33,8 +34,7 @@ void Logging::init() { logging::register_simple_filter_factory("Severity"); // Determine which log config to load, searching the working folder then the exe folder. - fs::path explicit_configuration_file(_explicitConfigurationFile); - if (_explicitConfigurationFileSet && fs::exists(explicit_configuration_file)) { + if (_explicitConfigurationFileSet && Poco::File(_explicitConfigurationFile).exists()) { std::ifstream loggingConfig(_explicitConfigurationFile); logging::init_from_stream(loggingConfig); _loggingConfigurationFile = _explicitConfigurationFile; @@ -44,26 +44,25 @@ void Logging::init() { boost::log::init_from_stream(s); _loggingConfigurationFile = "internal text"; } else { - auto current_path = fs::current_path().append(_defaultFileName); - if (fs::exists(current_path)) { - std::ifstream loggingConfig(current_path); + std::string filenameToCheck = Poco::Path::current() + _defaultFileName; + if (Poco::File(filenameToCheck).exists()) { + std::ifstream loggingConfig(filenameToCheck); logging::init_from_stream(loggingConfig); - _loggingConfigurationFile = current_path.string(); + _loggingConfigurationFile = filenameToCheck; } else { - auto start_path = fs::path(Environment::startProcessFolder() + _defaultFileName); - if (fs::exists(start_path)) { - std::ifstream loggingConfig(start_path); + filenameToCheck = moja::Environment::startProcessFolder() + _defaultFileName; + if (Poco::File(filenameToCheck).exists()) { + std::ifstream loggingConfig(filenameToCheck); logging::init_from_stream(loggingConfig); - _loggingConfigurationFile = start_path.string(); + _loggingConfigurationFile = filenameToCheck; } else { std::stringstream s; - s << R"([Sinks.console] - Destination=Console - Asynchronous = false - AutoFlush = true - Format = "<%TimeStamp%> (%Severity%) - %Message%" - Filter = "%Severity% >= info")" - << std::endl; + s << "[Sinks.console]" << std::endl; + s << "Destination=Console" << std::endl; + s << "Asynchronous = false" << std::endl; + s << "AutoFlush = true" << std::endl; + s << "Format = \"<%TimeStamp%> (%Severity%) - %Message%\"" << std::endl; + s << "Filter = \"%Severity% >= info\"" << std::endl; boost::log::init_from_stream(s); _loggingConfigurationFile = "internal default"; } diff --git a/Source/moja.core/src/pocojsonutils.cpp b/Source/moja.core/src/pocojsonutils.cpp index 6b89677..d00fe9c 100644 --- a/Source/moja.core/src/pocojsonutils.cpp +++ b/Source/moja.core/src/pocojsonutils.cpp @@ -1,8 +1,16 @@ #include "moja/pocojsonutils.h" +#include "moja/coreexceptions.h" +#include "moja/exception.h" + #include #include +#include + +using moja::FileName; +using moja::FileNotFoundException; +using moja::NotImplementedException; namespace moja { @@ -32,37 +40,44 @@ DynamicVector parsePocoJSONToDynamic(const Poco::JSON::Array::Ptr& val) { } else { auto object = val->get(i); if (object.isArray()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Array"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Array")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (object.isBoolean()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isDeque()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Deque"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Deque")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (object.isEmpty()) { } else if (object.isInteger()) { try { auto value = object.extract(); arrayDocument.push_back(value); - } catch (Poco::BadCastException&) { + } catch (Poco::BadCastException) { auto value = object.extract(); arrayDocument.push_back(value); } } else if (object.isList()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - List"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - List")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (object.isNumeric()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isSigned()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Signed"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Signed")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (object.isString()) { auto value = object.extract(); arrayDocument.push_back(value); } else if (object.isStruct()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Struct"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Struct")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (object.isVector()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Vector"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Vector")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Unexpected type"); + auto msg = (boost::format("Unhandled data type in parse of json into dynamic - Unexpected type")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } } } @@ -95,38 +110,44 @@ DynamicVar parsePocoJSONToDynamic(const Poco::JSON::Object::Ptr& val) { document[var.first] = subDocument; } else { if (var.second.isArray()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Array"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Array")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (var.second.isBoolean()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isDeque()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Deque"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Deque")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (var.second.isEmpty()) { } else if (var.second.isInteger()) { try { auto value = var.second.extract(); document[var.first] = value; - } catch (Poco::BadCastException&) { + } catch (Poco::BadCastException) { auto value = var.second.extract(); document[var.first] = value; } } else if (var.second.isList()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - List"); - + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - List")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (var.second.isNumeric()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isSigned()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Signed"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Signed")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (var.second.isString()) { auto value = var.second.extract(); document[var.first] = value; } else if (var.second.isStruct()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Struct"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Struct")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else if (var.second.isVector()) { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Vector"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Vector")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } else { - throw std::runtime_error("Unhandled data type in parse of json into dynamic - Unexpected type"); + auto msg = (boost::format("Unhandled data type in parse of json into Dynamic - Unexpected type")).str(); + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } } } @@ -164,7 +185,8 @@ DynamicVar parsePocoJSONToDynamic(const DynamicVar& data) { DynamicVar result = data.extract(); return result; } - throw std::runtime_error("Unhandled data type in parse of json into dynamic"); + auto msg = "Unhandled data type in parse of json into dynamic"; + BOOST_THROW_EXCEPTION(CoreAssertionViolationException() << moja::AssertMsg(msg)); } // -------------------------------------------------------------------------------------------- @@ -182,7 +204,7 @@ DynamicVar parsePocoVarToDynamic(const DynamicVar& var) { if (var.isInteger()) { try { return DynamicVar(var.extract()); - } catch (Poco::BadCastException&) { + } catch (Poco::BadCastException) { return DynamicVar(var.convert()); } } @@ -227,19 +249,24 @@ DynamicVar parsePocoVarToDynamic(const DynamicVar& var) { } } DynamicObject result; - for (auto [key, value] : s) { - result[key] = parsePocoVarToDynamic(value); + for (auto item : s) { + result[item.first] = parsePocoVarToDynamic(item.second); } return result; } - auto obj = var.extract(); - DynamicObject result; - for (auto [key, value] : *obj) { - result[key] = parsePocoVarToDynamic(value); + try { + auto obj = var.extract(); + DynamicObject result; + for (auto kvp : *obj) { + result[kvp.first] = parsePocoVarToDynamic(kvp.second); + } + return result; + } catch (std::exception) { } - return result; + auto msg = (boost::format("Unhandled data type in parse of Poco::Dynamic::Var into Dynamic")).str(); + BOOST_THROW_EXCEPTION(CoreUnhandledType() << moja::TypeErrorMsg(msg)); } } // namespace moja \ No newline at end of file diff --git a/Source/moja.datarepository/CMakeLists.txt b/Source/moja.datarepository/CMakeLists.txt index 23863af..ff090ac 100644 --- a/Source/moja.datarepository/CMakeLists.txt +++ b/Source/moja.datarepository/CMakeLists.txt @@ -75,6 +75,7 @@ set(MOJA_DataRepository_headers include/moja/${PACKAGE}/aspatialtileinfocollection.h include/moja/${PACKAGE}/cube.h include/moja/${PACKAGE}/datarepository.h + include/moja/${PACKAGE}/datarepositoryexceptions.h include/moja/${PACKAGE}/iproviderinterface.h include/moja/${PACKAGE}/iprovidernosqlinterface.h include/moja/${PACKAGE}/iproviderrelationalinterface.h diff --git a/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h b/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h new file mode 100644 index 0000000..bc56cf1 --- /dev/null +++ b/Source/moja.datarepository/include/moja/datarepository/datarepositoryexceptions.h @@ -0,0 +1,60 @@ +#ifndef MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ +#define MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ + +#include "moja/datarepository/_datarepository_exports.h" + +#include + +#include + +namespace moja { +namespace datarepository { + +struct DATAREPOSITORY_API DataRepositoryException : virtual std::exception, virtual boost::exception {}; + +struct DATAREPOSITORY_API AssertionViolationException : virtual DataRepositoryException {}; +typedef boost::error_info AssertMsg; + +struct DATAREPOSITORY_API ConnectionFailedException : virtual DataRepositoryException {}; +typedef boost::error_info ConnectionError; + +struct DATAREPOSITORY_API FileNotFoundException : virtual DataRepositoryException {}; +typedef boost::error_info FileName; + +struct DATAREPOSITORY_API FileReadException : virtual DataRepositoryException {}; +typedef boost::error_info FileName; +typedef boost::error_info Message; + +struct DATAREPOSITORY_API LayerNotFoundException : virtual DataRepositoryException {}; +typedef boost::error_info LayerName; + +struct DATAREPOSITORY_API NotImplementedException : virtual DataRepositoryException {}; +typedef boost::error_info Message; + +struct DATAREPOSITORY_API QueryException : virtual DataRepositoryException {}; +typedef boost::error_info SQL; +typedef boost::error_info SQLError; + +struct DATAREPOSITORY_API LandscapeDefinitionException : virtual DataRepositoryException {}; +typedef boost::error_info Component; +typedef boost::error_info Constraint; + +struct DATAREPOSITORY_API TileBlockCellIndexerInvalidParameterException : virtual DataRepositoryException {}; +typedef boost::error_info Component; +typedef boost::error_info Constraint; + +// -- Provider exceptions +struct DATAREPOSITORY_API ProviderUnsupportedException : virtual DataRepositoryException {}; +typedef boost::error_info ProviderType; + +struct DATAREPOSITORY_API ProviderNotFoundException : virtual DataRepositoryException {}; +typedef boost::error_info ProviderType; + +struct DATAREPOSITORY_API ProviderAlreadyExistsException : virtual DataRepositoryException {}; +typedef boost::error_info ProviderName; +typedef boost::error_info ProviderType; + +} // namespace datarepository +} // namespace moja + +#endif // MOJA_DATAREPOSITORY_DATAREPOSITORYEXCEPTIONS_H_ diff --git a/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h b/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h index d32f079..825a315 100644 --- a/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h +++ b/Source/moja.datarepository/include/moja/datarepository/rasterreaderinterface.h @@ -5,7 +5,9 @@ #include #include -#include + +#include +#include #include #include @@ -15,8 +17,6 @@ #include #include -namespace fs = moja::filesystem; - namespace moja { namespace datarepository { @@ -42,7 +42,8 @@ class DATAREPOSITORY_API MetaDataRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool MetaDataRasterReaderInterface::file_exists(const std::string& path) { - return fs::exists(path); + Poco::File pf(path); + return pf.exists(); } // -------------------------------------------------------------------------------------------- @@ -87,7 +88,8 @@ class DATAREPOSITORY_API TileRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool TileRasterReaderInterface::file_exists(const std::string& path) { - return fs::exists(path); + Poco::File pf(path); + return pf.exists(); } // -------------------------------------------------------------------------------------------- @@ -132,7 +134,8 @@ class DATAREPOSITORY_API StackRasterReaderInterface { // -------------------------------------------------------------------------------------------- inline bool StackRasterReaderInterface::file_exists(const std::string& path) { - return fs::exists(path); + Poco::File pf(path); + return pf.exists(); } // -------------------------------------------------------------------------------------------- diff --git a/Source/moja.datarepository/include/moja/datarepository/stack.h b/Source/moja.datarepository/include/moja/datarepository/stack.h index 828addf..be310cd 100644 --- a/Source/moja.datarepository/include/moja/datarepository/stack.h +++ b/Source/moja.datarepository/include/moja/datarepository/stack.h @@ -2,6 +2,7 @@ #define MOJA_DATAREPOSITORY_STACK_H_ #include "moja/datarepository/cube.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include diff --git a/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h b/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h index 73bea04..ed11db7 100644 --- a/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h +++ b/Source/moja.datarepository/include/moja/datarepository/tileblockcellindexer.h @@ -2,16 +2,16 @@ #define MOJA_DATAREPOSITORY_TILEBLOCKCELLINDEXER_H_ #include "moja/datarepository/_datarepository_exports.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include #include +#include #include #include -#include #include #include -#include namespace moja { namespace datarepository { @@ -183,14 +183,16 @@ struct DATAREPOSITORY_API BlockIdx { if (t_idx > indexer.tileDesc.indexLimit - 1) { std::stringstream ss; - ss << "tile index > index limit " << "tile.X:" << tile.X << " tile.Y:" << tile.Y; - throw std::range_error(ss.str()); + ss << "tile.X:" << tile.X << "tile.Y:" << tile.Y; + BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() + << Component(ss.str()) << Constraint("tileindex > indexLimit")); } if (b_idx > indexer.blockDesc.indexLimit - 1) { std::stringstream ss; - ss << "block index > index limit block.X:" << block.X << "block.Y:" << block.Y; - throw std::range_error(ss.str()); + ss << "block.X:" << block.X << "block.Y:" << block.Y; + BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() + << Component(ss.str()) << Constraint("blockindex > indexLimit")); } tileIdx = t_idx; @@ -201,14 +203,16 @@ struct DATAREPOSITORY_API BlockIdx { : tileCols(indexer.tileDesc.cols), blockCols(indexer.blockDesc.cols) { if (tileIdx > indexer.tileDesc.indexLimit - 1) { std::stringstream ss; - ss << "tile index > index limit tileIdx:" << tileIdx; - throw std::invalid_argument(ss.str()); + ss << "tileIdx:" << tileIdx; + BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() + << Component(ss.str()) << Constraint("tileindex > indexLimit")); } if (blockIdx > indexer.blockDesc.indexLimit - 1) { std::stringstream ss; - ss << "block index > index limit blockIdx:" << blockIdx; - throw std::invalid_argument(ss.str()); + ss << "blockIdx:" << blockIdx; + BOOST_THROW_EXCEPTION(TileBlockCellIndexerInvalidParameterException() + << Component(ss.str()) << Constraint("blockindex > indexLimit")); } this->tileIdx = tileIdx; diff --git a/Source/moja.datarepository/src/aspatialtileinfo.cpp b/Source/moja.datarepository/src/aspatialtileinfo.cpp index a21242f..e664cdb 100644 --- a/Source/moja.datarepository/src/aspatialtileinfo.cpp +++ b/Source/moja.datarepository/src/aspatialtileinfo.cpp @@ -1,6 +1,9 @@ #include "moja/datarepository/aspatialtileinfo.h" #include "moja/datarepository/aspatialtileinfocollection.h" +#include "moja/datarepository/datarepositoryexceptions.h" + +using moja::datarepository::LandscapeDefinitionException; namespace moja { namespace datarepository { @@ -8,7 +11,7 @@ namespace datarepository { AspatialTileInfo::AspatialTileInfo(const AspatialTileInfoCollection& tileInfoCollection, int id) : _tileInfoCollection(&tileInfoCollection) { if (id < 1) { - std::invalid_argument("Error tile id less than zero: " + std::to_string(id)); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("id") << Constraint("> 0")); } _id = id; diff --git a/Source/moja.datarepository/src/aspatialtileinfocollection.cpp b/Source/moja.datarepository/src/aspatialtileinfocollection.cpp index 702d48e..a124e92 100644 --- a/Source/moja.datarepository/src/aspatialtileinfocollection.cpp +++ b/Source/moja.datarepository/src/aspatialtileinfocollection.cpp @@ -1,6 +1,7 @@ #include "moja/datarepository/aspatialtileinfocollection.h" #include "moja/datarepository/aspatialtileinfo.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/iproviderrelationalinterface.h" #include @@ -15,11 +16,11 @@ namespace datarepository { AspatialTileInfoCollection::AspatialTileInfoCollection(const IProviderRelationalInterface& provider, int maxTileSize, int tileCacheSize) { if (maxTileSize < 1) { - throw std::invalid_argument("Error maxTileSize less that 1 " + std::to_string(maxTileSize)); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("maxTileSize") << Constraint("> 0")); } if (tileCacheSize < 1) { - throw std::invalid_argument("Error tileCacheSize less that 1 " + std::to_string(tileCacheSize)); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("tileCacheSize") << Constraint("> 0")); } _provider = &provider; diff --git a/Source/moja.datarepository/src/datarepository.cpp b/Source/moja.datarepository/src/datarepository.cpp index 1717fc8..55a77ef 100644 --- a/Source/moja.datarepository/src/datarepository.cpp +++ b/Source/moja.datarepository/src/datarepository.cpp @@ -1,10 +1,13 @@ #include "moja/datarepository/datarepository.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/providerrelationalsqlite.h" #include "moja/datarepository/providerspatialrastertiled.h" #include +#include + namespace moja { namespace datarepository { @@ -19,14 +22,14 @@ std::shared_ptr DataRepository::createProvider(const std::st const std::string& providerType, const DynamicObject& settings) const { const auto providerKey = ProviderKey(library, providerType); - const auto it = _providerRegistry->find(providerKey); + auto it = _providerRegistry->find(providerKey); if (it == _providerRegistry->end()) { - throw std::runtime_error("Error unsupported provider " + library + ":" + providerType); + BOOST_THROW_EXCEPTION(moja::datarepository::ProviderUnsupportedException() << ProviderType(providerType)); } auto providerInit = _providerRegistry->at(providerKey); auto providerInterface = providerInit(settings); if (providerInterface == nullptr) { - throw std::runtime_error("Error provider init failed " + library + ":" + providerType); + BOOST_THROW_EXCEPTION(moja::datarepository::ProviderUnsupportedException() << ProviderType(providerType)); } return providerInterface; } @@ -37,7 +40,8 @@ void DataRepository::addProvider(const std::string& name, const std::string& pro if (p == _providers.end()) { _providers.insert(std::pair>(name, provider)); } else { - throw std::runtime_error("Error provider already exists " + name + ":" + providerType); + BOOST_THROW_EXCEPTION(moja::datarepository::ProviderAlreadyExistsException() + << ProviderName(name) << ProviderType(providerType)); } } @@ -46,7 +50,7 @@ std::shared_ptr DataRepository::getProvider(const std::strin if (p != _providers.end()) { return p->second; } - throw std::runtime_error("Error provider not found " + name); + BOOST_THROW_EXCEPTION(moja::datarepository::ProviderNotFoundException() << ProviderName(name)); } } // namespace datarepository diff --git a/Source/moja.datarepository/src/landunitinfo.cpp b/Source/moja.datarepository/src/landunitinfo.cpp index cd78888..36d076a 100644 --- a/Source/moja.datarepository/src/landunitinfo.cpp +++ b/Source/moja.datarepository/src/landunitinfo.cpp @@ -1,20 +1,22 @@ #include "moja/datarepository/landunitinfo.h" #include "moja/datarepository/aspatialtileinfo.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include +using moja::datarepository::LandscapeDefinitionException; namespace moja { namespace datarepository { LandUnitInfo::LandUnitInfo(const AspatialTileInfo& tile, Int64 id, double area) { if (id < 1) { - throw std::invalid_argument("Error id less that 1 " + std::to_string(id)); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("id") << Constraint("> 0")); } if (area <= 0.0) { - throw std::invalid_argument("Error area less than or equal to 0.0 " + std::to_string(area)); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("area") << Constraint("> 0")); } _tile = &tile; diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index 4a0ed49..ba15231 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -1,14 +1,13 @@ #include "moja/datarepository/providerrelationalsqlite.h" -#include +#include "moja/datarepository/datarepositoryexceptions.h" +#include #include #include #include -namespace fs = moja::filesystem; - namespace moja { namespace datarepository { @@ -26,13 +25,14 @@ class SQLiteConnection { sqlite3_config(SQLITE_CONFIG_URI, 1); if (path.find(":memory:") == std::string::npos) { - if (!fs::exists(path)) { - throw std::runtime_error("Error SQLite database file not found " + path); + Poco::File file(path); + if (!file.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(path)); } } if (sqlite3_open_v2(path.c_str(), &_conn, SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX, 0) != SQLITE_OK) { - throw std::runtime_error("Error SQLite database connection failed: " + std::string(sqlite3_errmsg(_conn))); + BOOST_THROW_EXCEPTION(ConnectionFailedException() << ConnectionError(sqlite3_errmsg(_conn))); } sqlite3_busy_timeout(_conn, 60000); @@ -52,7 +52,7 @@ class SQLiteStatement { SQLiteStatement(const SQLiteConnection& conn, const std::string& query) { if (sqlite3_prepare_v2(conn, query.c_str(), int(query.size()), &_stmt, nullptr) != SQLITE_OK) { - throw std::runtime_error("Error SQLite prepare failed: " + std::string(sqlite3_errmsg(conn))); + BOOST_THROW_EXCEPTION(QueryException() << SQL(query) << SQLError(sqlite3_errmsg(conn))); } } @@ -122,7 +122,7 @@ class ProviderRelationalSQLite::impl { return std::vector(); break; default: - throw std::runtime_error("Error SQLite query failed: " + std::string(sqlite3_errmsg(_conn))); + BOOST_THROW_EXCEPTION(QueryException() << SQL(query) << SQLError(sqlite3_errmsg(_conn))); } std::vector result; diff --git a/Source/moja.datarepository/src/providerspatialrastertiled.cpp b/Source/moja.datarepository/src/providerspatialrastertiled.cpp index 1b4af52..124c5bb 100644 --- a/Source/moja.datarepository/src/providerspatialrastertiled.cpp +++ b/Source/moja.datarepository/src/providerspatialrastertiled.cpp @@ -108,7 +108,7 @@ ProviderSpatialRasterTiled::ProviderSpatialRasterTiled( layerblockLonSize, layercellLatSize, layercellLonSize), layer["nodata"], layer, attributeTable))); } else { - throw std::runtime_error("Unsuported layer data type: " + layerData); + BOOST_THROW_EXCEPTION(ProviderUnsupportedException() << ProviderType(layerData)); } } else if (layerType == "StackLayer") { // if (rasterReaderFactorySet.stackReaderFactory() == nullptr) { @@ -160,7 +160,7 @@ ProviderSpatialRasterTiled::ProviderSpatialRasterTiled( layerblockLonSize, layercellLatSize, layercellLonSize), layer["nodata"], layer))); } else { - throw std::runtime_error("Unsuported layer data type: " + layerData); + BOOST_THROW_EXCEPTION(ProviderUnsupportedException() << ProviderType(layerData)); } } else { // TODO: should throw exception @@ -176,12 +176,14 @@ DynamicVar ProviderSpatialRasterTiled::GetValue(const std::string& name, double } const IProviderLayer* ProviderSpatialRasterTiled::getLayer(const std::string& name) const { - if (_layers.find(name) == _layers.end()) throw std::runtime_error("Layer not found " + name); + if (_layers.find(name) == _layers.end()) BOOST_THROW_EXCEPTION(LayerNotFoundException() << LayerName(name)); return _layers.at(name).get(); } inline const TileBlockCellIndexer& ProviderSpatialRasterTiled::indexer(const std::string& layerName) { - const auto* layer = getLayer(layerName); + if (_layers.find(layerName) == _layers.end()) + BOOST_THROW_EXCEPTION(LayerNotFoundException() << LayerName(layerName)); + auto layer = _layers.at(layerName); return layer->indexer(); } diff --git a/Source/moja.datarepository/src/rasterreader.cpp b/Source/moja.datarepository/src/rasterreader.cpp index cb6d04d..fbfb3f9 100644 --- a/Source/moja.datarepository/src/rasterreader.cpp +++ b/Source/moja.datarepository/src/rasterreader.cpp @@ -1,32 +1,34 @@ #include "moja/datarepository/rasterreader.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/datarepository/tileblockcellindexer.h" #include #include -#include +#include +#include #include +#include +#include #include - -#include +#include #include +#include namespace moja { namespace datarepository { -namespace fs = moja::filesystem; - // -------------------------------------------------------------------------------------------- FlintMetaDataRasterReader::FlintMetaDataRasterReader(const std::string& path, const std::string& prefix, const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { - - const auto mata_path = fs::absolute(path) / fmt::format("{}.json", prefix); - _metaPath = mata_path.string(); + auto filePath = Poco::Path(path); + auto abs = filePath.absolute().toString(); + _metaPath = (boost::format("%1%%2%%3%.json") % abs % Poco::Path::separator() % prefix).str(); } DynamicObject FlintMetaDataRasterReader::readMetaData() const { @@ -40,7 +42,7 @@ DynamicObject FlintMetaDataRasterReader::readMetaData() const { auto layerMetadata = parsePocoJSONToDynamic(metadata).extract(); return layerMetadata; } else { - throw std::runtime_error("Error metadata file not found " + _metaPath); + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(_metaPath)); } } @@ -49,8 +51,7 @@ DynamicObject FlintMetaDataRasterReader::readMetaData() const { FlintTileRasterReader::FlintTileRasterReader(const std::string& path, const Point& origin, const std::string& prefix, const TileBlockCellIndexer& indexer, const DynamicObject& settings) : TileRasterReaderInterface(path, origin, prefix, indexer, settings) { - const auto tile_path = fs::path(path) / fmt::format("{}_{}.blk", prefix, tile_id(origin)); - _tilePath = tile_path.string(); + _tilePath = (boost::format("%1%%2%%3%_%4%.blk") % path % Poco::Path::separator() % prefix % tile_id(origin)).str(); } FlintTileRasterReader::~FlintTileRasterReader() {} @@ -101,7 +102,7 @@ void FlintTileRasterReader::readFlintBlockData(const BlockIdx& blk_idx, char* bl fileStream.open(_tilePath, std::ios::binary); std::streampos blockStart = blk_idx.blockIdx * block_size; if (fileStream.fail()) { - throw std::runtime_error("Error failed to open block data " + _tilePath + std::string(strerror(errno))); + BOOST_THROW_EXCEPTION(FileReadException() << FileName(_tilePath) << Message(strerror(errno))); } fileStream.seekg(blockStart); @@ -118,9 +119,7 @@ void FlintTileRasterReader::readFlintBlockData(const BlockIdx& blk_idx, char* bl FlintStackRasterReader::FlintStackRasterReader(const std::string& path, const Point& origin, const std::string& prefix, const TileBlockCellIndexer& indexer, const DynamicObject& settings) : StackRasterReaderInterface(path, origin, prefix, indexer, settings) { - - const auto tile_path = fs::path(path) / fmt::format("{}_{}.blk", prefix, stack_id(origin)); - _tilePath = tile_path.string(); + _tilePath = (boost::format("%1%%2%%3%_%4%.blk") % path % Poco::Path::separator() % prefix % stack_id(origin)).str(); } FlintStackRasterReader::~FlintStackRasterReader() {} @@ -172,7 +171,7 @@ void FlintStackRasterReader::readFlintBlockData(const BlockIdx& blk_idx, int nSe fileStream.open(_tilePath, std::ios::binary); std::streampos blockStart = blk_idx.blockIdx * block_size; if (fileStream.fail()) { - throw std::runtime_error("Error failed to open block data " + _tilePath + std::string(strerror(errno))); + BOOST_THROW_EXCEPTION(FileReadException() << FileName(_tilePath) << Message(strerror(errno))); } fileStream.seekg(blockStart); diff --git a/Source/moja.flint.configuration/CMakeLists.txt b/Source/moja.flint.configuration/CMakeLists.txt index 4dc97c7..01a32c4 100644 --- a/Source/moja.flint.configuration/CMakeLists.txt +++ b/Source/moja.flint.configuration/CMakeLists.txt @@ -27,6 +27,7 @@ configure_file(../templates/exports.h ${CMAKE_CURRENT_SOURCE_DIR}/include/moja/f set(MOJA_FLINT_Configuration_headers include/moja/flint/${PACKAGE}/_${PACKAGE}_exports.h include/moja/flint/configuration/configuration.h + include/moja/flint/configuration/configurationexceptions.h include/moja/flint/configuration/configtile.h include/moja/flint/configuration/configblock.h include/moja/flint/configuration/configcell.h diff --git a/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h b/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h new file mode 100644 index 0000000..f26d2e9 --- /dev/null +++ b/Source/moja.flint.configuration/include/moja/flint/configuration/configurationexceptions.h @@ -0,0 +1,55 @@ +#ifndef MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ +#define MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ + +#include "moja/flint/configuration/_configuration_exports.h" + +#include + +#include +#include + +namespace moja { +namespace flint { +namespace configuration { + +struct CONFIGURATION_API ConfigurationException : virtual std::exception, virtual boost::exception {}; + +struct CONFIGURATION_API AssertionViolationException : virtual ConfigurationException {}; +typedef boost::error_info AssertMsg; + +struct CONFIGURATION_API FileNotFoundException : virtual ConfigurationException {}; +typedef boost::error_info FileName; + +struct CONFIGURATION_API ModuleOrderOverlapException : virtual ConfigurationException {}; +typedef boost::error_info Order; +typedef boost::error_info> ModuleNames; + +struct CONFIGURATION_API ModuleParamsException : virtual ConfigurationException {}; +typedef boost::error_info Param; + +struct CONFIGURATION_API SpinupModuleOrderOverlapException : virtual ConfigurationException {}; +typedef boost::error_info Order; +typedef boost::error_info> SpinupModuleNames; + +struct CONFIGURATION_API LandscapeDefinitionException : virtual ConfigurationException {}; +typedef boost::error_info Component; +typedef boost::error_info Constraint; + +/// -- Provider exceptions +struct CONFIGURATION_API ProviderSettingsException : virtual ConfigurationException {}; +typedef boost::error_info ProviderName; +typedef boost::error_info ProviderLibrary; + +struct CONFIGURATION_API ProviderMissingTypeException : virtual ConfigurationException {}; +typedef boost::error_info ProviderName; + +struct CONFIGURATION_API ProviderInvalidNameTypeException : virtual ConfigurationException {}; +typedef boost::error_info ProviderName; +typedef boost::error_info ProviderLibrary; +typedef boost::error_info ProviderType; + +} // namespace configuration +} // namespace flint +} // namespace moja + +#endif // MOJA_FLINT_CONFIGURATION_CONFIGURATIONEXCEPTIONS_H_ diff --git a/Source/moja.flint.configuration/src/configblock.cpp b/Source/moja.flint.configuration/src/configblock.cpp index d66e083..6030361 100644 --- a/Source/moja.flint.configuration/src/configblock.cpp +++ b/Source/moja.flint.configuration/src/configblock.cpp @@ -1,6 +1,7 @@ #include "moja/flint/configuration/configblock.h" #include "moja/flint/configuration/configtile.h" +#include "moja/flint/configuration/configurationexceptions.h" #include #include @@ -13,19 +14,19 @@ namespace configuration { ConfigBlock::ConfigBlock(const ConfigTile& tile, int row, int col, int blockSizeX, int blockSizeY) : _tile(tile) { if (row < 0) { - throw std::invalid_argument("Error in config block row < 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("row") << Constraint(">= 0")); } if (col < 0) { - throw std::invalid_argument("Error in config block col < 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("col") << Constraint(">= 0")); } if (blockSizeX <= 0) { - throw std::invalid_argument("Error in config block blockSizeX <= 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("blockSizeX") << Constraint("> 0")); } if (blockSizeY <= 0) { - throw std::invalid_argument("Error in config block blockSizeY <= 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("blockSizeY") << Constraint("> 0")); } _row = row; diff --git a/Source/moja.flint.configuration/src/configcell.cpp b/Source/moja.flint.configuration/src/configcell.cpp index 906bc6f..c9ec13f 100644 --- a/Source/moja.flint.configuration/src/configcell.cpp +++ b/Source/moja.flint.configuration/src/configcell.cpp @@ -1,6 +1,7 @@ #include "moja/flint/configuration/configcell.h" #include "moja/flint/configuration/configblock.h" +#include "moja/flint/configuration/configurationexceptions.h" #include #include @@ -14,11 +15,11 @@ namespace configuration { ConfigCell::ConfigCell(const ConfigBlock& block, int row, int col) : _block(block) { if (row < 0) { - throw std::invalid_argument("Error in config cell row < 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("row") << Constraint(">= 0")); } if (col < 0) { - throw std::invalid_argument("Error in config cell col < 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("col") << Constraint(">= 0")); } _row = row; diff --git a/Source/moja.flint.configuration/src/configtile.cpp b/Source/moja.flint.configuration/src/configtile.cpp index 75b9c6c..845f6ae 100644 --- a/Source/moja.flint.configuration/src/configtile.cpp +++ b/Source/moja.flint.configuration/src/configtile.cpp @@ -1,6 +1,7 @@ #include "moja/flint/configuration/configtile.h" #include "moja/flint/configuration/configblock.h" +#include "moja/flint/configuration/configurationexceptions.h" #include @@ -15,19 +16,19 @@ namespace configuration { ConfigTile::ConfigTile(int xIndex, int yIndex, double xSize, double ySize, int xPixels, int yPixels) { if (xSize <= 0.0) { - throw std::invalid_argument("Error in config tile xSize <= 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("xSize") << Constraint("> 0.0")); } if (ySize <= 0.0) { - throw std::invalid_argument("Error in config tile ySize <= 0"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("ySize") << Constraint("> 0.0")); } if (xPixels < 1) { - throw std::invalid_argument("Error in config tile xPixels < 1"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("xPixels") << Constraint("> 0")); } if (yPixels < 1) { - throw std::invalid_argument("Error in config tile yPixels < 1"); + BOOST_THROW_EXCEPTION(LandscapeDefinitionException() << Component("yPixels") << Constraint("> 0")); } _xIndex = xIndex; diff --git a/Source/moja.flint.configuration/src/configuration.cpp b/Source/moja.flint.configuration/src/configuration.cpp index f2f9d65..369667f 100644 --- a/Source/moja.flint.configuration/src/configuration.cpp +++ b/Source/moja.flint.configuration/src/configuration.cpp @@ -1,5 +1,6 @@ #include "moja/flint/configuration/configuration.h" +#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/externalpool.h" #include "moja/flint/configuration/externalvariable.h" #include "moja/flint/configuration/flintdatavariable.h" @@ -261,12 +262,11 @@ void Configuration::addModule(const std::string& libraryName, const std::string& DynamicObject settings) { // It is an error for two modules to have the same order - simulation // results could be inconsistent. - const auto sameOrder = std::find_if(_modules.begin(), _modules.end(), - [order](std::shared_ptr other) { return other->order() == order; }); + auto sameOrder = std::find_if(_modules.begin(), _modules.end(), + [order](std::shared_ptr other) { return other->order() == order; }); if (sameOrder != _modules.end()) { - throw std::runtime_error("Error module order overlap in " + libraryName + " " + name + " = " + - (*sameOrder)->name()); + throw ModuleOrderOverlapException() << Order(order) << ModuleNames({libraryName, name, (*sameOrder)->name()}); } auto module = std::make_shared(libraryName, name, order, isProxy, settings); diff --git a/Source/moja.flint.configuration/src/json2configurationprovider.cpp b/Source/moja.flint.configuration/src/json2configurationprovider.cpp index daf9c70..5245e8b 100644 --- a/Source/moja.flint.configuration/src/json2configurationprovider.cpp +++ b/Source/moja.flint.configuration/src/json2configurationprovider.cpp @@ -1,18 +1,20 @@ #include "moja/flint/configuration/json2configurationprovider.h" +#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/iterationbase.h" #include "moja/flint/configuration/iterationtileindex.h" #include "moja/flint/configuration/library.h" #include "moja/flint/configuration/localdomain.h" #include "moja/flint/configuration/spinup.h" -#include #include #include +#include #include #include #include +#include #include #include @@ -26,36 +28,37 @@ using Poco::JSON::Stringifier; #include - namespace moja { namespace flint { namespace configuration { -namespace fs = moja::filesystem; - JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath) { - for (const auto& config_file : configFilePath) { - if (!fs::exists(config_file)) { - throw std::runtime_error("Error cant find config file " + config_file); + for (const std::string configFie : configFilePath) { + Poco::File file(configFie); + if (!file.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); } } _configFilePath = configFilePath; + //_configProviderFilePath = null; _hasProviderConfigFile = false; } JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath, const std::vector& configProviderFilePath) { - for (const auto& config_file : configFilePath) { - if (!fs::exists(config_file)) { - throw std::runtime_error("Error cant find config file " + config_file); + for (const auto& configFie : configFilePath) { + Poco::File file(configFie); + if (!file.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); } } _configFilePath = std::move(configFilePath); if (!configProviderFilePath.empty()) { - for (const auto& config_provider_file : configProviderFilePath) { - if (!fs::exists(config_provider_file)) { - throw std::runtime_error("Error cant find provider config file " + config_provider_file); + for (const auto& configProviderFile : configProviderFilePath) { + Poco::File fileProvider(configProviderFile); + if (!fileProvider.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configProviderFile)); } } _configProviderFilePath = std::move(configProviderFilePath); @@ -401,7 +404,7 @@ void JSON2ConfigurationProvider::createSpinup(DynamicVar& parsedJSON, Configurat if (enabled) { for (auto param : {"sequencer_library", "sequencer", "simulateLandUnit", "landUnitBuildSuccess"}) { if (!spinupStruct.contains(param)) { - throw std::runtime_error("Error missing spin up parameter " + std::string(param)); + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param((boost::format("Spinup.%1%") % param).str())); } } @@ -438,17 +441,18 @@ void JSON2ConfigurationProvider::createLibraries(DynamicVar& parsedJSON, Configu } bool JSON2ConfigurationProvider::fileExists(const std::string& path) { - return fs::exists(path); + Poco::File pf(path); + return pf.exists(); } void JSON2ConfigurationProvider::createProviders(DynamicVar& parsedJSON, Configuration& config) const { auto jsonStruct2 = *parsedJSON.extract(); auto provider = jsonStruct2.getObject("Providers"); auto& data = *(provider.get()); - for (auto& [name, var] : data) { - auto d = parsePocoJSONToDynamic(var); + for (auto& item : data) { + auto d = parsePocoJSONToDynamic(item.second); if (d.isEmpty()) { - throw std::runtime_error("Error provider settings are empty " + name); + BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(item.first)); } auto xx = d.extract(); std::string libName; @@ -459,7 +463,7 @@ void JSON2ConfigurationProvider::createProviders(DynamicVar& parsedJSON, Configu const std::string providerType = d["type"].convert(); auto& settings = d.extract(); - config.addProvider(name, libName, providerType, settings); + config.addProvider(item.first, libName, providerType, settings); } } @@ -630,10 +634,10 @@ void JSON2ConfigurationProvider::createModules(DynamicVar& parsedJSON, Configura if (!enabled) continue; } if (!moduleStruct.contains("library")) { - throw std::runtime_error("Error module definition missing library"); + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); } if (!moduleStruct.contains("order")) { - throw std::runtime_error("Error module definition missing order"); + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); } const auto& moduleLibraryName = moduleStruct["library"].extract(); @@ -662,10 +666,10 @@ void JSON2ConfigurationProvider::createSpinupModules(DynamicVar& parsedJSON, Con } if (!moduleStruct.contains("library")) { - throw std::runtime_error("Error spin up module definition missing library"); + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); } if (!moduleStruct.contains("order")) { - throw std::runtime_error("Error spin up module definition missing order"); + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); } const auto moduleLibraryName = moduleStruct["library"].extract(); diff --git a/Source/moja.flint.configuration/src/provider.cpp b/Source/moja.flint.configuration/src/provider.cpp index b794f78..f12c2e8 100644 --- a/Source/moja.flint.configuration/src/provider.cpp +++ b/Source/moja.flint.configuration/src/provider.cpp @@ -1,5 +1,7 @@ #include "moja/flint/configuration/provider.h" +#include "moja/flint/configuration/configurationexceptions.h" + #include namespace moja { @@ -10,16 +12,19 @@ Provider::Provider(const std::string& name, const std::string& library, const st const DynamicObject& settings) : _name(name), _library(library), _providerType(providerType), _settings(settings) { if (name.length() == 0 || all(name, boost::algorithm::is_space())) { - throw std::invalid_argument("Error provider name is empty"); + BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() + << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); } if (library.length() == 0 || all(library, boost::algorithm::is_space())) { - throw std::invalid_argument("Error provider library is empty"); + BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() + << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); } if (providerType.length() == 0 || all(providerType, boost::algorithm::is_space())) { - throw std::invalid_argument("Error provider type is empty"); + BOOST_THROW_EXCEPTION(ProviderInvalidNameTypeException() + << ProviderName(name) << ProviderLibrary(library) << ProviderType(providerType)); } if (settings.empty() || !settings.contains("type")) { - throw std::invalid_argument("Error provider settings is empty or invalid"); + BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(name) << ProviderLibrary(library)); } } diff --git a/Source/moja.flint.configuration/src/spinup.cpp b/Source/moja.flint.configuration/src/spinup.cpp index 9095357..f60d60b 100644 --- a/Source/moja.flint.configuration/src/spinup.cpp +++ b/Source/moja.flint.configuration/src/spinup.cpp @@ -1,6 +1,7 @@ #include "moja/flint/configuration/spinup.h" #include "moja/flint/configuration/configuration.h" +#include "moja/flint/configuration/configurationexceptions.h" #include "moja/flint/configuration/externalvariable.h" #include "moja/flint/configuration/flintdatavariable.h" #include "moja/flint/configuration/localdomain.h" @@ -51,8 +52,8 @@ void Spinup::addSpinupModule(const std::string& libraryName, const std::string& [order](std::shared_ptr other) { return other->order() == order; }); if (sameOrder != _modules.end()) { - throw std::runtime_error("Error spin up module order overlap in " + libraryName + " " + name + " = " + - (*sameOrder)->name()); + throw SpinupModuleOrderOverlapException() + << Order(order) << SpinupModuleNames({libraryName, name, (*sameOrder)->name()}); } auto module = std::make_shared(libraryName, name, order, createNew, settings); diff --git a/Source/moja.flint/CMakeLists.txt b/Source/moja.flint/CMakeLists.txt index ad9a6f4..44e88a2 100644 --- a/Source/moja.flint/CMakeLists.txt +++ b/Source/moja.flint/CMakeLists.txt @@ -179,6 +179,18 @@ set(MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS include/moja/flint/poolsimplecache.h ) +set(MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS + include/moja/flint/matrixublas.h + include/moja/flint/operationmanagerublas.h + include/moja/flint/operationresultfluxublas.h + include/moja/flint/operationresultfluxiteratorublas.h + include/moja/flint/operationresultublas.h + include/moja/flint/operationproportionalublas.h + include/moja/flint/operationstockublas.h + include/moja/flint/operationtransferublas.h + include/moja/flint/poolublas.h +) + set(MOJA_FLINT_MODULE_SOURCES src/aggregatorfluxstep.cpp src/aggregatorstockstep.cpp @@ -259,6 +271,14 @@ set(MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE src/operationstocksimplecache.cpp ) +set(MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE + src/poolublas.cpp + src/operationmanagerublas.cpp + src/operationproportionalublas.cpp + src/operationresultublas.cpp + src/operationstockublas.cpp +) + source_group("header files\\other" FILES ${MOJA_FLINT_HEADERS}) source_group("source files\\other" FILES ${MOJA_FLINT_SOURCES}) source_group("header files\\modules" FILES ${MOJA_FLINT_MODULE_HEADERS}) @@ -272,6 +292,8 @@ source_group("header files\\op_man_simple" FILES ${MOJA_FLINT_OPERATION_MANAGE source_group("source files\\op_man_simple" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_SOURCE}) source_group("header files\\op_man_simplecache" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS}) source_group("source files\\op_man_simplecache" FILES ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE}) +source_group("header files\\op_man_ublas" FILES ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS}) +source_group("source files\\op_man_ublas" FILES ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE}) set (SRCS ${MOJA_FLINT_GENERATED_HEADERS} ${MOJA_FLINT_SOURCES} ${MOJA_FLINT_HEADERS} @@ -280,7 +302,8 @@ set (SRCS ${MOJA_FLINT_GENERATED_HEADERS} ${MOJA_FLINT_FLINTDATA_SOURCES} ${MOJA_FLINT_FLINTDATA_HEADERS} ${MOJA_FLINT_IOPERATION_MANAGER_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLE_SOURCE} - ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE} ) + ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_SIMPLECACHE_SOURCE} + ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_HEADERS} ${MOJA_FLINT_OPERATION_MANAGER_UBLAS_SOURCE}) add_library( ${LIBNAME} ${LIB_MODE} ${SRCS} ${ProductVersionFiles}) diff --git a/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h b/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h index 636d62f..e62186c 100644 --- a/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h +++ b/Source/moja.flint/include/moja/flint/aggregatorfilewriter.h @@ -17,7 +17,7 @@ typedef Poco::Tuple runStatDa class FLINT_API AggregatorFileWriter : public flint::ModuleBase { public: - AggregatorFileWriter(std::mutex& fileMutex, AggregatorLandUnitSharedData& aggregatorLandUnitSharedData, + AggregatorFileWriter(Poco::Mutex& fileMutex, AggregatorLandUnitSharedData& aggregatorLandUnitSharedData, std::shared_ptr> dateDimension, std::shared_ptr> date2Dimension, std::shared_ptr> poolInfoDimension, @@ -47,7 +47,7 @@ class FLINT_API AggregatorFileWriter : public flint::ModuleBase { void writeFlux(); // -- Shared Data - std::mutex& _fileMutex; + Poco::Mutex& _fileMutex; AggregatorLandUnitSharedData& _aggregatorLandUnitSharedData; std::shared_ptr> _dateDimension; std::shared_ptr> _date2Dimension; diff --git a/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h index 2962d7b..7b130a0 100644 --- a/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/aspatiallocaldomaincontroller.h @@ -20,12 +20,12 @@ class FLINT_API AspatialLocalDomainController final : public LocalDomainControll AspatialLocalDomainController() : _spinupLandUnitController(_landUnitController), _runSpinUp(false) {} ~AspatialLocalDomainController() = default; - status configure(const configuration::Configuration& config) override; - status run() override; + void configure(const configuration::Configuration& config) override; + void run() override; + void startup() override; + void shutdown() override; void run(moja::datarepository::AspatialTileInfo& tile); - status startup() override; - status shutdown() override; private: std::unique_ptr _landscape; @@ -36,7 +36,7 @@ class FLINT_API AspatialLocalDomainController final : public LocalDomainControll LocalDomainControllerBase::ModuleMap _spinupModules; IVariable* _luId; - status run(const moja::datarepository::LandUnitInfo& lu); + void run(const moja::datarepository::LandUnitInfo& lu); }; } // namespace flint diff --git a/Source/moja.flint/include/moja/flint/flintexceptions.h b/Source/moja.flint/include/moja/flint/flintexceptions.h index ded06f9..567f4aa 100644 --- a/Source/moja.flint/include/moja/flint/flintexceptions.h +++ b/Source/moja.flint/include/moja/flint/flintexceptions.h @@ -1,55 +1,65 @@ -#pragma once +#ifndef MOJA_FLINT_FLINTEXCEPTIONS_H_ +#define MOJA_FLINT_FLINTEXCEPTIONS_H_ #include "moja/flint/_flint_exports.h" -#include -#include +#include + #include +#include + +namespace moja { +namespace flint { + +struct FLINT_API FLINTException : virtual std::exception, virtual boost::exception {}; + +struct FLINT_API PoolNotFoundException : virtual FLINTException {}; +typedef boost::error_info PoolName; +typedef boost::error_info PoolIndex; + +struct FLINT_API VariableNotFoundException : virtual FLINTException {}; +typedef boost::error_info VariableName; + +struct FLINT_API VariableEmptyWhenValueExpectedException : virtual FLINTException {}; +typedef boost::error_info VariableName; + +struct FLINT_API SequencerNotFoundException : virtual FLINTException {}; +typedef boost::error_info Details; +typedef boost::error_info LibraryName; +typedef boost::error_info SequencerName; + +struct FLINT_API TableNotFoundException : virtual FLINTException {}; +typedef boost::error_info TableName; + +struct FLINT_API PreconditionViolatedException : virtual FLINTException {}; +typedef boost::error_info Precondition; + +struct FLINT_API LandscapeDefinitionException : virtual FLINTException {}; +typedef boost::error_info Component; +typedef boost::error_info Constraint; + +struct FLINT_API IncompleteConfigurationException : virtual FLINTException {}; +typedef boost::error_info Item; +typedef boost::error_info Details; + +struct FLINT_API DuplicateModuleDefinedException : virtual FLINTException {}; +typedef boost::error_info Details; +typedef boost::error_info LibraryName; +typedef boost::error_info ModuleName; + +struct FLINT_API LocalDomainError : virtual FLINTException {}; +typedef boost::error_info Details; +typedef boost::error_info LibraryName; +typedef boost::error_info ModuleName; +typedef boost::error_info ErrorCode; + +struct FLINT_API SimulationError : virtual FLINTException {}; +typedef boost::error_info Details; +typedef boost::error_info LibraryName; +typedef boost::error_info ModuleName; +typedef boost::error_info ErrorCode; + +} // namespace flint +} // namespace moja -namespace moja::flint { - -class FLINT_API flint_exception : public std::runtime_error { - public: - flint_exception(const std::string& message) : std::runtime_error(message) {} - flint_exception(const char* message) : std::runtime_error(message) {} -}; - -class FLINT_API local_domain_error : public flint_exception { - public: - local_domain_error(const std::string& message, std::string library_name = "", std::string module_name = "", - int error_code = -1) - : flint_exception(message), - library_name_(std::move(library_name)), - module_name_(std::move(module_name)), - error_code_(error_code) {} - - [[nodiscard]] const std::string& library_name() const noexcept { return library_name_; } - [[nodiscard]] const std::string& module_name() const noexcept { return module_name_; } - [[nodiscard]] int error_code() const noexcept { return error_code_; } - - private: - std::string library_name_; - std::string module_name_; - int error_code_; -}; - -class FLINT_API simulation_error : public flint_exception { - public: - simulation_error(const std::string& message, std::string library_name = "", std::string module_name = "", - int error_code = -1) - : flint_exception(message), - library_name_(std::move(library_name)), - module_name_(std::move(module_name)), - error_code_(error_code) {} - - [[nodiscard]] const std::string& library_name() const noexcept { return library_name_; } - [[nodiscard]] const std::string& module_name() const noexcept { return module_name_; } - [[nodiscard]] int error_code() const noexcept { return error_code_; } - - private: - std::string library_name_; - std::string module_name_; - int error_code_; -}; - -} // namespace moja::flint +#endif // MOJA_FLINT_FLINTEXCEPTIONS_H_ diff --git a/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h index 703ecbc..a3b1020 100644 --- a/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/ilocaldomaincontroller.h @@ -1,9 +1,10 @@ -#pragma once +#ifndef MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ +#define MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ #include -#include -namespace moja::flint { +namespace moja { +namespace flint { class IVariable; class IModule; class ILandUnitController; @@ -14,18 +15,18 @@ class SequencerModuleBase; class ILocalDomainController { public: - using ModuleMapKey = std::pair; - using ModuleMap = std::map>; + typedef std::pair ModuleMapKey; + typedef std::map> ModuleMap; - virtual ~ILocalDomainController() = default; + virtual ~ILocalDomainController(void) = default; // configure the simulation - virtual status configure(const configuration::Configuration& config) = 0; + virtual void configure(const configuration::Configuration& config) = 0; // execute the simulation - virtual status run() = 0; - virtual status startup() = 0; - virtual status shutdown() = 0; + virtual void run() = 0; + virtual void startup() = 0; + virtual void shutdown() = 0; virtual int localDomainId() = 0; virtual void set_localDomainId(const int value) = 0; @@ -34,9 +35,17 @@ class ILocalDomainController { virtual std::map modules() const = 0; - virtual NotificationCenter& notification_center() = 0; - virtual const NotificationCenter& notification_center() const = 0; + // every simulation needs a notification center + NotificationCenter _notificationCenter; + // every simulation needs a sequencer + std::shared_ptr _sequencer; + + // Variable used to check if landUnit should be simulated + const flint::IVariable* _simulateLandUnit; + flint::IVariable* _landUnitBuildSuccess; }; -} // namespace moja::flint +} // namespace flint +} // namespace moja +#endif // MOJA_FLINT_ILOCALDOMAINCONTROLLER_H_ diff --git a/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h b/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h index 2a980a6..29cb035 100644 --- a/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h +++ b/Source/moja.flint/include/moja/flint/localdomaincontrollerbase.h @@ -20,10 +20,10 @@ class FLINT_API LocalDomainControllerBase : public ILocalDomainController { explicit LocalDomainControllerBase(std::shared_ptr libraryHandles); virtual ~LocalDomainControllerBase(void) = default; - status configure(const configuration::Configuration& config) override; - status run() override; - status startup() override; - status shutdown() override; + void configure(const configuration::Configuration& config) override; + void run() override; + void startup() override; + void shutdown() override; ILandUnitController& landUnitController() override { return _landUnitController; }; virtual void initialiseData() { _landUnitController.initialiseData(false); } @@ -32,14 +32,8 @@ class FLINT_API LocalDomainControllerBase : public ILocalDomainController { void set_localDomainId(const int value) override { _localDomainId = value; } std::map modules() const override; - NotificationCenter& notification_center() override { return _notificationCenter; } - const NotificationCenter& notification_center() const override { return _notificationCenter; } protected: - NotificationCenter _notificationCenter; - std::shared_ptr _sequencer; - const IVariable* _simulateLandUnit; - IVariable* _landUnitBuildSuccess; int _localDomainId; const configuration::Configuration* _config; LandUnitController _landUnitController; diff --git a/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h b/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h index e1fb244..dd5a768 100644 --- a/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h +++ b/Source/moja.flint/include/moja/flint/operationmanagersimplecache.h @@ -5,6 +5,8 @@ #include "moja/flint/ioperationmanager.h" #include "moja/flint/operationresultcollection.h" +#include + namespace moja { namespace flint { class IModule; @@ -50,8 +52,8 @@ class FLINT_API OperationManagerSimpleCache : public IOperationManager { const IPool* addPool(const std::string& name, const std::string& description, const std::string& units, double scale, int order, const std::shared_ptr transform) override { - throw std::logic_error("addPool not implemented"); - } + throw moja::NotImplementedException(); + }; const IPool* addPool(const std::string& name, double initValue = 0.0) override; const IPool* addPool(const std::string& name, const std::string& description, const std::string& units, double scale, int order, double initValue = 0.0) override; diff --git a/Source/moja.flint/include/moja/flint/outputerstream.h b/Source/moja.flint/include/moja/flint/outputerstream.h index c2a3b78..9a9a533 100644 --- a/Source/moja.flint/include/moja/flint/outputerstream.h +++ b/Source/moja.flint/include/moja/flint/outputerstream.h @@ -27,7 +27,8 @@ class FLINT_API OutputerStream : public ModuleBase { virtual void subscribe(NotificationCenter& notificationCenter) override; virtual void outputHeader(std::ostream& stream) const; - virtual void outputOnNotification(const std::string& notification, std::ostream& stream); + virtual void outputInit(std::ostream& stream); + virtual void outputEndStep(const std::string& notification, std::ostream& stream); virtual void outputShutdown(std::ostream& stream); void onSystemInit() override; diff --git a/Source/moja.flint/include/moja/flint/outputerstreamflux.h b/Source/moja.flint/include/moja/flint/outputerstreamflux.h index 4d24c3b..f3d891d 100644 --- a/Source/moja.flint/include/moja/flint/outputerstreamflux.h +++ b/Source/moja.flint/include/moja/flint/outputerstreamflux.h @@ -16,7 +16,8 @@ class FLINT_API OutputerStreamFlux : public ModuleBase { explicit OutputerStreamFlux() : ModuleBase(), _outputToScreen(false), _outputInfoHeader(false) {} virtual ~OutputerStreamFlux() {} - void outputOnNotification(const std::string& notification, std::ostream& stream) const; + void outputInit(std::ostream& stream) const; + void outputEndStep(std::ostream& stream) const; void outputShutdown(std::ostream& stream) const; void outputHeader(std::ostream& stream) const; diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h b/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h index 961edbd..1425859 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatoruuid.h @@ -5,6 +5,8 @@ #include +#include + #include #include @@ -43,7 +45,7 @@ class RecordAccumulatorUuid { RecordAccumulatorUuid() {} const TRecord* accumulate(TRecord record) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { // Found an existing ID for the key. @@ -103,7 +105,7 @@ class RecordAccumulatorUuid { const rec_accu_vec& records() const { return _records; }; private: - std::mutex _mutex; + Poco::Mutex _mutex; rec_accu_set _recordsIdx; rec_accu_vec _records; boost::uuids::random_generator _uuidGenerator; diff --git a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h index bfd9cc1..7a6e3af 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulatorwithmutex.h @@ -6,9 +6,9 @@ #include +#include #include -#include #include namespace moja { @@ -23,7 +23,7 @@ class RecordAccumulatorWithMutex { typedef typename rec_accu_vec::size_type rec_accu_size_type; Record* insert(Int64 id, std::shared_ptr> record) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); // ID has been assigned by user, assume that we can run with this _nextId = id + 1; // can't guarantee that this will be called in 'id increasing' order but a good guess perhaps record->setId(id); @@ -33,7 +33,7 @@ class RecordAccumulatorWithMutex { } Record* accumulate(std::shared_ptr> record) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); auto record_ptr = record.get(); auto it = _recordsIdx.find(record_ptr); if (it != _recordsIdx.end()) { @@ -51,7 +51,7 @@ class RecordAccumulatorWithMutex { } Record* accumulate(std::shared_ptr> record, Int64 requestedId) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); auto record_ptr = record.get(); auto it = _recordsIdx.find(record_ptr); if (it != _recordsIdx.end()) { @@ -105,13 +105,12 @@ class RecordAccumulatorWithMutex { const rec_accu_vec& records() const { return _records; }; private: - std::mutex _mutex; + Poco::Mutex _mutex; Int64 _nextId = 1; rec_accu_set _recordsIdx; rec_accu_vec _records; }; -// template template class RecordAccumulatorWithMutex2 { struct RecordComparer { @@ -140,7 +139,7 @@ class RecordAccumulatorWithMutex2 { RecordAccumulatorWithMutex2() : _nextId(1) {} const TRecord* insert(Int64 id, TRecord record) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); if (_records.size() == _records.capacity()) { _recordsIdx.clear(); _records.reserve(compute_size()); @@ -160,7 +159,7 @@ class RecordAccumulatorWithMutex2 { const TRecord* accumulate(TRecord record) { return accumulate(record, -1); } const TRecord* accumulate(TRecord record, Int64 requestedId) { - std::unique_lock lock(_mutex); + Poco::Mutex::ScopedLock lock(_mutex); auto it = _recordsIdx.find(&record); if (it != _recordsIdx.end()) { // Found an existing ID for the key. @@ -221,17 +220,12 @@ class RecordAccumulatorWithMutex2 { _records.clear(); } - void shrink_to_fit() { - _recordsIdx.clear(); - _records.shrink_to_fit(); - } - rec_accu_size_type size() { return _records.size(); } const rec_accu_vec& records() const { return _records; }; private: - std::mutex _mutex; + Poco::Mutex _mutex; std::atomic _nextId; rec_accu_set _recordsIdx; rec_accu_vec _records; diff --git a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h index 933b7d1..75ab107 100644 --- a/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/spatialtiledlocaldomaincontroller.h @@ -10,8 +10,6 @@ #include #include -#include - #include #include @@ -121,12 +119,12 @@ class FLINT_API SpatialTiledLocalDomainController : public LocalDomainController bool isThread = false); ~SpatialTiledLocalDomainController(void) = default; - status configure(const flint::configuration::Configuration& config) override; - status run() override; + void configure(const flint::configuration::Configuration& config) override; + void run() override; - status startup() override; + void startup() override; - status shutdown() override; + void shutdown() override; private: class InternalThreadBlocks; @@ -140,7 +138,7 @@ class FLINT_API SpatialTiledLocalDomainController : public LocalDomainController // -- General mutable std::shared_ptr _provider; std::shared_ptr _spatiallocationinfo; - std::mutex _blockListMutex; + Poco::Mutex _blockListMutex; std::queue _blockIdxList; // list of block to sim in thread/non-threaded processing. std::map, std::vector> _blockCellIdxList; // list of cells to sim, mapped by blockIdx. Used in thread/non-threaded processing diff --git a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h index 7ac9810..7040a66 100644 --- a/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h +++ b/Source/moja.flint/include/moja/flint/threadedaspatiallocaldomaincontroller.h @@ -10,6 +10,8 @@ #include +#include + #include #include @@ -20,7 +22,7 @@ class ThreadedAspatialLocalDomainController; class AspatialLocalDomainThread { public: - AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, std::mutex& tileListMutex, + AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, Poco::Mutex& tileListMutex, std::queue& tileList, const configuration::Configuration* config); @@ -33,7 +35,7 @@ class AspatialLocalDomainThread { ThreadedAspatialLocalDomainController* _parent; configuration::Configuration _threadConfig; int _threadId; - std::mutex& _tileListMutex; + Poco::Mutex& _tileListMutex; std::queue& _tileList; const configuration::Configuration* _config; std::shared_ptr _ldc; @@ -41,11 +43,14 @@ class AspatialLocalDomainThread { class FLINT_API ThreadedAspatialLocalDomainController final : public flint::LocalDomainControllerBase { public: - ThreadedAspatialLocalDomainController(); - ~ThreadedAspatialLocalDomainController() override = default; + ThreadedAspatialLocalDomainController(void); + ~ThreadedAspatialLocalDomainController() = default; + + virtual void configure(const flint::configuration::Configuration& config) override; + virtual void run() override; - status configure(const configuration::Configuration& config) override; - status run() override; + virtual void startup() override {} + virtual void shutdown() override {} std::vector> tasks() const { return _tasks; } @@ -56,7 +61,7 @@ class FLINT_API ThreadedAspatialLocalDomainController final : public flint::Loca std::vector _threads; std::unique_ptr _landscape; - std::mutex _tileListMutex; + Poco::Mutex _tileListMutex; std::queue _tileList; }; diff --git a/Source/moja.flint/include/moja/flint/writesystemconfig.h b/Source/moja.flint/include/moja/flint/writesystemconfig.h index 9fc66f6..ccae118 100644 --- a/Source/moja.flint/include/moja/flint/writesystemconfig.h +++ b/Source/moja.flint/include/moja/flint/writesystemconfig.h @@ -3,7 +3,7 @@ #include -#include +#include namespace moja { namespace flint { @@ -35,7 +35,7 @@ class SpatialLocationInfo; /// class FLINT_API WriteSystemConfig : public flint::ModuleBase { public: - explicit WriteSystemConfig(std::mutex& fileHandlingMutex) + explicit WriteSystemConfig(Poco::Mutex& fileHandlingMutex) : ModuleBase(), _fileHandlingMutex(fileHandlingMutex), notificationType(OnNotificationType::TimingInit), @@ -58,9 +58,25 @@ class FLINT_API WriteSystemConfig : public flint::ModuleBase { void onOutputStep() override; void onError(std::string msg) override; + // --- RAII class for file handle + class FileHandle { + typedef FILE* ptr; + + public: + explicit FileHandle(std::string const& name, std::string const& mode = std::string("r")) + : _wrapped_file(fopen(name.c_str(), mode.c_str())) {} + ~FileHandle() { + if (_wrapped_file) fclose(_wrapped_file); + } + operator ptr() const { return _wrapped_file; } + + private: + ptr _wrapped_file; + }; + private: // Mutexes - std::mutex& _fileHandlingMutex; + Poco::Mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; diff --git a/Source/moja.flint/include/moja/flint/writevariablegrid.h b/Source/moja.flint/include/moja/flint/writevariablegrid.h index 46bc23f..080efc7 100644 --- a/Source/moja.flint/include/moja/flint/writevariablegrid.h +++ b/Source/moja.flint/include/moja/flint/writevariablegrid.h @@ -7,9 +7,10 @@ #include "moja/flint/ipool.h" #include "moja/flint/modulebase.h" +#include + #include #include -#include namespace moja { namespace flint { @@ -18,7 +19,7 @@ class SpatialLocationInfo; class FLINT_API WriteVariableGrid : public ModuleBase { public: - explicit WriteVariableGrid(std::mutex& fileHandlingMutex) + explicit WriteVariableGrid(Poco::Mutex& fileHandlingMutex) : ModuleBase(), _fileHandlingMutex(fileHandlingMutex), _useIndexesForFolderName(false), @@ -40,10 +41,26 @@ class FLINT_API WriteVariableGrid : public ModuleBase { void onOutputStep() override; void onError(std::string msg) override; + // --- RAII class for file handle + class FileHandle { + typedef FILE* ptr; + + public: + explicit FileHandle(std::string const& name, std::string const& mode = std::string("r")) + : _wrapped_file(fopen(name.c_str(), mode.c_str())) {} + ~FileHandle() { + if (_wrapped_file) fclose(_wrapped_file); + } + operator ptr() const { return _wrapped_file; } + + private: + ptr _wrapped_file; + }; + // --- Base classs for data layer class DataSettingsB { public: - DataSettingsB(std::mutex& fileHandlingMutex, int hdrDataType) + DataSettingsB(Poco::Mutex& fileHandlingMutex, int hdrDataType) : notificationType(OnNotificationType::TimingInit), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -104,14 +121,14 @@ class FLINT_API WriteVariableGrid : public ModuleBase { const flint::IVariable* _variable; std::vector _pool; std::string _tileFolderPath; - std::mutex& _fileHandlingMutex; + Poco::Mutex& _fileHandlingMutex; }; // --- Templated version of Base classs for data layer types template class DataSettingsT : public DataSettingsB { public: - DataSettingsT(std::mutex& fileHandlingMutex, int hdrDataType) : DataSettingsB(fileHandlingMutex, hdrDataType){}; + DataSettingsT(Poco::Mutex& fileHandlingMutex, int hdrDataType) : DataSettingsB(fileHandlingMutex, hdrDataType){}; ~DataSettingsT() = default; virtual void configure(std::string& globalOutputPath, bool useIndexesForFolderName, bool forceVariableFolderName, @@ -140,7 +157,7 @@ class FLINT_API WriteVariableGrid : public ModuleBase { private: // Mutexes - std::mutex& _fileHandlingMutex; + Poco::Mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; @@ -158,4 +175,4 @@ class FLINT_API WriteVariableGrid : public ModuleBase { } // namespace flint } // namespace moja -#endif // MOJA_FLINT_WRITESPATIALDATAGRID_H_ +#endif // MOJA_FLINT_WRITESPATIALDATAGRID_H_ \ No newline at end of file diff --git a/Source/moja.flint/src/aggregatorfilewriter.cpp b/Source/moja.flint/src/aggregatorfilewriter.cpp index 94ab343..0e4042a 100644 --- a/Source/moja.flint/src/aggregatorfilewriter.cpp +++ b/Source/moja.flint/src/aggregatorfilewriter.cpp @@ -10,9 +10,9 @@ #include #include #include -#include #include +#include #include #include #include @@ -20,12 +20,11 @@ #include #include +#include using Poco::format; using Poco::NotFoundException; -namespace fs = moja::filesystem; - namespace moja { namespace flint { @@ -67,8 +66,9 @@ void AggregatorFileWriter::onSystemInit() { try { // create file here and append flux data later, this will work in threaded mode as well // file created on onSystemInit, shared Mutex around file access used to append to it. - - if (fs::exists(_fileName)) fs::remove(_fileName); + Poco::File outputFile(_fileName); + if (outputFile.exists()) outputFile.remove(); + outputFile.createFile(); if (_writeFileHeader) { Poco::FileOutputStream streamFile(_fileName); @@ -106,6 +106,20 @@ void AggregatorFileWriter::onSystemInit() { // -------------------------------------------------------------------------------------------- void AggregatorFileWriter::onSystemShutdown() { + try { + // std::vector runStatData; + + // MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":File write start"; // << Filename perhaps + } catch (Poco::AssertionViolationException& exc) { + MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":AssertionViolationException - " << exc.displayText(); + throw; + } catch (const std::exception& e) { + MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":Unknown exception: - " << e.what(); + throw; + } catch (...) { + MOJA_LOG_DEBUG << _simulationUnitData->_localDomainId << ":Unknown Exception"; + throw; + } } // -------------------------------------------------------------------------------------------- @@ -163,7 +177,7 @@ void AggregatorFileWriter::writeFlux() { // -- Write Flux Facts // append to end of file using _fileMutex to keep things safe - std::unique_lock lock(_fileMutex); + Poco::ScopedLockWithUnlock lock(_fileMutex); // append to file here // Poco::OutputLineEndingConverter output(stream, Poco::LineEnding::NEWLINE_CRLF); diff --git a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp index 7dd8b85..f213480 100644 --- a/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/aspatiallocaldomaincontroller.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -28,7 +29,7 @@ using moja::flint::configuration::LocalDomainType; namespace moja { namespace flint { -status AspatialLocalDomainController::configure(const configuration::Configuration& config) { +void AspatialLocalDomainController::configure(const configuration::Configuration& config) { LocalDomainControllerBase::configure(config); // Build landscape. @@ -130,11 +131,9 @@ status AspatialLocalDomainController::configure(const configuration::Configurati _spinupLandUnitController.initialiseData(false); _luId = _landUnitController.getVariable("LandUnitId"); } - return status(status_code::Ok); } -status AspatialLocalDomainController::run(const LandUnitInfo& lu) { - status run_result; +void AspatialLocalDomainController::run(const LandUnitInfo& lu) { try { _luId->set_value(lu.id()); _landUnitController.initialiseData(true); @@ -146,23 +145,24 @@ status AspatialLocalDomainController::run(const LandUnitInfo& lu) { } if (!_simulateLandUnit->value()) { - return run_result; + return; } _notificationCenter.postNotification(moja::signals::PreTimingSequence); if (!_landUnitBuildSuccess->value()) { - return run_result; + return; } - if(!_sequencer->Run(_notificationCenter, _landUnitController)) { - run_result = status(status_code::Error, "Sequencer failed"); - } + _sequencer->Run(_notificationCenter, _landUnitController); + } catch (const Exception& e) { + MOJA_LOG_FATAL << e.displayText(); + } catch (const boost::exception& e) { + MOJA_LOG_FATAL << boost::diagnostic_information(e); } catch (const std::exception& e) { MOJA_LOG_FATAL << e.what(); } _landUnitController.clearAllOperationResults(); - return run_result; } void AspatialLocalDomainController::run(AspatialTileInfo& tile) { @@ -179,52 +179,41 @@ void AspatialLocalDomainController::run(AspatialTileInfo& tile) { MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); } -status AspatialLocalDomainController::startup() { - auto base_status = LocalDomainControllerBase::startup(); - if (base_status.ok() && _runSpinUp) { +void AspatialLocalDomainController::startup() { + LocalDomainControllerBase::startup(); + if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainInit); - return status(status_code::Ok); } - return base_status; } -status AspatialLocalDomainController::shutdown() { - auto base_status = LocalDomainControllerBase::shutdown(); +void AspatialLocalDomainController::shutdown() { + LocalDomainControllerBase::shutdown(); if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainShutdown); - return status(status_code::Ok); } - return base_status; } -status AspatialLocalDomainController::run() { - - status result(status_code::Ok); +void AspatialLocalDomainController::run() { + startup(); auto startTime = DateTime::now(); - if (const auto startup_status = startup(); startup_status.ok()) { - auto total = _landscape->getTotalLUCount(); - auto count = 0; - for (auto& lu : *_landscape) { - count++; - MOJA_LOG_INFO << std::setfill(' ') << std::setw(10) << count << " of " << std::setfill(' ') << std::setw(10) - << total; - if (auto run_status = run(lu); !run_status.ok()) { - MOJA_LOG_ERROR << run_status.message(); - } - } - if (auto shutdown_status = shutdown(); !shutdown_status.ok()) { - MOJA_LOG_ERROR << shutdown_status.message(); - } - } else { - MOJA_LOG_ERROR << startup_status.message(); + + auto total = _landscape->getTotalLUCount(); + auto count = 0; + for (auto lu : *_landscape) { + count++; + MOJA_LOG_INFO << std::setfill(' ') << std::setw(10) << count << " of " << std::setfill(' ') << std::setw(10) + << total; + run(lu); } - + shutdown(); + + //_notificationCenter.postNotification(moja::signals::SystemShutdown); + auto endTime = DateTime::now(); auto ldSpan = endTime - startTime; MOJA_LOG_INFO << "LocalDomain: Start Time : " << startTime; MOJA_LOG_INFO << "LocalDomain: Finish Time : " << endTime; MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); - return result; } } // namespace flint diff --git a/Source/moja.flint/src/compositetransform.cpp b/Source/moja.flint/src/compositetransform.cpp index f73512f..248d441 100644 --- a/Source/moja.flint/src/compositetransform.cpp +++ b/Source/moja.flint/src/compositetransform.cpp @@ -6,6 +6,9 @@ #include +using moja::flint::IncompleteConfigurationException; +using moja::flint::VariableNotFoundException; + namespace moja { namespace flint { @@ -15,17 +18,19 @@ void CompositeTransform::configure(DynamicObject config, const ILandUnitControll _dataRepository = &dataRepository; if (!config.contains("vars")) { - throw std::runtime_error("Error composite transform settings missing vars"); + BOOST_THROW_EXCEPTION(IncompleteConfigurationException() << Item("vars")); } auto varNames = config["vars"]; if (varNames.size() < 1) { - throw std::runtime_error("Error composite transform vars at least one variable name required"); + BOOST_THROW_EXCEPTION(IncompleteConfigurationException() + << Item("vars") << Details("At least one variable name required")); } for (std::string varName : varNames) { if (std::find(_varNames.begin(), _varNames.end(), varName) != _varNames.end()) { - throw std::runtime_error("Error composite transform vars duplicate variable reference"); + BOOST_THROW_EXCEPTION(IncompleteConfigurationException() + << Item("vars") << Details("Duplicate variable reference")); } _varNames.push_back(varName); @@ -51,10 +56,10 @@ void CompositeTransform::controllerChanged(const ILandUnitController& controller const DynamicVar& CompositeTransform::value() const { if (_variables.empty()) { - for (const auto& varName : _varNames) { - const auto* var = _landUnitController->getVariable(varName); + for (auto varName : _varNames) { + auto var = _landUnitController->getVariable(varName); if (var == nullptr) { - throw std::runtime_error("Error composite transform variable not found " + varName); + BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(varName)); } _variables.push_back(var); diff --git a/Source/moja.flint/src/externalvariable.cpp b/Source/moja.flint/src/externalvariable.cpp index a7cb3c7..037c685 100644 --- a/Source/moja.flint/src/externalvariable.cpp +++ b/Source/moja.flint/src/externalvariable.cpp @@ -2,13 +2,18 @@ #include "moja/flint/itransform.h" +#include + +#include + namespace moja { namespace flint { const DynamicVar& ExternalVariable::value() const { return _transform->value(); } void ExternalVariable::set_value(DynamicVar) { - throw std::runtime_error("Can't set External Variable: "+ _info.name); + auto msg = (boost::format("Can't set External Variable: %1%.") % _info.name).str(); + throw ApplicationException(msg); } void ExternalVariable::controllerChanged(const ILandUnitController& controller) const { diff --git a/Source/moja.flint/src/landunitcontroller.cpp b/Source/moja.flint/src/landunitcontroller.cpp index 7f84f3d..1b14b14 100644 --- a/Source/moja.flint/src/landunitcontroller.cpp +++ b/Source/moja.flint/src/landunitcontroller.cpp @@ -7,6 +7,7 @@ #include "moja/flint/ivariable.h" #include "moja/flint/operationmanagersimple.h" #include "moja/flint/operationmanagersimplecache.h" +#include "moja/flint/operationmanagerublas.h" #include #include @@ -36,7 +37,9 @@ void LandUnitController::configure(const configuration::Configuration& config) { auto const operationManagerConfig = config.localDomain()->operationManagerObject(); - if (operationManagerConfig->name() == "Simple") { + if (operationManagerConfig->name() == "Ublas") { + _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); + } else if (operationManagerConfig->name() == "Simple") { _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); } else if (operationManagerConfig->name() == "SimpleCache") { _operationManager = std::make_shared(_timing, operationManagerConfig->settings()); @@ -87,7 +90,7 @@ void LandUnitController::initialisePools() const { _operationManager->initialise void LandUnitController::addVariable(std::string name, std::shared_ptr variable) { if (name.length() == 0 || all(name, boost::algorithm::is_space())) { - throw std::invalid_argument("variable name cannot be empty"); + throw std::invalid_argument("name cannot be empty"); } _variablesMap.insert(std::make_pair(name, variable.get())); _variables.push_back(variable); @@ -97,7 +100,7 @@ IVariable* LandUnitController::getVariable(const std::string& name) { const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { MOJA_LOG_FATAL << "Variable not found: " << name; - throw std::invalid_argument("Error variable not found " + name); + BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); } return v->second; } @@ -106,7 +109,7 @@ const IVariable* LandUnitController::getVariable(const std::string& name) const const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { MOJA_LOG_FATAL << "Variable not found: " << name; - throw std::invalid_argument("Error variable not found " + name); + BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); } return v->second; } diff --git a/Source/moja.flint/src/libraryfactory.cpp b/Source/moja.flint/src/libraryfactory.cpp index bc194b5..88f55d2 100644 --- a/Source/moja.flint/src/libraryfactory.cpp +++ b/Source/moja.flint/src/libraryfactory.cpp @@ -79,9 +79,9 @@ struct FLINTAggregationSharedDataObject { FLINTAggregationSharedDataObject flint_aggregation_shared_data; -std::mutex _fileHandlingMutexVarGridWriter; -std::mutex _fileHandlingMutexConfigWriter; -std::mutex _fileHandlingMutexAggregationFileWriter; +Poco::Mutex _fileHandlingMutexVarGridWriter; +Poco::Mutex _fileHandlingMutexConfigWriter; +Poco::Mutex _fileHandlingMutexAggregationFileWriter; // Flint Data Factory std::shared_ptr createEventQueueFactory(const std::string& eventTypeStr, int id, const std::string& name, diff --git a/Source/moja.flint/src/librarymanager.cpp b/Source/moja.flint/src/librarymanager.cpp index b6a10fb..a4be831 100644 --- a/Source/moja.flint/src/librarymanager.cpp +++ b/Source/moja.flint/src/librarymanager.cpp @@ -7,6 +7,7 @@ #include "moja/flint/modulebase.h" #include +#include #include #include @@ -144,9 +145,7 @@ void LibraryManager::AddLibrary(LibraryType libraryType, const std::string& inLi break; } case LibraryType::Unknown: - default: { - throw std::runtime_error("Unknown library type defined " + inLibraryName); - } + default: { throw LibraryLoadException("Unknown library type defined", inLibraryName); } } // Update hash table @@ -203,7 +202,7 @@ ModuleInterfacePtr LibraryManager::GetModule(const std::string& libraryName, con auto moduleInit = _moduleRegistry.at(moduleKey); auto moduleInterface = moduleInit(); if (!moduleInterface) { - throw std::runtime_error("Failed to initialise the module " + moduleName); + throw LibraryLoadException("Failed to initialise the module", moduleName); } libraryInfo->moduleList.push_back(moduleInterface); @@ -222,7 +221,7 @@ ModuleInterfacePtr LibraryManager::GetModule(const std::string& libraryName, con } #endif catch (...) { - throw std::runtime_error("Module doesn't exist " + moduleName); + throw LibraryLoadException("Module doesn't exist", moduleName); } } @@ -235,13 +234,13 @@ TransformInterfacePtr LibraryManager::GetTransform(const std::string& libraryNam std::shared_ptr transformInterface = transformInit(); if (!transformInterface) { - throw std::runtime_error("Failed to initialise the transform " + transformName); + throw LibraryLoadException("Failed to initialise the transform", transformName); } libraryInfo->transformList.push_back(transformInterface); return transformInterface; } catch (...) { - throw std::runtime_error("Transform doesn't exist " + transformName); + throw LibraryLoadException("Transform doesn't exist", transformName); } } @@ -255,7 +254,7 @@ FlintDataInterfacePtr LibraryManager::GetFlintData(const std::string& libraryNam std::shared_ptr flintDataInterface = flintDataInit(); if (!flintDataInterface) { - throw std::runtime_error("Failed to initialise the flintData " + flintDataName); + throw LibraryLoadException("Failed to initialise the flintData", flintDataName); } flintDataInterface->libraryName = libraryName; flintDataInterface->typeName = flintDataName; @@ -268,7 +267,7 @@ FlintDataInterfacePtr LibraryManager::GetFlintData(const std::string& libraryNam } return flintDataInterface; } catch (...) { - throw std::runtime_error("flintData doesn't exist " + flintDataName); + throw LibraryLoadException("flintData doesn't exist", flintDataName); } } @@ -283,28 +282,28 @@ ModuleInterfacePtr LibraryManager::GetManagedModule(const std::string& managedMo auto moduleKey = ModuleKey(libraryName, managedModuleName); const auto moduleInit = _moduleRegistry[moduleKey]; if (moduleInit == nullptr) { - throw std::runtime_error("Failed to initialise the managed module " + managedModuleName); + throw LibraryLoadException("Failed to initialise the managed module", managedModuleName); } std::shared_ptr libraryInfo = (*libraryInfoIt).second; if (libraryInfo->GetLibraryType() != LibraryType::Managed) { - throw std::runtime_error("Incorrect type for managed module " + managedModuleName); + throw LibraryLoadException("Incorrect type for managed module", managedModuleName); } std::shared_ptr moduleInterface; std::map modulePathMap = FindLibraryPathsExact(managedModuleName); if (modulePathMap.size() != 1) { - throw std::runtime_error("Managed module path issue" + managedModuleName); + throw LibraryLoadException("Managed module path issue", managedModuleName); } try { moduleInterface = moduleInit(); } catch (...) { - throw std::runtime_error("Managed module issue " + managedModuleName); + throw LibraryLoadException("Managed module issue", managedModuleName); } if (!moduleInterface) { - throw std::runtime_error("Failed to initialise the module " + managedModuleName); + throw LibraryLoadException("Failed to initialise the module", managedModuleName); } libraryInfo->moduleList.push_back(moduleInterface); @@ -328,12 +327,12 @@ ProviderInterfacePtr LibraryManager::GetProvider(const std::string& libraryName, auto providerInit = _providerRegistry.at(providerKey); auto providerInterface = providerInit(settings); if (!providerInterface) { - throw std::runtime_error("Failed to initialise the provider " + providerName); + throw LibraryLoadException("Failed to initialise the provider", providerName); } libraryInfo->providerList.push_back(providerInterface); return providerInterface; } catch (...) { - throw std::runtime_error("Provider doesn't exist " + providerName); + throw LibraryLoadException("Provider doesn't exist", providerName); } } @@ -435,15 +434,15 @@ bool LibraryManager::LoadInternalLibrary(std::shared_ptr li AddLibrary(LibraryType::Internal, libraryHandles->libraryName); auto libraryInfo = _libraries[libraryHandles->libraryName]; if (libraryInfo == nullptr) { - throw std::runtime_error("Library not found " + libraryHandles->libraryName); + throw LibraryLoadException("Library not found", libraryHandles->libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { MOJA_LOG_DEBUG << (boost::format("LibraryMLoadInternalLibraryanager: %s") % "calling registrations").str(); if (libraryInfo->GetLibraryType() != LibraryType::Internal) { - throw std::runtime_error("Attempt to load library already loaded as different type" + - libraryHandles->libraryName); + throw LibraryLoadException("Attempt to load library already loaded as different type", + libraryHandles->libraryName); } auto internalLibraryInfo = std::static_pointer_cast(libraryInfo); @@ -467,12 +466,12 @@ bool LibraryManager::LoadManagedLibrary() { // Grab the library info. This has the file name of the library, as well as other info. auto libraryInfo = _libraries[libraryName]; if (libraryInfo == nullptr) { - throw std::runtime_error("Library not found " + libraryName); + throw LibraryLoadException("Library not found", libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { if (libraryInfo->GetLibraryType() != LibraryType::Managed) { - throw std::runtime_error("Attempt to load library already loaded as different type " + libraryName); + throw LibraryLoadException("Attempt to load library already loaded as different type", libraryName); } auto managedLibraryInfo = std::static_pointer_cast(_libraries[libraryName]); @@ -485,11 +484,11 @@ bool LibraryManager::LoadManagedLibrary() { // Skip this check if file manager has not yet been initialized if (!libraryFileToLoad.exists()) { - throw std::runtime_error("Library not found " + managedLibraryInfo->filename); + throw LibraryLoadException("Library not found", managedLibraryInfo->filename); } if (!CheckModuleCompatibility(libraryFileToLoad.path())) { - throw std::runtime_error("Library not compatible " + libraryFileToLoad.path()); + throw LibraryLoadException("Library not compatible", libraryFileToLoad.path()); } managedLibraryInfo->handle = std::make_unique(libraryFileToLoad.path()); @@ -538,12 +537,12 @@ bool LibraryManager::LoadExternalLibrary(const std::string& libraryName, const s // Grab the library info. This has the file name of the library, as well as other info. auto libraryInfo = _libraries[libraryName]; if (libraryInfo == nullptr) { - throw std::runtime_error("Library not found " + libraryName); + throw LibraryLoadException("Library not found", libraryName); } if (libraryInfo->libraryHandles->getModuleRegistrations == nullptr) { if (libraryInfo->GetLibraryType() != LibraryType::External) { - throw std::runtime_error("Attempt to load library already loaded as different type " + libraryName); + throw LibraryLoadException("Attempt to load library already loaded as different type", libraryName); } auto externalLibraryInfo = std::static_pointer_cast(_libraries[libraryName]); @@ -555,11 +554,11 @@ bool LibraryManager::LoadExternalLibrary(const std::string& libraryName, const s externalLibraryInfo->handle = nullptr; if (!libraryFileToLoad.exists()) { - throw std::runtime_error("Library not found " + externalLibraryInfo->filename); + throw LibraryLoadException("Library not found", externalLibraryInfo->filename); } if (!CheckModuleCompatibility(libraryFileToLoad.path())) { - throw std::runtime_error("Library not compatible " + libraryFileToLoad.path()); + throw LibraryLoadException("Library not compatible", libraryFileToLoad.path()); } externalLibraryInfo->handle = std::make_unique(libraryFileToLoad.path()); diff --git a/Source/moja.flint/src/localdomaincontrollerbase.cpp b/Source/moja.flint/src/localdomaincontrollerbase.cpp index aa85c80..8c000fe 100644 --- a/Source/moja.flint/src/localdomaincontrollerbase.cpp +++ b/Source/moja.flint/src/localdomaincontrollerbase.cpp @@ -23,26 +23,28 @@ #include #include +using moja::flint::SequencerNotFoundException; + namespace moja { namespace flint { // -------------------------------------------------------------------------------------------- LocalDomainControllerBase::LocalDomainControllerBase() - : _localDomainId(0), _config(nullptr) { + : _localDomainId(0), _config(nullptr), _landUnitController(), _dataRepository(), _libraryManager() { _dataRepository.setProviderRegistry(_libraryManager.getProviderRegistry()); } // -------------------------------------------------------------------------------------------- LocalDomainControllerBase::LocalDomainControllerBase(std::shared_ptr libraryHandles) - : _localDomainId(0), _config(nullptr), _libraryManager(libraryHandles) { + : _localDomainId(0), _config(nullptr), _landUnitController(), _dataRepository(), _libraryManager(libraryHandles) { _dataRepository.setProviderRegistry(_libraryManager.getProviderRegistry()); } // -------------------------------------------------------------------------------------------- -status LocalDomainControllerBase::configure(const configuration::Configuration& config) { +void LocalDomainControllerBase::configure(const configuration::Configuration& config) { // Keep pointer to config _config = &config; @@ -56,7 +58,7 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& _landUnitController.configure(config); // Load the configured Libraries - for (const auto* lib : config.libraries()) { + for (auto lib : config.libraries()) { switch (lib->type()) { case configuration::LibraryType::Internal: // Not required, all internals modules are loaded automatically @@ -90,15 +92,17 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& // Load the configured modules (and handle Proxies) _moduleMap.clear(); - for (const auto* module : config.modules()) { + for (const auto& module : config.modules()) { ModuleMapKey key(module->libraryName(), module->name()); if (_moduleMap.find(key) != _moduleMap.end()) { - return status(status_code::Error, "Error duplicate modules " + module->libraryName() + " " + module->name()); + // duplicate Key found + throw DuplicateModuleDefinedException() << Details("Error duplicate module defined") + << LibraryName(module->libraryName()) << SequencerName(module->name()); } _moduleMap[key] = _libraryManager.GetModule(module->libraryName(), module->name()); _moduleMap[key]->setLandUnitController(_landUnitController); if (module->isProxy()) { - auto proxy = dynamic_cast(modules()[key]); + auto proxy = static_cast(modules()[key]); auto proxyModules = module->settings()["proxyModules"]; for (auto& item : proxyModules) { DynamicObject proxyModule = item.extract(); @@ -124,8 +128,9 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& ModuleMapKey sequencerKey(config.localDomain()->sequencerLibrary(), config.localDomain()->sequencer()); _sequencer = std::dynamic_pointer_cast(_moduleMap[sequencerKey]); if (_sequencer == nullptr) { - return status(status_code::Error, "Error finding sequencer " + config.localDomain()->sequencerLibrary() + " " + - config.localDomain()->sequencer()); + throw SequencerNotFoundException() << Details("Error finding sequencer") + << LibraryName(config.localDomain()->sequencerLibrary()) + << SequencerName(config.localDomain()->sequencer()); } _sequencer->configure(timing); @@ -141,7 +146,73 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& std::map, TransformInterfacePtr> transforms; std::map, FlintDataInterfacePtr> flintData; - +#if 0 + for (const auto var : config.variables2()) { + if (var->variableType() == configuration::VariableType::Internal) { + auto variable = static_cast(var); + _landUnitController.addVariable(variable->name(), std::make_shared(variable->value(), VariableInfo{ variable->name() })); + } + else if (var->variableType() == configuration::VariableType::External) { + auto variable = static_cast(var); + + const auto& transformConfig = variable->transform(); + const auto libraryName = transformConfig.libraryName(); + const auto variableName = variable->name(); + auto key = std::make_pair(libraryName, variableName); + transforms[key] = _libraryManager.GetTransform(libraryName, transformConfig.typeName()); + _landUnitController.addVariable(variable->name(), std::make_shared(transforms[key], VariableInfo{ variable->name() })); + } + else if (var->variableType() == configuration::VariableType::FlintData) { + auto variable = static_cast(var); + + const auto& flintDataConfig = variable->flintdata(); + const auto libraryName = flintDataConfig.libraryName(); + const auto variableName = variable->name(); + auto key = std::make_pair(libraryName, variableName); + flintData[key] = _libraryManager.GetFlintData(libraryName, flintDataConfig.typeName()); + _landUnitController.addVariable(variable->name(), std::make_shared(flintData[key], libraryName, flintDataConfig.typeName(), VariableInfo{ variable->name() })); + } + } + + //// Now go back over and check for nested things + //for (auto var : config.variables2()) { + // if (var->variableType() == configuration::VariableType::Internal) { + // auto variable = static_cast(var); + // auto value = variable->value(); + // variable->set_value(checkForNestedVariables(variable->name(), value)); + // } + //} + + // New version of Variables + // Now Configure external, internal and flintdata + + // config external (transforms) + for (const auto var : config.variables2()) { + if (var->variableType() == configuration::VariableType::External) { + auto variable = static_cast(var); + + const auto& transformConfig = variable->transform(); + const auto libraryName = transformConfig.libraryName(); + const auto variableName = variable->name(); + auto key = std::make_pair(libraryName, variableName); + transforms[key]->configure(transformConfig.settings(), _landUnitController, _dataRepository); + } + } + + // config flintdata + for (const auto var : config.variables2()) { + if (var->variableType() == configuration::VariableType::FlintData) { + auto variable = static_cast(var); + + const auto& flintDataConfig = variable->flintdata(); + const auto libraryName = flintDataConfig.libraryName(); + const auto variableName = variable->name(); + auto key = std::make_pair(libraryName, variableName); + flintData[key]->configure(flintDataConfig.settings(), _landUnitController, _dataRepository); + } + } + +#else // Configure Variables for (const auto variable : config.variables()) { auto value = variable->value(); @@ -218,6 +289,8 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& transforms[key]->configure(transformConfig.settings(), _landUnitController, _dataRepository); } +#endif + // Configure the simulateLandUnit & landUnitBuildSuccess variables here _simulateLandUnit = _landUnitController.getVariable(config.localDomain()->simulateLandUnit()); _landUnitBuildSuccess = _landUnitController.getVariable(config.localDomain()->landUnitBuildSuccess()); @@ -226,39 +299,31 @@ status LocalDomainControllerBase::configure(const configuration::Configuration& // variables (some special handling for vectors & structs _landUnitController.initialiseData(false); /// TODO: check if this is required here. initialiseData is also called /// in SpatialTiledLocalDomainController::runCell - return status(status_code::Ok); } // -------------------------------------------------------------------------------------------- -status LocalDomainControllerBase::run() { +void LocalDomainControllerBase::run() { _notificationCenter.postNotification(moja::signals::PreTimingSequence); _notificationCenter.postNotification(signals::LocalDomainProcessingUnitInit); - const bool sequencer_result = _sequencer->Run(_notificationCenter, _landUnitController); + _sequencer->Run(_notificationCenter, _landUnitController); _notificationCenter.postNotification(signals::LocalDomainProcessingUnitShutdown); - return sequencer_result ? status(status_code::Ok) : status(status_code::Error, "Sequencer failed"); } // -------------------------------------------------------------------------------------------- -status LocalDomainControllerBase::startup() { - _notificationCenter.postNotification(moja::signals::LocalDomainInit); - return status(status_code::Ok); -} +void LocalDomainControllerBase::startup() { _notificationCenter.postNotification(moja::signals::LocalDomainInit); } // -------------------------------------------------------------------------------------------- -status LocalDomainControllerBase::shutdown() { - _notificationCenter.postNotification(moja::signals::LocalDomainShutdown); - return status(status_code::Ok); -} +void LocalDomainControllerBase::shutdown() { _notificationCenter.postNotification(moja::signals::LocalDomainShutdown); } // -------------------------------------------------------------------------------------------- std::map LocalDomainControllerBase::modules() const { std::map results; - for (auto& [key, module_ptr] : _moduleMap) { - results.emplace(key, module_ptr.get()); + for (auto module : _moduleMap) { + results.insert(std::pair(module.first, module.second.get())); } return results; } diff --git a/Source/moja.flint/src/lookuprandomtransform.cpp b/Source/moja.flint/src/lookuprandomtransform.cpp index 4506699..6415e36 100644 --- a/Source/moja.flint/src/lookuprandomtransform.cpp +++ b/Source/moja.flint/src/lookuprandomtransform.cpp @@ -10,6 +10,9 @@ #include +using moja::flint::IncompleteConfigurationException; +using moja::flint::VariableNotFoundException; + namespace moja { namespace flint { @@ -25,7 +28,7 @@ void LookupRandomTransform::configure(DynamicObject config, const ILandUnitContr } } if (!validConfiguration) { - throw std::runtime_error("Expected configuration item not found " + std::string(key)); + throw IncompleteConfigurationException() << Item(key) << Details("Expected configuration item not found"); } } //_fromVarName = config["from"].convert(); @@ -44,9 +47,14 @@ void LookupRandomTransform::controllerChanged(const ILandUnitController& control } const DynamicVar& LookupRandomTransform::value() const { - const auto* to = _landUnitController->getVariable(_toVarName); + // auto from = _landUnitController->getVariable(_fromVarName); + // if (from == nullptr) { + // throw VariableNotFoundException() << VariableName(_fromVarName); + //} + + auto to = _landUnitController->getVariable(_toVarName); if (to == nullptr) { - throw std::runtime_error("Variable not found " + _toVarName); + throw VariableNotFoundException() << VariableName(_toVarName); } // auto& fromValue = from->value(); diff --git a/Source/moja.flint/src/lookuptransform.cpp b/Source/moja.flint/src/lookuptransform.cpp index c0ea5c8..db06ce7 100644 --- a/Source/moja.flint/src/lookuptransform.cpp +++ b/Source/moja.flint/src/lookuptransform.cpp @@ -7,6 +7,9 @@ #include #include +using moja::flint::IncompleteConfigurationException; +using moja::flint::VariableNotFoundException; + namespace moja { namespace flint { @@ -22,7 +25,7 @@ void LookupTransform::configure(DynamicObject config, const ILandUnitController& } if (!validConfiguration) { - throw std::runtime_error("Expected configuration item not found " + std::string(key)); + throw IncompleteConfigurationException() << Item(key) << Details("Expected configuration item not found"); } } @@ -38,18 +41,18 @@ void LookupTransform::controllerChanged(const ILandUnitController& controller) { }; const DynamicVar& LookupTransform::value() const { - const auto* from = _landUnitController->getVariable(_fromVarName); + auto from = _landUnitController->getVariable(_fromVarName); if (from == nullptr) { - throw std::runtime_error("From variable not found " + _fromVarName); + throw VariableNotFoundException() << VariableName(_fromVarName); } - const auto* to = _landUnitController->getVariable(_toVarName); + auto to = _landUnitController->getVariable(_toVarName); if (to == nullptr) { - throw std::runtime_error("To variable not found " + _fromVarName); + throw VariableNotFoundException() << VariableName(_toVarName); } - const auto& fromValue = from->value(); - const auto& toValue = to->value(); + auto& fromValue = from->value(); + auto& toValue = to->value(); if (fromValue.isEmpty() || toValue.isEmpty()) { setCachedValue(DynamicVar()); return _cachedValue; diff --git a/Source/moja.flint/src/operationmanagersimple.cpp b/Source/moja.flint/src/operationmanagersimple.cpp index ffc38cb..306dde7 100644 --- a/Source/moja.flint/src/operationmanagersimple.cpp +++ b/Source/moja.flint/src/operationmanagersimple.cpp @@ -9,15 +9,18 @@ #include "moja/flint/operationstocksimple.h" #include "moja/flint/poolsimple.h" +#include #include #include #include -#include +#include namespace moja { namespace flint { +#define OPERATION_MANAGER_CACHE_SIZE 100 + // -------------------------------------------------------------------------------------------- OperationManagerSimple::OperationManagerSimple(Timing& timing, const DynamicObject& config) : _timing(timing) { @@ -115,11 +118,15 @@ void OperationManagerSimple::applyOperations() { if (!_allowNegativeTransfers && FloatCmp::lessThan(val, 0.0)) { auto amount = flux.transferType() == OperationTransferType::Proportional ? val * 100.0 : val; - throw simulation_error( - fmt::format("Negative transfer by {}: {} {} {} to {}", flux.metaData()->moduleName, amount, - (flux.transferType() == OperationTransferType::Proportional ? "% of" : "from"), - getPool(srcIx)->name(), getPool(dstIx)->name()), - "moja.flint", "OperationManagerSimple", 0); + BOOST_THROW_EXCEPTION(SimulationError() + << Details((boost::format("Negative transfer by %1%: %2% %3% %4% to %5%") % + flux.metaData()->moduleName % amount % + (flux.transferType() == OperationTransferType::Proportional ? "% of" + : "from") % + getPool(srcIx)->name() % getPool(dstIx)->name()) + .str()) + << LibraryName("moja.flint") << ModuleName("OperationManagerSimple") + << ErrorCode(0)); } if (_warnNegativeTransfers && FloatCmp::lessThan(val, 0.0)) { @@ -222,7 +229,7 @@ const IPool* OperationManagerSimple::addPool(const std::string& name, const std: } if (_poolValues.size() == _poolValues.capacity()) { - throw std::runtime_error( + throw ApplicationException( "maximum pool definitions exceeded. Only 255 pools allowed"); // to protect the references held by PoolSimple // wrappers } @@ -253,17 +260,17 @@ const IPool* OperationManagerSimple::addPool(PoolMetaData& metadata, double init const IPool* OperationManagerSimple::getPool(const std::string& name) const { try { - auto& r = _poolNameObjectMap.at(name); + auto r = _poolNameObjectMap.at(name); return r.get(); } catch (...) { - throw std::invalid_argument("Error pool not found " + name); + throw PoolNotFoundException() << PoolName(name); } } // -------------------------------------------------------------------------------------------- const IPool* OperationManagerSimple::getPool(int index) const { - if (index >= _poolObjects.size() || index < 0) throw std::invalid_argument("Error in get pool index out of range"); + if (index >= _poolObjects.size() || index < 0) throw PoolNotFoundException() << PoolName("Bad index"); auto& r = _poolObjects[index]; return r.get(); } diff --git a/Source/moja.flint/src/operationmanagersimplecache.cpp b/Source/moja.flint/src/operationmanagersimplecache.cpp index 42ae1e1..8a5dca5 100644 --- a/Source/moja.flint/src/operationmanagersimplecache.cpp +++ b/Source/moja.flint/src/operationmanagersimplecache.cpp @@ -219,15 +219,15 @@ const IPool* OperationManagerSimpleCache::addPool(PoolMetaData& metadata, double const IPool* OperationManagerSimpleCache::getPool(const std::string& name) const { try { - auto& r = _poolNameObjectMap.at(name); + auto r = _poolNameObjectMap.at(name); return r.get(); } catch (...) { - throw std::invalid_argument("Error pool not found " + name); + throw PoolNotFoundException() << PoolName(name); } } const IPool* OperationManagerSimpleCache::getPool(int index) const { - if (index >= _poolObjects.size() || index < 0) throw std::invalid_argument("Error in get pool index out of range"); + if (index >= _poolObjects.size() || index < 0) throw PoolNotFoundException() << PoolName("Bad index"); auto& r = _poolObjects[index]; return r.get(); } diff --git a/Source/moja.flint/src/outputerstream.cpp b/Source/moja.flint/src/outputerstream.cpp index 3d4218e..e4c5215 100644 --- a/Source/moja.flint/src/outputerstream.cpp +++ b/Source/moja.flint/src/outputerstream.cpp @@ -8,7 +8,8 @@ #include #include -#include + +#include #include @@ -20,8 +21,6 @@ #define DL_CHR "," #define STOCK_PRECISION 15 -namespace fs = moja::filesystem; - namespace moja { namespace flint { @@ -143,16 +142,12 @@ void OutputerStream::outputHeader(std::ostream& stream) const { // -------------------------------------------------------------------------------------------- -void OutputerStream::outputOnNotification(const std::string& notification, std::ostream& stream) { +void OutputerStream::outputInit(std::ostream& stream) { const auto& timingL = *_landUnitData->timing(); - stream << notification << DL_CHR << - timingL.step() << DL_CHR << - // timingL.curEndDate().addMicroseconds(-1) << DL_CHR << - timingL.curStartDate() << DL_CHR << - timingL.fractionOfStep() << DL_CHR << - timingL.stepLengthInYears() << DL_CHR << - std::setprecision(STOCK_PRECISION); + stream << "onTimingPostInit" << DL_CHR << timingL.step() << DL_CHR << timingL.curEndDate().addMicroseconds(-1) + << DL_CHR << timingL.fractionOfStep() << DL_CHR << timingL.stepLengthInYears() << DL_CHR; + stream << std::setprecision(STOCK_PRECISION); auto pools = _landUnitData->poolCollection(); for (auto& it : pools) { @@ -190,6 +185,47 @@ void OutputerStream::outputOnNotification(const std::string& notification, std:: // -------------------------------------------------------------------------------------------- +void OutputerStream::outputEndStep(const std::string& notification, std::ostream& stream) { + const auto& timingL = *_landUnitData->timing(); + stream << notification << DL_CHR << timingL.step() << DL_CHR << timingL.curEndDate().addMicroseconds(-1) << DL_CHR + << timingL.fractionOfStep() << DL_CHR << timingL.stepLengthInYears() << DL_CHR; + stream << std::setprecision(STOCK_PRECISION); + auto pools = _landUnitData->poolCollection(); + for (auto& it : pools) { + stream << it->value() << DL_CHR; + } + for (auto var : _variables) { + auto varPtr = std::get<3>(var); + auto varName = std::get<1>(var); + auto varProp = std::get<2>(var); + + if (varPtr == nullptr) { + stream << "(missing variable)"; + } else if (varProp != "") { + auto varValue = varPtr->value(); + if (varValue.type() == typeid(std::shared_ptr)) { + auto flintDataVariable = varPtr->value().extract>(); + auto propValue = flintDataVariable->getProperty(varProp); + outputDynamicToStream(stream, propValue); + } else { + if (varValue.isStruct()) { + auto varStruct = varValue.extract(); + auto varStructProp = varStruct.contains(varProp) ? varStruct[varProp] : DynamicVar(); + outputDynamicToStream(stream, varStructProp); + } else + outputDynamicToStream(stream, varValue); + } + } else { + // TODO: extend this to do property of array/struct objects + outputDynamicToStream(stream, varPtr->value()); + } + stream << DL_CHR; + } + stream << std::endl; +} + +// -------------------------------------------------------------------------------------------- + void OutputerStream::outputShutdown(std::ostream& stream) { using namespace std::chrono; @@ -204,9 +240,10 @@ void OutputerStream::outputShutdown(std::ostream& stream) { // -------------------------------------------------------------------------------------------- void OutputerStream::onSystemInit() { - fs::path outputFile(_fileName); - if (fs::exists(_fileName)) fs::remove(_fileName); - + Poco::File outputFile(_fileName); + if (outputFile.exists()) outputFile.remove(); + outputFile.createFile(); + _streamFile.open(_fileName, std::ios::out); _output.addStream(_streamFile); if (_outputToScreen) _output.addStream(std::cout); @@ -231,35 +268,28 @@ void OutputerStream::onLocalDomainInit() { // -------------------------------------------------------------------------------------------- -void OutputerStream::onTimingPostInit() { - outputOnNotification("onTimingPostInit", _output); -} +void OutputerStream::onTimingPostInit() { outputInit(_output); } // -------------------------------------------------------------------------------------------- void OutputerStream::onOutputStep() { if (_outputOnOutputStep) { const auto& timingL = *_landUnitData->timing(); - if (!_outputAnnually || timingL.curStartDate().month() == 12) { - outputOnNotification("onOutputStep", _output); - } + + if (!_outputAnnually || timingL.curStartDate().month() == 12) outputEndStep("onOutputStep", _output); } } // -------------------------------------------------------------------------------------------- void OutputerStream::onTimingEndStep() { - if (_outputOnTimingEndStep) { - outputOnNotification("onTimingEndStep", _output); - } + if (_outputOnTimingEndStep) outputEndStep("onTimingEndStep", _output); } // -------------------------------------------------------------------------------------------- void OutputerStream::onPostDisturbanceEvent() { - if (_outputOnPostDisturbanceEvent) { - outputOnNotification("onPostDisturbanceEvent", _output); - } + if (_outputOnPostDisturbanceEvent) outputEndStep("onPostDisturbanceEvent", _output); } } // namespace flint diff --git a/Source/moja.flint/src/outputerstreamflux.cpp b/Source/moja.flint/src/outputerstreamflux.cpp index 4988af2..9fdd81f 100644 --- a/Source/moja.flint/src/outputerstreamflux.cpp +++ b/Source/moja.flint/src/outputerstreamflux.cpp @@ -5,12 +5,11 @@ #include "moja/flint/ipool.h" #include "moja/flint/itiming.h" -// #include -#include #include #include #include -#include + +#include #include // std::setprecision #include @@ -19,8 +18,6 @@ #define DL_CHR "," #define STOCK_PRECISION 15 -namespace fs = moja::filesystem; - namespace moja { namespace flint { @@ -53,19 +50,38 @@ void OutputerStreamFlux::outputHeader(std::ostream& stream) const { stream << "Started:" << start << std::endl; stream << "==========================================================================" << std::endl; } - stream << - "step" << DL_CHR << - "step_date" << DL_CHR << - "module_name" << DL_CHR << - "disturbance_type" << DL_CHR << - "source_pool" << DL_CHR << - "sink_pool" << DL_CHR << - "value" << std::endl; + stream << "step" << DL_CHR << "step date" << DL_CHR << "module name" << DL_CHR << "disturbance_type" << DL_CHR + << "source pool" DL_CHR << "sink pool" << DL_CHR << "value" << std::endl; +} + +// -------------------------------------------------------------------------------------------- + +void OutputerStreamFlux::outputInit(std::ostream& stream) const { + const auto timingL = _landUnitData->timing(); + + for (auto operationResult : _landUnitData->getOperationLastAppliedIterator()) { + const auto mdata = operationResult->metaData(); + for (auto f : operationResult->operationResultFluxCollection()) { + const auto srcIx = f->source(); + const auto dstIx = f->sink(); + if (srcIx == dstIx) // don't process diagonal + continue; + + const auto val = f->value(); + const auto srcPool = _landUnitData->getPool(srcIx); + const auto dstPool = _landUnitData->getPool(dstIx); + + stream << timingL->step() << DL_CHR << timingL->curStartDate() << DL_CHR; + stream << mdata->moduleName << DL_CHR; + stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val + << std::endl; + } + } } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::outputOnNotification(const std::string& notification, std::ostream& stream) const { +void OutputerStreamFlux::outputEndStep(std::ostream& stream) const { const auto& timingL = _landUnitData->timing(); for (auto operationResult : _landUnitData->getOperationLastAppliedIterator()) { const auto mdata = operationResult->metaData(); @@ -79,19 +95,10 @@ void OutputerStreamFlux::outputOnNotification(const std::string& notification, s const auto srcPool = _landUnitData->getPool(srcIx); const auto dstPool = _landUnitData->getPool(dstIx); - auto fluxTypeInfoRecordId = flint::FluxType::Unclassified; - if (operationResult->hasDataPackage()) { - auto dataPacket = operationResult->dataPackage().extract>(); - fluxTypeInfoRecordId = dataPacket->_fluxType; - } - - stream << timingL->step() << DL_CHR << - timingL->curStartDate() << DL_CHR << - mdata->moduleName << DL_CHR << - int(fluxTypeInfoRecordId) << DL_CHR << - srcPool->name() << DL_CHR << - dstPool->name() << DL_CHR << - std::setprecision(STOCK_PRECISION) << val << std::endl; + stream << timingL->step() << DL_CHR << timingL->curStartDate() << DL_CHR; + stream << mdata->moduleName << DL_CHR; + stream << srcPool->name() << DL_CHR << dstPool->name() << DL_CHR << std::setprecision(STOCK_PRECISION) << val + << std::endl; } } } @@ -110,7 +117,10 @@ void OutputerStreamFlux::outputShutdown(std::ostream& stream) const { // -------------------------------------------------------------------------------------------- void OutputerStreamFlux::onSystemInit() { - if (fs::exists(_fileName)) fs::remove(_fileName); + Poco::File outputFile(_fileName); + if (outputFile.exists()) outputFile.remove(); + outputFile.createFile(); + _streamFile.open(_fileName, std::ios::out); _output.addStream(_streamFile); if (_outputToScreen) _output.addStream(std::cout); @@ -126,15 +136,15 @@ void OutputerStreamFlux::onSystemShutdown() { // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onTimingPostInit() { outputOnNotification("onTimingPostInit", _output); } +void OutputerStreamFlux::onTimingPostInit() { outputInit(_output); } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onTimingEndStep() { outputOnNotification("onTimingEndStep", _output); } +void OutputerStreamFlux::onTimingEndStep() { outputEndStep(_output); } // -------------------------------------------------------------------------------------------- -void OutputerStreamFlux::onPostDisturbanceEvent() { outputOnNotification("onPostDisturbanceEvent", _output); } +void OutputerStreamFlux::onPostDisturbanceEvent() { outputEndStep(_output); } } // namespace flint } // namespace moja diff --git a/Source/moja.flint/src/poolsimplecache.cpp b/Source/moja.flint/src/poolsimplecache.cpp index 0d2906d..aac2317 100644 --- a/Source/moja.flint/src/poolsimplecache.cpp +++ b/Source/moja.flint/src/poolsimplecache.cpp @@ -36,7 +36,7 @@ int PoolSimpleCache::idx() const { return _idx; } double PoolSimpleCache::value() const { if (!_initialised) { - throw std::runtime_error("Error Pool Simple Cache has not been initialized"); + throw PreconditionViolatedException() << Precondition("PoolSimpleCache has not been initialised"); } return (*_pools)[_idx]; } diff --git a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp index e8e0e16..6ac1ca7 100644 --- a/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp +++ b/Source/moja.flint/src/spatialtiledlocaldomaincontroller.cpp @@ -20,9 +20,11 @@ #include #include +#include #include #include +#include #include #include @@ -172,11 +174,63 @@ void StatsUnitRecord::mojaLog(const std::string& levelStr, int localDomainId, da << "LU/second: " << std::setfill(' ') << std::setw(8) << luPerSecond << "]"; } +#define RETRY_ATTEMPTS 10000 +#define RETRY_SLEEP std::chrono::milliseconds(200) + +// void StatsUnitRecord::sqliteCreateTable(bool dropExisting, int localDomainId, std::string databaseName, std::string +// tableName) { auto retry = false; auto maxRetries = RETRY_ATTEMPTS; do { try { retry = false; +// SQLite::Connector::registerConnector(); +// Session session("SQLite", databaseName); +// +// if (dropExisting) +// session << "DROP TABLE IF EXISTS " << tableName, now; +// session << "CREATE TABLE IF NOT EXISTS " << tableName << " (id UNSIGNED BIG INT NOT NULL, runId +//VARCHAR(255), elapsedTimeTotal UNSIGNED BIG INT NOT NULL, elapsedTimeFramework UNSIGNED BIG INT NOT NULL, +//elapsedTimeSpinup UNSIGNED BIG INT NOT NULL, elapsedTimeProcessed UNSIGNED BIG INT NOT NULL, unitsTotal UNSIGNED BIG +//INT NOT NULL, unitsProcessed UNSIGNED BIG INT NOT NULL, unitsNotProcessed UNSIGNED BIG INT NOT NULL, unitsWithErrors +//UNSIGNED BIG INT NOT NULL, tileIdx INTEGER, blockIdx INTEGER)", now; +// +// SQLite::Connector::unregisterConnector(); +// } +// catch (SQLite::DBLockedException&) { +// MOJA_LOG_DEBUG << localDomainId << ":DBLockedException - " << maxRetries << " retries +//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << +//localDomainId << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; +// } +// } +// } while (retry); +//} +// +// void StatsUnitRecord::sqliteWrite(std::shared_ptr>& collection, int +// localDomainId, std::string databaseName, std::string tableName) { auto retry = false; auto maxRetries = +//RETRY_ATTEMPTS; do { try { retry = false; SQLite::Connector::registerConnector(); Session session("SQLite", +//databaseName); +// +// // -- collection +// if (collection->size() != 0) { +// MOJA_LOG_DEBUG << localDomainId << ":SQLite " << tableName << " - inserted " << +//collection->size() << " records"; session.begin(); session << "INSERT INTO " << tableName << " VALUES(?, ?, ?, ?, ?, +//?, ?, ?, ?, ?, ?, ?)", bind(collection->getPersistableCollection()), now; session.commit(); +// } +// +// SQLite::Connector::unregisterConnector(); +// } +// catch (SQLite::DBLockedException&) { +// MOJA_LOG_DEBUG << localDomainId << ":DBLockedException - " << maxRetries << " retries +//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << +//localDomainId << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; +// } +// } +// } while (retry); +//} + +// -- + // -------------------------------------------------------------------------------------------- class SpatialTiledLocalDomainController::InternalThreadBlocks { public: - InternalThreadBlocks(int threadId, std::string runId, std::mutex& blockIdxListMutex, + InternalThreadBlocks(int threadId, std::string runId, Poco::Mutex& blockIdxListMutex, std::queue& blockIdxList, int blockIdxListSize, std::map, std::vector>& blockCellIdxList, const configuration::Configuration* config, @@ -194,7 +248,7 @@ class SpatialTiledLocalDomainController::InternalThreadBlocks { private: int _threadId; std::string _runId; - std::mutex& _blockIdxListMutex; + Poco::Mutex& _blockIdxListMutex; std::queue& _blockIdxList; std::map, std::vector>& _blockCellIdxList; const configuration::Configuration* _config; @@ -205,7 +259,7 @@ class SpatialTiledLocalDomainController::InternalThreadBlocks { // -------------------------------------------------------------------------------------------- SpatialTiledLocalDomainController::InternalThreadBlocks::InternalThreadBlocks( - int threadId, std::string runId, std::mutex& blockIdxListMutex, std::queue& blockIdxList, + int threadId, std::string runId, Poco::Mutex& blockIdxListMutex, std::queue& blockIdxList, int blockIdxListSize, std::map, std::vector>& blockCellIdxList, const configuration::Configuration* config, std::shared_ptr>& globalStatsDimension, @@ -259,7 +313,7 @@ void SpatialTiledLocalDomainController::InternalThreadBlocks::operator()() { while (keepRunning) { // Pop a block index from the queue. // ToDo: could make the number of blocks to pop here configurable - std::unique_lock lock(_blockIdxListMutex); + Poco::ScopedLockWithUnlock lock(_blockIdxListMutex); if (_blockIdxList.size() > 0) { _ldc->_blockIdxListPosition = int(_blockIdxList.size()); auto blockIdx = _blockIdxList.front(); @@ -385,7 +439,7 @@ SpatialTiledLocalDomainController::SpatialTiledLocalDomainController( // -------------------------------------------------------------------------------------------- -status SpatialTiledLocalDomainController::configure(const configuration::Configuration& config) { +void SpatialTiledLocalDomainController::configure(const configuration::Configuration& config) { LocalDomainControllerBase::configure(config); _doLogging = config.localDomain()->doLogging(); _numThreads = config.localDomain()->numThreads(); @@ -611,7 +665,6 @@ status SpatialTiledLocalDomainController::configure(const configuration::Configu _tasks.push_back(task); } } - return status(status_code::Ok); } // -------------------------------------------------------------------------------------------- @@ -738,21 +791,50 @@ bool SpatialTiledLocalDomainController::runCellSpinUp(std::shared_ptr_stopWatchProcessed.stop(); blockStatsUnit->_unitsProcessed++; return true; - } catch (const flint::simulation_error& e) { - _spatiallocationinfo->_errorCode = e.error_code(); - _spatiallocationinfo->_library = e.library_name(); - _spatiallocationinfo->_module = e.module_name(); - _spatiallocationinfo->_message = e.what(); - _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); + } catch (const flint::SimulationError& e) { + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + const int* errorCode = boost::get_error_info(e); + _spatiallocationinfo->_errorCode = *errorCode; + _spatiallocationinfo->_library = libraryName; + _spatiallocationinfo->_module = moduleName; + _spatiallocationinfo->_message = details; + _spinupNotificationCenter.postNotification(moja::signals::Error, details); + return true; + } catch (const flint::LocalDomainError& e) { + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + const int* errorCode = boost::get_error_info(e); + _spatiallocationinfo->_errorCode = *errorCode; + _spatiallocationinfo->_library = libraryName; + _spatiallocationinfo->_module = moduleName; + _spatiallocationinfo->_message = details; + _spinupNotificationCenter.postNotification(moja::signals::Error, details); + } catch (flint::VariableNotFoundException& e) { + std::string str = + ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); + } catch (flint::VariableEmptyWhenValueExpectedException& e) { + std::string str = + ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); + blockStatsUnit->_unitsNotProcessed++; + blockStatsUnit->_unitsWithError++; return true; - } catch (const flint::local_domain_error& e) { - _spatiallocationinfo->_errorCode = e.error_code(); - _spatiallocationinfo->_library = e.library_name(); - _spatiallocationinfo->_module = e.module_name(); - _spatiallocationinfo->_message = e.what(); - _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); + } catch (const Poco::Exception& e) { + std::string str = ((boost::format("In Spinup: %1%") % e.displayText()).str()); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); + } catch (const boost::exception& e) { + std::string str = ((boost::format("In Spinup: %1%") % boost::diagnostic_information(e)).str()); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); + } catch (const Exception& e) { + std::string str = e.displayText(); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); } catch (const std::exception& e) { - _spinupNotificationCenter.postNotification(moja::signals::Error, e.what()); + std::string str = e.what(); + _spinupNotificationCenter.postNotification(moja::signals::Error, str); } catch (...) { _spinupNotificationCenter.postNotification(moja::signals::Error, "unknown exception"); } @@ -848,26 +930,55 @@ bool SpatialTiledLocalDomainController::runCell(std::shared_ptr } // This error is recoverable, retunr true for success - catch (const flint::simulation_error& e) { + catch (const flint::SimulationError& e) { // This error is recoverable, drop cell and continue simulation - _spatiallocationinfo->_errorCode = e.error_code(); - _spatiallocationinfo->_library = e.library_name(); - _spatiallocationinfo->_module = e.module_name(); - _spatiallocationinfo->_message = e.what(); - _notificationCenter.postNotification(moja::signals::Error, e.what()); + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + const int* errorCode = boost::get_error_info(e); + _spatiallocationinfo->_errorCode = *errorCode; + _spatiallocationinfo->_library = libraryName; + _spatiallocationinfo->_module = moduleName; + _spatiallocationinfo->_message = details; + _notificationCenter.postNotification(moja::signals::Error, details); blockStatsUnit->_unitsNotProcessed++; blockStatsUnit->_unitsWithError++; return true; } // All other catches are failures for the localdomain, return false! - catch (const flint::local_domain_error& e) { - _spatiallocationinfo->_errorCode = e.error_code(); - _spatiallocationinfo->_library = e.library_name(); - _spatiallocationinfo->_module = e.module_name(); - _spatiallocationinfo->_message = e.what(); - _notificationCenter.postNotification(moja::signals::Error, e.what()); + catch (const flint::LocalDomainError& e) { + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + const int* errorCode = boost::get_error_info(e); + _spatiallocationinfo->_errorCode = *errorCode; + _spatiallocationinfo->_library = libraryName; + _spatiallocationinfo->_module = moduleName; + _spatiallocationinfo->_message = details; + _notificationCenter.postNotification(moja::signals::Error, details); + } catch (flint::VariableNotFoundException& e) { + std::string str = ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); + _notificationCenter.postNotification(moja::signals::Error, str); + } catch (flint::VariableEmptyWhenValueExpectedException& e) { + std::string str = ((boost::format("Variable not found: %1%") % *(boost::get_error_info(e))).str()); + _notificationCenter.postNotification(moja::signals::Error, str); + blockStatsUnit->_unitsNotProcessed++; + blockStatsUnit->_unitsWithError++; + return true; + } catch (const Poco::Exception& e) { + std::string str = ((boost::format("In Main: %1%") % e.displayText()).str()); + _notificationCenter.postNotification(moja::signals::Error, str); + } catch (const boost::exception& e) { + std::string str = boost::diagnostic_information(e); + _notificationCenter.postNotification(moja::signals::Error, str); + } catch (const Exception& e) { + std::string str = e.displayText(); + blockStatsUnit->_unitsNotProcessed++; + blockStatsUnit->_unitsWithError++; + _notificationCenter.postNotification(moja::signals::Error, str); } catch (const std::exception& e) { - _notificationCenter.postNotification(moja::signals::Error, e.what()); + std::string str = e.what(); + _notificationCenter.postNotification(moja::signals::Error, str); } catch (...) { _notificationCenter.postNotification(moja::signals::Error, "unknown exception"); } @@ -878,7 +989,44 @@ bool SpatialTiledLocalDomainController::runCell(std::shared_ptr // -------------------------------------------------------------------------------------------- -status SpatialTiledLocalDomainController::run() { +// void SpatialTiledLocalDomainController::writeRunSummaryToSQLite(bool insertRunList, +// std::vector& runSummaryData) { auto retry = false; auto maxRetries = RETRY_ATTEMPTS; do { try { +// retry = false; +// SQLite::Connector::registerConnector(); +// Session session("SQLite", _sqliteDatabaseName); +// if (_createStatsTablesSQLIte) { +// +// if (insertRunList) +// session << "DROP TABLE IF EXISTS run_list", now; +// session << "DROP TABLE IF EXISTS run_properties", now; +// } +// if (insertRunList) +// session << "CREATE TABLE IF NOT EXISTS run_list (id INTEGER PRIMARY KEY AUTOINCREMENT, run_id +//VARCHAR(64) NOT NULL, run_desc VARCHAR(255))", now; session << "CREATE TABLE IF NOT EXISTS run_properties (id INTEGER +//PRIMARY KEY AUTOINCREMENT, run_id_fk VARCHAR(64), property_name VARCHAR(255), property_info VARCHAR(255), +//property_value VARCHAR(255))", now; +// +// if (insertRunList) { +// auto insertStr = (boost::format("INSERT INTO run_list (run_id, run_desc) VALUES('%1%','%2%')") +//% _runId % _runDesc).str(); session.begin(); session << insertStr, now; session.commit(); +// } +// +// session.begin(); +// session << "INSERT INTO run_properties (run_id_fk, property_name, property_info, property_value) +//VALUES(?, ?, ?, ?)", bind(runSummaryData), now; session.commit(); SQLite::Connector::unregisterConnector(); +// } +// catch (SQLite::DBLockedException&) { +// MOJA_LOG_DEBUG << localDomainId() << ":DBLockedException - " << maxRetries << " retries +//remaining"; std::this_thread::sleep_for(RETRY_SLEEP); retry = maxRetries-- > 0; if (!retry) { MOJA_LOG_DEBUG << +//localDomainId() << ":Exceeded MAX RETIRES (" << RETRY_ATTEMPTS << ")"; throw; +// } +// } +// } while (retry); +//} + +// -------------------------------------------------------------------------------------------- + +void SpatialTiledLocalDomainController::run() { using namespace std::chrono; using clock = system_clock; using timepoint = time_point; @@ -890,7 +1038,14 @@ status SpatialTiledLocalDomainController::run() { std::string runSummaryInfoStr; if (_writeStatsToSQLIte && (!_threadedSystem || (_threadedSystem && _isMaster))) { - + /// Write first part of Stats to SQLite + /// Stats are finalized after simulation + // StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, localDomainId(), _sqliteDatabaseName, + // "run_stats_global"); StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, localDomainId(), + // _sqliteDatabaseName, "run_stats_tile"); StatsUnitRecord::sqliteCreateTable(_createStatsTablesSQLIte, + // localDomainId(), _sqliteDatabaseName, "run_stats_block"); + + //(boost::format("Thread %1%") % _localDomainId).str(); runSummaryInfoStr = !_threadedSystem ? "Process" : "Master"; if (!_threadedSystem && _landUnitController.hasVariable("spatialLocationInfo")) { auto _spatiallocationinfo = std::static_pointer_cast( @@ -945,7 +1100,7 @@ status SpatialTiledLocalDomainController::run() { } _spatiallocationinfo->_engGlobal.seed(_spatiallocationinfo->_randomSeedGlobal); - while (!_blockIdxList.empty()) { + while (_blockIdxList.size() > 0) { if (!_threadedSystem) { _blockIdxListPosition = int(_blockIdxList.size()); } @@ -1002,7 +1157,7 @@ status SpatialTiledLocalDomainController::run() { break; // This will exit on error return from runCell } } else { - const auto key = std::make_pair(blockIdx.tileIdx, blockIdx.blockIdx); + std::pair key(blockIdx.tileIdx, blockIdx.blockIdx); if (_blockCellIdxList.find(key) != _blockCellIdxList.end()) { auto& blockCells = _blockCellIdxList[key]; for (const auto& cell : blockCells) { @@ -1039,6 +1194,50 @@ status SpatialTiledLocalDomainController::run() { } _notificationCenter.postNotification(signals::LocalDomainProcessingUnitShutdown); // End Processing unit } + } catch (const Exception& e) { + _spatiallocationinfo->_errorCode = 1; + MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting..." + << "Module: " << _spatiallocationinfo->_module << ", " + << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx + << "," << _spatiallocationinfo->_cellIdx << ", " + << "msg:" << e.displayText(); + exit(1); + } catch (const boost::exception& e) { + _spatiallocationinfo->_errorCode = 1; + MOJA_LOG_FATAL << "boost::exception caught at LocalDomain level, exiting..." + << "Module: " << _spatiallocationinfo->_module << ", " + << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx + << "," << _spatiallocationinfo->_cellIdx << ", " + << "msg:" << boost::diagnostic_information(e); + exit(1); + } catch (const Poco::FileException& e) { + _spatiallocationinfo->_errorCode = 1; + MOJA_LOG_FATAL << "Poco::FileException caught at LocalDomain level, exiting..." + << "Module: " << _spatiallocationinfo->_module << ", " + << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx + << "," << _spatiallocationinfo->_cellIdx << ", " + << "msg:" << e.displayText(); + ; + exit(1); + } + // catch (const Poco::Data::SQLite::SQLiteException& e) { + // _spatiallocationinfo->_errorCode = 1; + // MOJA_LOG_FATAL + // << "Poco::Data::SQLite::SQLiteException caught at LocalDomain level, exiting..." + // << "Module: " << _spatiallocationinfo->_module << ", " + // << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx << + //"," << _spatiallocationinfo->_cellIdx << ", " + // << "msg:" << e.displayText(); + // exit(1); + //} + catch (const Poco::Exception& e) { + _spatiallocationinfo->_errorCode = 1; + MOJA_LOG_FATAL << "Poco::Exception caught at LocalDomain level, exiting..." + << "Module: " << _spatiallocationinfo->_module << ", " + << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx + << "," << _spatiallocationinfo->_cellIdx << ", " + << "msg: " << e.displayText(); + exit(1); } catch (const std::exception& e) { _spatiallocationinfo->_errorCode = 1; MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." @@ -1046,7 +1245,7 @@ status SpatialTiledLocalDomainController::run() { << "Location: " << _spatiallocationinfo->_tileIdx << "," << _spatiallocationinfo->_blockIdx << "," << _spatiallocationinfo->_cellIdx << ", " << "msg:" << e.what(); - return status(status_code::Error, std::string("Error in local domain") + e.what()); + exit(1); } catch (...) { _spatiallocationinfo->_errorCode = 1; MOJA_LOG_FATAL << "exception caught at LocalDomain level, exiting..." @@ -1055,7 +1254,7 @@ status SpatialTiledLocalDomainController::run() { << "," << _spatiallocationinfo->_cellIdx << ", " << "msg:" << "unknown exception"; - return status(status_code::Error, std::string("Error in local domain unknown exception")); + exit(1); } } @@ -1077,10 +1276,23 @@ status SpatialTiledLocalDomainController::run() { datarepository::TileIdx tileIdxObject(tileRec->_tileIdx, _provider->indexer()); tileRec->mojaLog("Tile", localDomainId(), &tileIdxObject, nullptr, true); } + // for (auto& i : _tileStatsDimension->getPersistableCollection()) { + // StatsUnitRecord tileStatsRecord(i.get<1>(), seconds(i.get<2>()), seconds(i.get<3>()), + //seconds(i.get<4>()), seconds(i.get<5>()), i.get<6>(), i.get<7>(), i.get<8>(), i.get<9>(), i.get<10>(), + //i.get<11>(), StatsDurationType::Seconds); datarepository::TileIdx tileIdxObject(i.get<10>(), + //_provider->indexer()); tileStatsRecord.mojaLog("Tile", localDomainId(), &tileIdxObject, nullptr, true); + //} MOJA_LOG_INFO << std::setfill(' ') << std::setw(3) << localDomainId() << ":" << "Summary of processing for full run"; + // for (auto& i : _globalStatsDimension->getPersistableCollection()) { + // StatsUnitRecord tileStatsRecord(i.get<1>(), seconds(i.get<2>()), seconds(i.get<3>()), + //seconds(i.get<4>()), seconds(i.get<5>()), i.get<6>(), i.get<7>(), i.get<8>(), i.get<9>(), i.get<10>(), + //i.get<11>(), StatsDurationType::Seconds); totalUnits += tileStatsRecord._unitsTotal; totalUnitsProcessed += + //tileStatsRecord._unitsProcessed; tileStatsRecord.mojaLog("Global", localDomainId(), nullptr, nullptr, true); + //} + for (auto& rec : _globalStatsDimension->records()) { StatsUnitRecord* tileRec = static_cast(rec.get()); totalUnits += tileRec->_unitsTotal; @@ -1124,6 +1336,11 @@ status SpatialTiledLocalDomainController::run() { auto durMS = duration_cast(runTime).count(); auto durS = duration_cast(runTime).count(); + // StatsUnitRecord::sqliteWrite(_globalStatsDimension, localDomainId(), _sqliteDatabaseName, "run_stats_global"); + // StatsUnitRecord::sqliteWrite(_tileStatsDimension, localDomainId(), _sqliteDatabaseName, "run_stats_tile"); + // StatsUnitRecord::sqliteWrite(_blockStatsDimension, localDomainId(), _sqliteDatabaseName, + // "run_stats_block"); + std::vector runSummaryData; runSummaryData.push_back(RunSummaryDataRecord(_runId, "run_system_finish", runSummaryInfoStr, (boost::format("%1%") % moja::put_time(&t2, "%c %Z")).str())); @@ -1163,11 +1380,11 @@ status SpatialTiledLocalDomainController::run() { runSummaryData.push_back(RunSummaryDataRecord(_runId, "run_thread_%1%_time_milliseconds", runSummaryInfoStr, (boost::format("%1%") % durMS).str())); + // writeRunSummaryToSQLite(false, runSummaryData); } - return status(status_code::Ok); } -status SpatialTiledLocalDomainController::startup() { +void SpatialTiledLocalDomainController::startup() { try { if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::SystemInit); @@ -1179,15 +1396,21 @@ status SpatialTiledLocalDomainController::startup() { _spinupNotificationCenter.postNotification(moja::signals::LocalDomainInit); } } + } catch (const flint::SimulationError& e) { + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting." + << " | Library: " << libraryName << " | Module: " << moduleName << " | Details: " << details; + exit(1); } catch (const std::exception& e) { - MOJA_LOG_FATAL << "std::exception caught at LocalDomain level " + MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." << "msg:" << e.what(); - return status(status_code::Error, std::string("Exception caught at LocalDomain level ") + e.what()); + exit(1); } - return status(status_code::Ok); } -status SpatialTiledLocalDomainController::shutdown() { +void SpatialTiledLocalDomainController::shutdown() { try { // When running threaded we don't want main process to fire LocalDomainShutdown if (!_threadedSystem || (_threadedSystem && _isThread)) { @@ -1199,12 +1422,22 @@ status SpatialTiledLocalDomainController::shutdown() { if (_runSpinUp) { _spinupNotificationCenter.postNotification(moja::signals::SystemShutdown); } + } catch (const flint::SimulationError& e) { + std::string details = *(boost::get_error_info
(e)); + std::string libraryName = *(boost::get_error_info(e)); + std::string moduleName = *(boost::get_error_info(e)); + MOJA_LOG_FATAL << "Exception caught at LocalDomain level, exiting." + << " | Library: " << libraryName << " | Module: " << moduleName << " | Details: " << details; + exit(1); } catch (const std::exception& e) { - MOJA_LOG_FATAL << "std::exception caught at LocalDomain level msg:" << e.what(); - return status(status_code::Error, std::string("Exception caught at LocalDomain level ") + e.what()); + MOJA_LOG_FATAL << "std::exception caught at LocalDomain level, exiting..." + << "msg:" << e.what(); + exit(1); } - return status(status_code::Ok); } +#undef RETRY_ATTEMPTS +#undef RETRY_SLEEP + } // namespace flint } // namespace moja diff --git a/Source/moja.flint/src/spinuplandunitcontroller.cpp b/Source/moja.flint/src/spinuplandunitcontroller.cpp index 25eda1e..86dddbe 100644 --- a/Source/moja.flint/src/spinuplandunitcontroller.cpp +++ b/Source/moja.flint/src/spinuplandunitcontroller.cpp @@ -5,6 +5,7 @@ #include "moja/flint/ioperation.h" #include "moja/flint/ivariable.h" #include "moja/flint/landunitcontroller.h" +#include "moja/flint/operationmanagerublas.h" #include @@ -87,7 +88,7 @@ void SpinupLandUnitController::addVariable(std::string name, std::shared_ptrsecond.get(); } @@ -95,7 +96,7 @@ IVariable* SpinupLandUnitController::getVariable(const std::string& name) { const IVariable* SpinupLandUnitController::getVariable(const std::string& name) const { const auto v = _variablesMap.find(name); if (v == _variablesMap.end()) { - std::runtime_error("Error in spin up landunit controller variable not found " + name); + BOOST_THROW_EXCEPTION(VariableNotFoundException() << VariableName(name)); } return v->second.get(); } diff --git a/Source/moja.flint/src/sqlquerytransform.cpp b/Source/moja.flint/src/sqlquerytransform.cpp index 4c14b93..759a7f2 100644 --- a/Source/moja.flint/src/sqlquerytransform.cpp +++ b/Source/moja.flint/src/sqlquerytransform.cpp @@ -9,12 +9,13 @@ #include #include -#include +#include + +#include #include #include -#include using moja::datarepository::IProviderRelationalInterface; @@ -67,9 +68,8 @@ void SQLQueryTransform::configure(DynamicObject config, const ILandUnitControlle } std::string SQLQueryTransform::readSQLFile(const std::string& path) { - namespace fs = moja::filesystem; - if (!fs::exists(path)) { - throw std::runtime_error("SQL file not found " + path); + if (!Poco::File(path).exists()) { + throw FileNotFoundException(path); } std::ifstream file(path); @@ -169,7 +169,7 @@ std::string SQLQueryTransform::formatVariableValues(const IVariable& var, Dynami if (_allowEmptyVarValues) { return "NULL"; } else { - throw std::runtime_error("SQL value NULL when value expected"); + BOOST_THROW_EXCEPTION(VariableEmptyWhenValueExpectedException() << VariableName(var.info().name)); } } diff --git a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp index 43778a7..6400ccc 100644 --- a/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp +++ b/Source/moja.flint/src/threadedaspatiallocaldomaincontroller.cpp @@ -11,6 +11,8 @@ #include #include +#include + #include using moja::flint::ILocalDomainController; @@ -22,7 +24,7 @@ namespace moja { namespace flint { AspatialLocalDomainThread::AspatialLocalDomainThread(ThreadedAspatialLocalDomainController* parent, int threadId, - std::mutex& tileListMutex, + Poco::Mutex& tileListMutex, std::queue& tileList, const configuration::Configuration* config) : _parent(parent), @@ -47,8 +49,8 @@ void AspatialLocalDomainThread::operator()() { auto keepRunning = true; while (keepRunning) { // Pop a block index from the queue - std::unique_lock lock(_tileListMutex); - if (!_tileList.empty()) { + Poco::ScopedLockWithUnlock lock(_tileListMutex); + if (_tileList.size() > 0) { auto tile = _tileList.front(); _tileList.pop(); MOJA_LOG_INFO << std::setfill(' ') << std::setw(3) << _threadId @@ -68,7 +70,7 @@ void AspatialLocalDomainThread::operator()() { ThreadedAspatialLocalDomainController::ThreadedAspatialLocalDomainController() : LocalDomainControllerBase() {} -status ThreadedAspatialLocalDomainController::configure(const configuration::Configuration& config) { +void ThreadedAspatialLocalDomainController::configure(const configuration::Configuration& config) { // Call base class configure LocalDomainControllerBase::configure(config); @@ -92,10 +94,9 @@ status ThreadedAspatialLocalDomainController::configure(const configuration::Con _tasks.push_back(task); } - return status(status_code::Ok); } -status ThreadedAspatialLocalDomainController::run() { +void ThreadedAspatialLocalDomainController::run() { auto startTime = DateTime::now(); for (const auto& task : _tasks) { @@ -105,13 +106,14 @@ status ThreadedAspatialLocalDomainController::run() { for (auto& thread : _threads) { thread.join(); } - + + //_notificationCenter.postNotification(moja::signals::SystemShutdown); + auto endTime = DateTime::now(); auto ldSpan = endTime - startTime; MOJA_LOG_INFO << "LocalDomain: Start Time : " << startTime; MOJA_LOG_INFO << "LocalDomain: Finish Time : " << endTime; MOJA_LOG_INFO << "LocalDomain: Total Time (seconds) : " << ldSpan.totalSeconds(); - return status(status_code::Ok); } } // namespace flint diff --git a/Source/moja.flint/src/writesystemconfig.cpp b/Source/moja.flint/src/writesystemconfig.cpp index 14ce4bf..849e727 100644 --- a/Source/moja.flint/src/writesystemconfig.cpp +++ b/Source/moja.flint/src/writesystemconfig.cpp @@ -20,36 +20,18 @@ #include #include #include -#include -#include +#include +#include -#include +#include #include #include -namespace fs = moja::filesystem; - namespace moja { namespace flint { - // --- RAII class for file handle -class FileHandle { - typedef FILE* ptr; - - public: - explicit FileHandle(const moja::filesystem::path& name, std::string const& mode = std::string("r")) - : _wrapped_file(fopen(name.string().c_str(), mode.c_str())) {} - ~FileHandle() { - if (_wrapped_file) fclose(_wrapped_file); - } - operator ptr() const { return _wrapped_file; } - - private: - ptr _wrapped_file; -}; - // -------------------------------------------------------------------------------------------- void WriteSystemConfig::configure(const DynamicObject& config) { @@ -94,16 +76,25 @@ void WriteSystemConfig::subscribe(NotificationCenter& notificationCenter) { // -------------------------------------------------------------------------------------------- void WriteSystemConfig::onSystemInit() { - fs::path workingFolder(_outputPath); - if (!fs::exists(workingFolder)) { - fs::create_directories(workingFolder); + Poco::File workingFolder(_outputPath); + if (!workingFolder.exists()) { + try { + workingFolder.createDirectories(); + } catch (Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed + in 1.7.8 */ + } } - if (fs::exists(workingFolder) && !fs::is_directory(workingFolder)) { - MOJA_LOG_ERROR << "Error creating spatial tiled point configurations output folder: " << workingFolder.string(); + if (workingFolder.exists() && !workingFolder.isDirectory()) { + MOJA_LOG_ERROR << "Error creating spatial tiled point configurations output folder: " << _outputPath; } - auto outputFolderPath = workingFolder / _name; - if (!fs::exists(outputFolderPath)) { - fs::create_directories(outputFolderPath); + auto outputFolderPath = (boost::format("%1%%2%%3%") % workingFolder.path() % Poco::Path::separator() % _name).str(); + Poco::File outputFolder(outputFolderPath); + if (!outputFolder.exists()) { + try { + outputFolder.createDirectories(); + } catch (Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed + in 1.7.8 */ + } } } @@ -265,7 +256,7 @@ void outputDynamicToStream(std::ofstream& fout, const DynamicVar& object, int le if (object.type() == typeid(DateTime)) { DateTime dt = object.extract(); std::string simpleDateStr = - fmt::format("{ \"$date\": \"{}/{}/{}\" }", dt.year(), dt.month(), dt.day()); + (boost::format("{ \"$date\": \"%1%/%2%/%3%\" }") % dt.year() % dt.month() % dt.day()).str(); fout << simpleDateStr; // fout << "\"" << escape_json(simpleDateStr) << "\""; } else if (object.type() == typeid(Int16)) { @@ -322,13 +313,18 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { auto timestep = timing->step(); auto timesubstep = timing->subStep(); + Poco::File workingFolder(_outputPath); auto configFilename = - fs::path(_outputPath) / _name / - fmt::format("{:05}_{:03}_{:06}_{}_{}_{}.json", _spatialLocationInfo ? _spatialLocationInfo->_tileIdx : 0, - _spatialLocationInfo ? _spatialLocationInfo->_blockIdx : 0, - _spatialLocationInfo ? _spatialLocationInfo->_cellIdx : 0, notificationStr, timestep, timesubstep); - - if (fs::exists(configFilename)) fs::remove(configFilename); // delete existing config file + (boost::format("%1%%2%%3%%4%%5%_%6%_%7%_%8%_%9%_%10%.json") % workingFolder.path() % Poco::Path::separator() % + _name % Poco::Path::separator() % + boost::io::group(std::setfill('0'), std::setw(5), _spatialLocationInfo ? _spatialLocationInfo->_tileIdx : 0) % + boost::io::group(std::setfill('0'), std::setw(3), _spatialLocationInfo ? _spatialLocationInfo->_blockIdx : 0) % + boost::io::group(std::setfill('0'), std::setw(6), _spatialLocationInfo ? _spatialLocationInfo->_cellIdx : 0) % + notificationStr % timestep % timesubstep) + .str(); + + Poco::File configFile(configFilename); + if (configFile.exists()) configFile.remove(false); // delete existing config file // configFile.createFile() FileHandle pFile(configFilename, "wb"); @@ -357,12 +353,15 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { DynamicObject localdomain; auto localDomainConfig = config->localDomain(); localdomain["type"] = configuration::LocalDomain::localDomainTypeToStr(localDomainConfig->type()); - localdomain["start_date_init"] = fmt::format("{}/{}/{}", config->startDate().year(), - config->startDate().month(), config->startDate().day()); - localdomain["start_date"] = fmt::format("{}/{}/{}", timing->curStartDate().year(), - timing->curStartDate().month(), timing->curStartDate().day()); + localdomain["start_date_init"] = (boost::format("%1%/%2%/%3%") % config->startDate().year() % + config->startDate().month() % config->startDate().day()) + .str(); + localdomain["start_date"] = (boost::format("%1%/%2%/%3%") % timing->curStartDate().year() % + timing->curStartDate().month() % timing->curStartDate().day()) + .str(); localdomain["end_date"] = - fmt::format("{}/{}/{}", config->endDate().year(), config->endDate().month(), config->endDate().day()); + (boost::format("%1%/%2%/%3%") % config->endDate().year() % config->endDate().month() % config->endDate().day()) + .str(); localdomain["sequencer_library"] = localDomainConfig->sequencerLibrary(); localdomain["sequencer"] = localDomainConfig->sequencer(); localdomain["simulateLandUnit"] = localDomainConfig->simulateLandUnit(); diff --git a/Source/moja.flint/src/writevariablegrid.cpp b/Source/moja.flint/src/writevariablegrid.cpp index fd2422f..b9289f9 100644 --- a/Source/moja.flint/src/writevariablegrid.cpp +++ b/Source/moja.flint/src/writevariablegrid.cpp @@ -9,38 +9,21 @@ #include #include -#include -#include +#include +#include + #include #include -#include +#include +#include #include -#include -#include - -namespace fs = moja::filesystem; +#include namespace moja { namespace flint { - // --- RAII class for file handle -class FileHandle { - typedef FILE* ptr; - - public: - explicit FileHandle(const fs::path& name, std::string const& mode = std::string("r")) - : _wrapped_file(fopen(name.string().c_str(), mode.c_str())) {} - ~FileHandle() { - if (_wrapped_file) fclose(_wrapped_file); - } - operator ptr() const { return _wrapped_file; } - - private: - ptr _wrapped_file; -}; - void WriteVariableGrid::configure(const DynamicObject& config) { _globalOutputPath = config.contains("output_path") ? config["output_path"].convert() : ""; @@ -315,12 +298,32 @@ template void WriteVariableGrid::DataSettingsT::doSystemInit(flint::ILandUnitDataWrapper* _landUnitData) { flint::VariableAndPoolStringBuilder databaseNameBuilder(_landUnitData, _outputPath); _outputPath = databaseNameBuilder.result(); - const fs::path working_folder(_outputPath); + std::string variableFolder; if (_forceVariableFolderName) { - fs::create_directories(working_folder / _name); + variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); } else { - fs::create_directories(working_folder); + variableFolder = ""; + } + + Poco::File workingFolder(_outputPath); + const auto spatialOutputFolderPath = (boost::format("%1%%2%") % workingFolder.path() % variableFolder + // % Poco::Path::separator() + // % _name + ) + .str(); + + try { + workingFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + } + + Poco::File spatialOutputFolder(spatialOutputFolderPath); + try { + spatialOutputFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ } } @@ -347,25 +350,43 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainInit(flint::ILandUnitData template void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitInit( std::shared_ptr spatialLocationInfo) { - fs::path working_folder(_outputPath); + Poco::File workingFolder(_outputPath); std::string variableFolder; if (_forceVariableFolderName) { - working_folder /= _name; + variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); + } else { + variableFolder = ""; } - fs::path tile_folder; if (_useIndexesForFolderName) { - tile_folder = (working_folder /= fmt::format("{:06}", spatialLocationInfo->_tileIdx)); + _tileFolderPath = (boost::format("%1%%2%%3%%4%") % + workingFolder.path() + // % Poco::Path::separator() + // % _name + % variableFolder % Poco::Path::separator() % + boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx)) + .str(); } else { - tile_folder = (working_folder /= fmt::format("{}{:03}{}{:03}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lon), - spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lat))); - } - _tileFolderPath = tile_folder.string(); - std::unique_lock lock(_fileHandlingMutex); - fs::create_directories(tile_folder); + _tileFolderPath = + (boost::format("%1%%2%%3%%4%%5%_%6%%7%") % + workingFolder.path() + // % Poco::Path::separator() + // % _name + % variableFolder % Poco::Path::separator() % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % + (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat))) + .str(); + } + + Poco::File tileFolder(_tileFolderPath); + Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + try { + tileFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + } } // -------------------------------------------------------------------------------------------- @@ -382,31 +403,40 @@ void WriteVariableGrid::DataSettingsT::initializeData(std::shared_ptr void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( std::shared_ptr spatialLocationInfo) { - std::unique_lock lock(_fileHandlingMutex); + Poco::Mutex::ScopedLock lock(_fileHandlingMutex); // for (const auto& timestepData : _data) { typename std::unordered_map>::iterator itPrev; - for (auto it = _data.begin(); it != _data.end(); ++it) { - const auto& [step, data] = (*it); + const auto& timestepData = (*it); - fs::path tileFolder(_tileFolderPath); + Poco::File tileFolder(_tileFolderPath); std::string folderLocStr; if (_useIndexesForFolderName) { - folderLocStr = fmt::format("{}_{:06}_{:02}_{}", _name, spatialLocationInfo->_tileIdx, - spatialLocationInfo->_blockIdx, step); + folderLocStr = (boost::format("%1%_%2%") % + boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx) % + boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) + .str(); } else { - folderLocStr = fmt::format( - "{}_{}{:03}_{}{:03}_{:02}_{}", _name, spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lon), spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lat), spatialLocationInfo->_blockIdx, step); + folderLocStr = + (boost::format("%1%%2%_%3%%4%_%5%") % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % + (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat)) % + boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) + .str(); } - auto filenameGrd = tileFolder / fmt::format("{}.grd", folderLocStr); - auto filenameHdr = tileFolder / fmt::format("{}.hdr", folderLocStr); + auto filenameGrd = (boost::format("%1%%2%%3%_%4%_%5%.grd") % tileFolder.path() % Poco::Path::separator() % _name % + folderLocStr % timestepData.first) + .str(); + auto filenameHdr = (boost::format("%1%%2%%3%_%4%_%5%.hdr") % tileFolder.path() % Poco::Path::separator() % _name % + folderLocStr % timestepData.first) + .str(); - if (fs::exists(filenameGrd)) { - fs::remove(filenameGrd); // delete existing file + Poco::File blockGrd(filenameGrd); + if (blockGrd.exists()) { + blockGrd.remove(false); // delete existing file } int cellRows = spatialLocationInfo->_cellRows; @@ -416,24 +446,25 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( FileHandle pFile(filenameGrd, "wb"); if (_subtractPrevValue) { if (it == _data.begin()) // prev value is 0 - fwrite(data.data(), sizeof(T), numCells, pFile); + fwrite(timestepData.second.data(), sizeof(T), numCells, pFile); else { // I know i have a prevIt - const auto& [step_prev, data_prev] = (*itPrev); + const auto& timestepDataPrev = (*itPrev); std::vector newData; - auto it1 = data.begin(); - auto it2 = data_prev.begin(); - for (; it1 != data.end() && it2 != data_prev.end(); ++it1, ++it2) { + auto it1 = timestepData.second.begin(); + auto it2 = timestepDataPrev.second.begin(); + for (; it1 != timestepData.second.end() && it2 != timestepDataPrev.second.end(); ++it1, ++it2) { newData.push_back(*it1 - *it2); } fwrite(newData.data(), sizeof(T), numCells, pFile); } } else { - fwrite(data.data(), sizeof(T), numCells, pFile); + fwrite(timestepData.second.data(), sizeof(T), numCells, pFile); } - std::ofstream out(filenameHdr); + boost::iostreams::stream_buffer buf(filenameHdr); + std::ostream out(&buf); out << "ENVI" << std::endl; out << "description = { " << filenameGrd << " }" << std::endl; @@ -462,7 +493,6 @@ void WriteVariableGrid::DataSettingsT::doLocalDomainProcessingUnitShutdown( "\"Greenwich\",0],UNIT[\"Degree\",0.017453292519943295]] }" << std::endl; out << "band names = { Band 1 }" << std::endl; - out.close(); itPrev = it; } _data.clear(); diff --git a/Source/moja.flint/tests/CMakeLists.txt b/Source/moja.flint/tests/CMakeLists.txt index f46cf07..3bf5f88 100644 --- a/Source/moja.flint/tests/CMakeLists.txt +++ b/Source/moja.flint/tests/CMakeLists.txt @@ -14,9 +14,11 @@ set(TEST_SRCS src/flinttests.cpp src/landunitcontrollertests.cpp src/lookuptransformtests.cpp + src/matrixtestsublas.cpp src/operationmanagertestutils.cpp src/operationmanagersimpletests.cpp src/operationmanagersimplecachetests.cpp + src/operationmanagerublastests.cpp src/sqlquerytransformtests.cpp src/recordaccumulatorintegrationtests.cpp ) diff --git a/Source/moja.flint/tests/src/matrixtestseigen.cpp b/Source/moja.flint/tests/src/matrixtestseigen.cpp new file mode 100644 index 0000000..db57fc2 --- /dev/null +++ b/Source/moja.flint/tests/src/matrixtestseigen.cpp @@ -0,0 +1,301 @@ +#include +#include "moja/flint/timing.h" +#include "moja/datetime.h" +#include "moja/flint/matrixeigen.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include // std::setprecision + +// warning C4996: 'std::_Copy_impl': Function call with parameters that may be unsafe - this call relies on the caller to check that the passed values are correct. +#pragma warning( push ) +#pragma warning( disable : 4996 ) +#include "Eigen/Dense" +#include "Eigen/Sparse" +#pragma warning( pop ) + +using namespace moja; +using namespace moja::flint; + +BOOST_AUTO_TEST_SUITE(MatrixTestsEigen); + +typedef Eigen::SparseMatrix EigenSparseMat; // declares a column-major sparse matrix type of double +typedef Eigen::DiagonalMatrix EigenDiagonal; +typedef Eigen::MatrixXd EigenMat; // declares a dynamic matrix of type double +typedef Eigen::VectorXd EigenVec; // declares a dynamic vector of type double +typedef Eigen::MatrixXd moja_identity; + +typedef Eigen::Triplet test_eigen_triplet; +typedef std::vector test_eigen_tripletlist; + + +void eigen_transferPool(EigenMat& _matrix, int source, int sink, double proportion) { + //Eigen::Triplet t(source, sink, proportion); + + _matrix(source, sink) += proportion; + _matrix(source, source) -= proportion; +} + +void eigen_transferPool_Sparse(test_eigen_tripletlist& tripletList, int source, int sink, double value) { + tripletList.push_back(moja_eigen_triplet(sink, source, value)); +} + +template +inline T sum_kahan3e(const EigenMat& xs) { + if (xs.size() == 0) return 0; + T sumP(0); + T sumN(0); + T tP(0); + T tN(0); + T cP(0); + T cN(0); + T yP(0); + T yN(0); + for (size_t i = 0, size = xs.size(); i < size; i++) { + if ((*(xs.data() + i)) > 0) { + yP = (*(xs.data() + i)) - cP; + tP = sumP + yP; + cP = (tP - sumP) - yP; + sumP = tP; + } + else { + yN = (*(xs.data() + i)) - cN; + tN = sumN + yN; + cN = (tN - sumN) - yN; + sumN = tN; + } + } + return sumP + sumN; +} + + +// -------------------------------------------------------------------------------------------- + +#if 0 + +BOOST_AUTO_TEST_CASE(flint_Matrix_eigen_DoSomeMatrixStuff) { + + // Shows setIdentity blanks matrix in bounds of the 1's being inserted in diag + + Eigen::Matrix4i m1 = Eigen::Matrix4i::Zero(); + m1.block<3, 3>(1, 0).setIdentity(); + std::cout << "m1" << std::endl; + std::cout << m1 << std::endl; + + Eigen::Matrix4i m2 = Eigen::Matrix4i::Zero(); + m2(0, 1) = 9; + m2(0, 2) = 7; + m2.block<3, 3>(1, 0).setIdentity(); + std::cout << "m2" << std::endl; + std::cout << m2 << std::endl; + + Eigen::Matrix4i m3 = Eigen::Matrix4i::Zero(); + m3(0, 1) = 9; + m3(0, 2) = 7; + m3(1, 2) = 5; + m3.setIdentity(); + std::cout << "m3" << std::endl; + std::cout << m3 << std::endl; + + // Play with Vectors + + Eigen::Matrix4i m4 = Eigen::Matrix4i::Identity(); + + Eigen::Vector2i v1 = Eigen::Vector2i::Zero(); + std::cout << "v1" << std::endl; + std::cout << v1 << std::endl; + + Eigen::Vector4i v2 = Eigen::Vector4i::Identity(); + std::cout << "v2" << std::endl; + std::cout << v2 << std::endl; + + Eigen::Vector4i r_v1 = m4 * v2; + std::cout << "r_v1" << std::endl; + std::cout << r_v1 << std::endl; + + //Eigen::Vector4i r_v2 = m4 * v1; + //std::cout << "r_v2" << std::endl; + //std::cout << r_v2 << std::endl; +} + +#endif + +// -------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_iterate_data) { + Eigen::Matrix test = Eigen::Matrix::Random(5, 5); + + auto now = time(0); + sum_kahan3e(test); +} + +void simpleConstructAndCalc(int length) { + auto testName = boost::unit_test::framework::current_test_case().p_name; + auto testSuiteName = (boost::unit_test::framework::get(boost::unit_test::framework::current_test_case().p_parent_id)).full_name(); + + auto startConstruct = clock(); + + moja_eigen_matrix _testMatrix; + _testMatrix.resize(length, length); + _testMatrix.setIdentity(); + //_testMatrix.setZero(); + moja_eigen_vector _testVector(length); + _testVector.setRandom(length); + + auto finishConstruct = clock(); + + //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << ": Mat : " << _testMatrix << std::endl; + //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << ": Vec : " << _testVector << std::endl; + + auto startCalcs = clock(); + + for (int i = 0; i < 100; i++) + for (int j = 0; j < 10000; j++) { + moja_eigen_vector result = MOJA_EIGEN_MAT_PROD(_testMatrix, _testVector); + } + + auto finishCalcs = clock(); + auto lengthCalcs = finishCalcs - startCalcs; + auto lengthConstruct = finishConstruct - startConstruct; + + // CLOCKS_PER_SEC + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << " : For size (" << length << "): construction : " << lengthConstruct << std::endl; + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(40) << std::setfill(' ') << testName << " : For size (" << length << "): calculations : " << lengthCalcs << std::endl; +} + +// -------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(flint_Matrix_eigen_SpeedChecks) { + + for (int i = 10; i <= 60; i+=10) + simpleConstructAndCalc(i); +} + + +// -------------------------------------------------------------------------------------------- + +BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_001) { + const int numVars = 6; + EigenVec pools(numVars); + pools.setZero(); + + pools[0] = 0.02; + pools[1] = 8.681; + pools[2] = 0.05; + pools[3] = 0; + pools[4] = 20.96; + pools[5] = 0; + + EigenMat _I(numVars, numVars); + _I.setIdentity(); + + EigenDiagonal diag = pools.asDiagonal(); + EigenMat forOutput = diag; + + EigenMat matrix(numVars, numVars); + matrix.setIdentity(); + + eigen_transferPool(matrix, 0, 5, 0.250852); + eigen_transferPool(matrix, 1, 5, 0.012596); + eigen_transferPool(matrix, 2, 5, 0); + eigen_transferPool(matrix, 3, 5, 0.0190627); + eigen_transferPool(matrix, 4, 5, 0.00126915); + eigen_transferPool(matrix, 0, 2, 0.0397643); + eigen_transferPool(matrix, 1, 2, 0.00199667); + eigen_transferPool(matrix, 3, 2, 0.00302176); + eigen_transferPool(matrix, 4, 3, 0); + eigen_transferPool(matrix, 0, 4, 0.0413873); + eigen_transferPool(matrix, 1, 4, 0.00207817); + eigen_transferPool(matrix, 2, 4, 0); + eigen_transferPool(matrix, 3, 4, 0.00314509); + eigen_transferPool(matrix, 4, 4, 0.000410574); + + auto a = matrix - _I; + auto b = diag * a; + auto flux = b * 1; // timeScale + + pools = pools.transpose() * matrix; // timeScale + + BOOST_CHECK_CLOSE(pools[0], 0.013359927999999998, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[1], 8.5362804379599986, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[2], 0.068128378269999998, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[3], 0, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[4], 20.952266955769996, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[5], 0.14096429999999999, 0.000000000000001); +} + +// -------------------------------------------------------------------------------------------- +// NOTE: Eigen will NOT remove the 0.0 results from the final sparse matrix + +BOOST_AUTO_TEST_CASE(moja_Matrix_eigen_zero_transfer_test) { + auto testName = boost::unit_test::framework::current_test_case().p_name; + auto testSuiteName = (boost::unit_test::framework::get(boost::unit_test::framework::current_test_case().p_parent_id)).full_name(); + + const int numVars = 6; + EigenVec pools(numVars); + pools.setZero(); + + pools[0] = 0.02; + pools[1] = 8.681; + pools[2] = 0.05; + pools[3] = 0; + pools[4] = 20.96; + pools[5] = 0; + + EigenMat _I(numVars, numVars); + _I.setIdentity(); + + EigenDiagonal diag = pools.asDiagonal(); + test_eigen_tripletlist tripletList; + EigenSparseMat matrix; + matrix.resize(numVars, numVars); + matrix.reserve(5); + //matrix.setZero(); + + //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix blank" << std::endl; + //std::cout << matrix << std::endl; + + eigen_transferPool_Sparse(tripletList, 0, 5, 0.0); + eigen_transferPool_Sparse(tripletList, 0, 4, 0.1); + + matrix.setFromTriplets(tripletList.begin(), tripletList.end()); + + //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix set" << std::endl; + //std::cout << matrix << std::endl; + + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": matrix iteration" << std::endl; + for (int k = 0; k < matrix.outerSize(); ++k) { + for (EigenSparseMat::InnerIterator flux(matrix, k); flux; ++flux) { + auto srcIx = flux.col(); + auto dstIx = flux.row(); + auto val = flux.value(); + + std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; + } + } + + EigenSparseMat fluxes = matrix * diag; + + //std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": fluxes" << std::endl; + //std::cout << fluxes << std::endl; + + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << testName << ": fluxes iteration" << std::endl; + for (int k = 0; k < fluxes.outerSize(); ++k) { + for (EigenSparseMat::InnerIterator flux(fluxes, k); flux; ++flux) { + auto srcIx = flux.col(); + auto dstIx = flux.row(); + auto val = flux.value(); + + std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; + } + } + +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/Source/moja.flint/tests/src/matrixtestsublas.cpp b/Source/moja.flint/tests/src/matrixtestsublas.cpp new file mode 100644 index 0000000..e8df16d --- /dev/null +++ b/Source/moja.flint/tests/src/matrixtestsublas.cpp @@ -0,0 +1,159 @@ +#include + +// warning C4996: 'std::_Uninitialized_copy0': Function call with parameters that may be unsafe - this call relies on +// the caller to check that the passed values are correct. +#pragma warning(push) +#pragma warning(disable : 4996) +// Work-around needed for ublas when using boost >= 1.64 +//#include +#include +#include +#include +#include +#include +#pragma warning(pop) + +#include + +#include +#include +#include // std::setprecision +#include +#include +#include +#include +#include +#include + +BOOST_AUTO_TEST_SUITE(MatrixTestsuBlas); + +namespace ublas = boost::numeric::ublas; +typedef ublas::vector moja_vector; +typedef ublas::compressed_matrix moja_matrix; +typedef ublas::diagonal_matrix moja_diagonal; +typedef ublas::identity_matrix moja_identity; +typedef moja_matrix::iterator1 it1_t; +typedef moja_matrix::iterator2 it2_t; + +void transferPool(moja_matrix& _matrix, int source, int sink, double proportion) { + _matrix(source, sink) += proportion; + _matrix(source, source) -= proportion; +} + +BOOST_AUTO_TEST_CASE(flint_Matrix_flux) { + const int test_size = 6; + + moja_vector pools(test_size); + pools[0] = 0.02; + pools[1] = 8.681; + pools[2] = 0.05; + pools[3] = 0; + pools[4] = 20.96; + pools[5] = 0; + + moja_identity _I(test_size); + moja_diagonal diag(test_size, pools.data()); + moja_matrix matrix(_I); + + transferPool(matrix, 0, 5, 0.250852); + transferPool(matrix, 1, 5, 0.012596); + transferPool(matrix, 2, 5, 0); + transferPool(matrix, 3, 5, 0.0190627); + transferPool(matrix, 4, 5, 0.00126915); + transferPool(matrix, 0, 2, 0.0397643); + transferPool(matrix, 1, 2, 0.00199667); + transferPool(matrix, 3, 2, 0.00302176); + transferPool(matrix, 4, 3, 0); + transferPool(matrix, 0, 4, 0.0413873); + transferPool(matrix, 1, 4, 0.00207817); + transferPool(matrix, 2, 4, 0); + transferPool(matrix, 3, 4, 0.00314509); + transferPool(matrix, 4, 4, 0.000410574); + + // Flux + //-0.00664007 0 0.000795285 0 0.000827746 0.00501704 + // 0 - 0.144719 0.0173331 0 0.0180406 0.109346 + // 0 0 0 0 0 0 + // 0 0 0 0 0 0 + // 0 0 0 0 - 0.0266013 0.0266013 + // 0 0 0 0 0 0 + + auto a = matrix - _I; + auto b = ublas::prec_prod(diag, a); + auto flux = b * 1; // timeScale + + pools = ublas::prec_prod(pools, matrix) * 1; // timeScale + + BOOST_CHECK_CLOSE(pools[0], 0.013359927999999998, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[1], 8.5362804379599986, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[2], 0.068128378269999998, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[3], 0, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[4], 20.952266955769996, 0.000000000000001); + BOOST_CHECK_CLOSE(pools[5], 0.14096429999999999, 0.000000000000001); +} + +// -------------------------------------------------------------------------------------------- +// NOTE: UBLAS will remove the 0.0 results from the final sparse matrix + +BOOST_AUTO_TEST_CASE(moja_Matrix_ublas_zero_transfer_test) { + auto testName = boost::unit_test::framework::current_test_case().p_name; + auto testSuiteName = (boost::unit_test::framework::get( + boost::unit_test::framework::current_test_case().p_parent_id)) + .full_name(); + + const int test_size = 6; + moja_vector pools(test_size); + + pools[0] = 0.02; + pools[1] = 8.681; + pools[2] = 0.05; + pools[3] = 0; + pools[4] = 20.96; + pools[5] = 0; + + moja_diagonal diag(test_size, pools.data()); + + moja_matrix matrix(test_size, test_size, test_size); + + // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << + // testName << ": matrix blank" << std::endl; std::cout << matrix << std::endl; + + matrix(0, 5) = 0.0; + matrix(0, 4) = 0.1; + + // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << + // testName << ": matrix set" << std::endl; std::cout << matrix << std::endl; + + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') + << testName << ": matrix iteration" << std::endl; + for (it1_t it1 = matrix.begin1(); it1 != matrix.end1(); ++it1) { + for (it2_t it2 = it1.begin(); it2 != it1.end(); ++it2) { + // std::cout << "(" << it2.index1() << "," << it2.index2() << ") = "; + // std::cout << *it2 << std::endl; + auto srcIx = it2.index1(); + auto dstIx = it2.index2(); + auto val = *it2; + std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; + } + } + + moja_matrix fluxes = ublas::prec_prod(diag, matrix); + + // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << + // testName << ": fluxes" << std::endl; std::cout << fluxes << std::endl; + + std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') + << testName << ": fluxes iteration" << std::endl; + for (it1_t it1 = fluxes.begin1(); it1 != fluxes.end1(); ++it1) { + for (it2_t it2 = it1.begin(); it2 != it1.end(); ++it2) { + // std::cout << "(" << it2.index1() << "," << it2.index2() << ") = "; + // std::cout << *it2 << std::endl; + auto srcIx = it2.index1(); + auto dstIx = it2.index2(); + auto val = *it2; + std::cout << "Src: " << srcIx << ", Dst: " << dstIx << ", Val: " << val << std::endl; + } + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/Source/moja.flint/tests/src/operationmanagertestutils.cpp b/Source/moja.flint/tests/src/operationmanagertestutils.cpp index 83e3f9f..4ec3165 100644 --- a/Source/moja.flint/tests/src/operationmanagertestutils.cpp +++ b/Source/moja.flint/tests/src/operationmanagertestutils.cpp @@ -234,6 +234,24 @@ void test_NoResultFluxIteration(mf::IOperationManager& manager, mf::IModule& mod BOOST_CHECK_EQUAL(manager.operationResultsPending().size(), 1); + // UBLAS will remove the zero result flux in the product, other implementations won't + + // if (manager.operationResultsPending().size() == 0) + // return; + + // auto operationResult = *(manager.operationResultsPending().begin()); + + // std::cout << std::setw(40) << std::setfill(' ') << testSuiteName << ": " << std::setw(50) << std::setfill(' ') << + // testName << ": fluxes" << std::endl; + + // int count = 0; + // for (auto p : operationResult->operationResultFluxCollection()) { + // std::cout << "Src: "<< p->source() << ", Snk: " << p->sink() << ", Val: " << p->value() << std::endl; + // count++; + //} + + // BOOST_CHECK_EQUAL(count, 0); + data.clear(); } diff --git a/Source/moja.flint/tests/src/operationmanagerublastests.cpp b/Source/moja.flint/tests/src/operationmanagerublastests.cpp new file mode 100644 index 0000000..aa6a26a --- /dev/null +++ b/Source/moja.flint/tests/src/operationmanagerublastests.cpp @@ -0,0 +1,155 @@ +#include "flinttests.h" +#include "operationmanagertestutils.h" + +#include +#include +#include + +#include + +#include + +#include + +using namespace moja; +using namespace moja::flint; +namespace conf = moja::flint::configuration; + +// -------------------------------------------------------------------------------------------- +struct PoolInitValueAndResult { + std::string name; + double value; + double result; + + std::shared_ptr poolHandle; +}; + +// -------------------------------------------------------------------------------------------- + +#define OP_MANAGER_TO_USE OperationManagerUblas + +// -------------------------------------------------------------------------------------------- + +struct UblasFixture { + Timing timing; + ModuleBase module; + DynamicObject config; + OperationManagerUblas manager; + + UblasFixture() : config({{"use_kahan", true}}), manager(timing, config) { + timing.setStartDate(DateTime(2001, 1, 1)); + timing.setStartDate(DateTime(2013, 12, 31)); + timing.init(); + + auto& metadata = module.metaData(); + metadata.setDefaults(); + metadata.moduleName = "testModuleUblas"; + } +}; + +BOOST_FIXTURE_TEST_SUITE(Ublas_operationmanagertests, UblasFixture); + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_NoPoolIteration) { test_NoPoolIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SinglePoolIteration) { test_SinglePoolIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_MultiplePoolIteration) { test_MultiplePoolIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_NoResultIteration) { test_NoResultIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SingleResultIteration) { test_SingleResultIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_MultipleResultIteration) { test_MultipleResultIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_NoResultFluxIteration) { test_NoResultFluxIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SingleResultFluxIteration) { test_SingleResultFluxIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_MultipleResultFluxIteration) { test_MultipleResultFluxIteration(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SingleProportionTransfer) { test_SingleProportionTransfer(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SingleStockTransfer) { test_SingleStockTransfer(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_DoubleProportionalTransfer) { test_DoubleProportionalTransfer(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_DoubleStockTransfer) { test_DoubleStockTransfer(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_DoubleStockAndApplyTransfer) { test_DoubleStockAndApplyTransfer(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_TwoOperationsStockAndProportional) { +// test_TwoOperationsStockAndProportional(manager, module); +// } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_Kahan_summation_issues_Proportion) { +// test_Kahan_summation_issues_Proportion(manager, module); +// } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_Kahan_summation_issues_Proportion_with_SpinUp) { +// test_Kahan_summation_issues_Proportion_with_SpinUp(manager, module); +// } + +#if 0 // don't do performance testing, too slow for blas + +// -------------------------------------------------------------------------------------------- +BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestProportionalSLEEK) { + test_PerformanceTestProportionalSLEEK(manager, module); +} + +// -------------------------------------------------------------------------------------------- +BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestStockSLEEK) { + test_PerformanceTestStockSLEEK(manager, module); +} + +// -------------------------------------------------------------------------------------------- +BOOST_AUTO_TEST_CASE(Ublas_PerformanceTestCBM) { + test_PerformanceTestCBM(manager, module); +} + +#endif + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_SubmitOperationAddsToPendingQueue) { SubmitOperationAddsToPendingQueue(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ClearLastAppliedOperationResults) { ClearLastAppliedOperationResults(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsAppendsToCommittedQueue) { +// ApplyOperationsAppendsToCommittedQueue(manager, module); +// } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsCorrectlyUpdatesPoolsForSimpleCase) { +// ApplyOperationsCorrectlyUpdatesPoolsForSimpleCase(manager, module); +// } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsCorrectlyUpdatesPoolsForComplexCase) { +// ApplyOperationsCorrectlyUpdatesPoolsForComplexCase(manager, module); +// } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ApplyAndGetOperationsLastApplied) { ApplyAndGetOperationsLastApplied(manager, module); } + +// // -------------------------------------------------------------------------------------------- +// BOOST_AUTO_TEST_CASE(Ublas_ApplyOperationsClearsPendingQueue) { ApplyOperationsClearsPendingQueue(manager, module); } + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h b/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h index 6b54f0a..25241ff 100644 --- a/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h +++ b/Source/moja.modules.gdal/include/moja/modules/gdal/writevariablegeotiff.h @@ -9,7 +9,6 @@ #include #include -#include namespace moja { @@ -34,7 +33,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { float64 = 7, }; - explicit WriteVariableGeotiff(std::mutex& fileHandlingMutex) + explicit WriteVariableGeotiff(Poco::Mutex& fileHandlingMutex) : _fileHandlingMutex(fileHandlingMutex), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -58,7 +57,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { // --- Base class for data layer class DataSettingsB { public: - DataSettingsB(std::mutex& fileHandlingMutex, data_type dataType) + DataSettingsB(Poco::Mutex& fileHandlingMutex, data_type dataType) : notificationType(OnNotificationType::TimingInit), _useIndexesForFolderName(false), _forceVariableFolderName(true), @@ -118,14 +117,14 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { const flint::IVariable* _variable; std::vector _pool; std::string _tileFolderPath; - std::mutex& _fileHandlingMutex; + Poco::Mutex& _fileHandlingMutex; }; // --- Templated version of Base class for data layer types template class DataSettingsT : public DataSettingsB { public: - DataSettingsT(std::mutex& fileHandlingMutex, data_type dataType) + DataSettingsT(Poco::Mutex& fileHandlingMutex, data_type dataType) : DataSettingsB(fileHandlingMutex, dataType){}; ~DataSettingsT() = default; @@ -158,7 +157,7 @@ class GDAL_API WriteVariableGeotiff : public flint::ModuleBase { private: // Mutexes - std::mutex& _fileHandlingMutex; + Poco::Mutex& _fileHandlingMutex; // FlintData std::shared_ptr _spatialLocationInfo; diff --git a/Source/moja.modules.gdal/src/libraryfactory.cpp b/Source/moja.modules.gdal/src/libraryfactory.cpp index 3686ba5..4e8f11c 100644 --- a/Source/moja.modules.gdal/src/libraryfactory.cpp +++ b/Source/moja.modules.gdal/src/libraryfactory.cpp @@ -21,7 +21,7 @@ namespace gdal { extern "C" { -std::mutex _fileHandlingMutexVarGeotiffWriter; +Poco::Mutex _fileHandlingMutexVarGeotiffWriter; MOJA_LIB_API int getModuleRegistrations(ModuleRegistration* outModuleRegistrations) { int index = 0; diff --git a/Source/moja.modules.gdal/src/rasterreadergdal.cpp b/Source/moja.modules.gdal/src/rasterreadergdal.cpp index 8280705..aeee1dc 100644 --- a/Source/moja.modules.gdal/src/rasterreadergdal.cpp +++ b/Source/moja.modules.gdal/src/rasterreadergdal.cpp @@ -1,5 +1,6 @@ #include "moja/modules/gdal/rasterreadergdal.h" +#include "moja/datarepository/datarepositoryexceptions.h" #include "moja/logging.h" #include "gdalcpp.h" @@ -15,6 +16,7 @@ #include #include +#include namespace moja { namespace modules { @@ -39,11 +41,19 @@ std::tuple seqChunk2geog(int tile_idx, int block_idx) { MetaDataRasterReaderGDAL::MetaDataRasterReaderGDAL(const std::string& path, const std::string& prefix, const DynamicObject& settings) : MetaDataRasterReaderInterface(path, prefix, settings) { - const auto filePath = fs::absolute(path).replace_extension(".json"); - _path = filePath.string(); - _metaDataRequired = true; - if (settings.contains("metadata_required")) { - _metaDataRequired = settings["metadata_required"].extract(); + try { + auto filePath = Poco::Path(path); + auto parent = filePath.parent().toString(); + auto abs = filePath.parent().absolute().toString(); + _path = (boost::format("%1%%2%.json") % abs % filePath.getBaseName()).str(); + _metaDataRequired = true; + if (settings.contains("metadata_required")) { + _metaDataRequired = settings["metadata_required"].extract(); + } + } catch (...) { + BOOST_THROW_EXCEPTION(flint::LocalDomainError() + << flint::Details("GDAL Error in constructor") << flint::LibraryName("moja.modules.gdal") + << flint::ModuleName(BOOST_CURRENT_FUNCTION) << flint::ErrorCode(1)); } } @@ -59,7 +69,7 @@ DynamicObject MetaDataRasterReaderGDAL::readMetaData() const { return layerMetadata; } else { if (_metaDataRequired) { - throw std::runtime_error("Error metadata file not found " + _path); + BOOST_THROW_EXCEPTION(datarepository::FileNotFoundException() << datarepository::FileName(_path)); } else { return DynamicObject(); } @@ -164,11 +174,18 @@ void TileRasterReaderGDAL::readBlockData(const datarepository::BlockIdx& blkIdx, } else { MOJA_LOG_ERROR << "RunId (" << _runId << ") - " << "GDAL - read error, target (" << _path << ")"; - - throw std::runtime_error("GDAL error reading block data " + _path); + const auto str = fmt::format("GDAL read error: {}", _path); + BOOST_THROW_EXCEPTION(flint::LocalDomainError() + << flint::Details(str) << flint::LibraryName("moja.modules.gdal") + << flint::ModuleName(BOOST_CURRENT_FUNCTION) << flint::ErrorCode(1)); } } catch (const gdalcpp::gdal_error& err) { - throw std::runtime_error("GDAL error reading block data " + _path + " " + err.what()); + BOOST_THROW_EXCEPTION(datarepository::FileReadException() + << datarepository::FileName(_path) << datarepository::Message(err.what())); + } catch (...) { + MOJA_LOG_ERROR << "RunId (" << _runId << ") - " + << "GDAL - exception (" << _path << ")"; + throw; } } } // namespace gdal diff --git a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp index 015ae0d..7885df3 100644 --- a/Source/moja.modules.gdal/src/writevariablegeotiff.cpp +++ b/Source/moja.modules.gdal/src/writevariablegeotiff.cpp @@ -1,5 +1,7 @@ #include "moja/modules/gdal/writevariablegeotiff.h" +#include "moja/exception.h" + #include #include #include @@ -9,9 +11,10 @@ #include #include -#include -#include +#include +#include +#include #include #include @@ -22,8 +25,6 @@ #include #include -namespace fs = moja::filesystem; - namespace moja { namespace modules { namespace gdal { @@ -303,11 +304,32 @@ void WriteVariableGeotiff::DataSettingsT::doSystemInit(flint::ILandUnitDataWr flint::VariableAndPoolStringBuilder databaseNameBuilder(_landUnitData, _outputPath); _outputPath = databaseNameBuilder.result(); - fs::path spatialOutputFolderPath(_outputPath); + std::string variableFolder; if (_forceVariableFolderName) { - spatialOutputFolderPath /= _name; + variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); + } else { + variableFolder = ""; + } + + Poco::File workingFolder(_outputPath); + const auto spatialOutputFolderPath = (boost::format("%1%%2%") % workingFolder.path() % variableFolder + // % Poco::Path::separator() + // % _name + ) + .str(); + + try { + workingFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ + } + + Poco::File spatialOutputFolder(spatialOutputFolderPath); + try { + spatialOutputFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ } - fs::create_directories(spatialOutputFolderPath); } // -------------------------------------------------------------------------------------------- @@ -333,24 +355,43 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainInit(flint::ILandUnitD template void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitInit( std::shared_ptr spatialLocationInfo) { - fs::path workingFolder(_outputPath); + Poco::File workingFolder(_outputPath); + std::string variableFolder; if (_forceVariableFolderName) { - workingFolder /= _name; + variableFolder = (boost::format("%1%%2%") % Poco::Path::separator() % _name).str(); + } else { + variableFolder = ""; } - fs::path tileFolder; if (_useIndexesForFolderName) { - tileFolder = workingFolder / fmt::format("{:06}", spatialLocationInfo->_tileIdx); + _tileFolderPath = (boost::format("%1%%2%%3%%4%") % + workingFolder.path() + // % Poco::Path::separator() + // % _name + % variableFolder % Poco::Path::separator() % + boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx)) + .str(); } else { - tileFolder = workingFolder / fmt::format("{}{:03}_{}{:03}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lon), - spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lat)); + _tileFolderPath = + (boost::format("%1%%2%%3%%4%%5%_%6%%7%") % + workingFolder.path() + // % Poco::Path::separator() + // % _name + % variableFolder % Poco::Path::separator() % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % + (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat))) + .str(); + } + + Poco::File tileFolder(_tileFolderPath); + Poco::Mutex::ScopedLock lock(_fileHandlingMutex); + try { + tileFolder.createDirectories(); + } catch ( + Poco::FileExistsException&) { /* Poco has a bug here, exception shouldn't be thrown, has been fixed in 1.7.8 */ } - - std::unique_lock lock(_fileHandlingMutex); - fs::create_directories(tileFolder); } // -------------------------------------------------------------------------------------------- @@ -408,7 +449,7 @@ struct dataset_closer { GDALDataset* gdal_dataset_; }; -static std::shared_ptr create_gdalraster(const fs::path& path, int rows, int cols, +static std::shared_ptr create_gdalraster(const Poco::File& path, int rows, int cols, GDALDataType datatype, double* transform) { if (GDALGetDriverCount() == 0) { GDALAllRegister(); @@ -420,9 +461,11 @@ static std::shared_ptr create_gdalraster(const fs::path& path, i options = CSLSetNameValue(options, "TILED", "YES"); options = CSLSetNameValue(options, "COMPRESS", "DEFLATE"); - auto dataset = driver->Create(path.string().c_str(), cols, rows, 1, datatype, options); + auto dataset = driver->Create(path.path().c_str(), cols, rows, 1, datatype, options); if (dataset == nullptr) { - throw std::runtime_error("Could not create raster file: " + path.string()); + std::ostringstream oss; + oss << "Could not create raster file: " << path.path() << std::endl; + throw ApplicationException(oss.str()); } dataset->SetGeoTransform(transform); OGRSpatialReference srs; @@ -438,17 +481,23 @@ static std::shared_ptr create_gdalraster(const fs::path& path, i template void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown( std::shared_ptr spatialLocationInfo) { - std::unique_lock lock(_fileHandlingMutex); + Poco::Mutex::ScopedLock lock(_fileHandlingMutex); - fs::path tileFolder(_tileFolderPath); + Poco::File tileFolder(_tileFolderPath); std::string folderLocStr; if (_useIndexesForFolderName) { - folderLocStr = fmt::format("{:06}_{:02}", spatialLocationInfo->_tileIdx, spatialLocationInfo->_blockIdx); + folderLocStr = + (boost::format("%1%_%2%") % boost::io::group(std::setfill('0'), std::setw(6), spatialLocationInfo->_tileIdx) % + boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) + .str(); } else { - folderLocStr = fmt::format("{}{:03}_{}{:03}_{:02}", spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lon), - spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "", - std::abs(spatialLocationInfo->_tileLatLon.lat), spatialLocationInfo->_blockIdx); + folderLocStr = + (boost::format("%1%%2%_%3%%4%_%5%") % (spatialLocationInfo->_tileLatLon.lon < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lon)) % + (spatialLocationInfo->_tileLatLon.lat < 0 ? "-" : "") % + boost::io::group(std::setfill('0'), std::setw(3), std::abs(spatialLocationInfo->_tileLatLon.lat)) % + boost::io::group(std::setfill('0'), std::setw(2), spatialLocationInfo->_blockIdx)) + .str(); } int cellRows = spatialLocationInfo->_cellRows; @@ -462,38 +511,42 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown typename std::unordered_map>::iterator itPrev; for (auto it = _data.begin(); it != _data.end(); ++it) { - const auto& [step, data] = (*it); + auto& timestepData = (*it); + + auto filename = (boost::format("%1%%2%%3%_%4%_%5%.tif") % tileFolder.path() % Poco::Path::separator() % _name % + folderLocStr % timestepData.first) + .str(); - auto block_path = tileFolder /= fmt::format("{}_{}_{}.tif", _name, folderLocStr, step); - if (fs::exists(block_path)) { - fs::remove(block_path); // delete existing file + Poco::File block_path(filename); + if (block_path.exists()) { + block_path.remove(false); // delete existing file } auto band = create_gdalraster(block_path, cellRows, cellCols, gdal_type(_dataType), adfGeoTransform); auto err = band->SetNoDataValue(_nodataValue); if (err != CE_None) { std::ostringstream oss; - oss << "Could not set raster file nodata: " << block_path.string() << " (" << _nodataValue << ")" << std::endl; - throw std::runtime_error(oss.str()); + oss << "Could not set raster file nodata: " << block_path.path() << " (" << _nodataValue << ")" << std::endl; + throw ApplicationException(oss.str()); } if (_subtractPrevValue) { if (it == _data.begin()) { // prev value is 0 - auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, (void*)data.data(), cellCols, + auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, timestepData.second.data(), cellCols, cellRows, gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.string() << std::endl; - throw std::runtime_error(oss.str()); + oss << "Could not write to raster file: " << block_path.path() << std::endl; + throw ApplicationException(oss.str()); } } else { // I know i have a prevIt - const auto& [step_prev, data_prev] = (*itPrev); + const auto& timestepDataPrev = (*itPrev); std::vector newData; - auto it1 = data.begin(); - auto it2 = data_prev.begin(); - for (; it1 != data.end() && it2 != data_prev.end(); ++it1, ++it2) { + auto it1 = timestepData.second.begin(); + auto it2 = timestepDataPrev.second.begin(); + for (; it1 != timestepData.second.end() && it2 != timestepDataPrev.second.end(); ++it1, ++it2) { newData.push_back(*it1 - *it2); } @@ -501,17 +554,17 @@ void WriteVariableGeotiff::DataSettingsT::doLocalDomainProcessingUnitShutdown gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.string() << std::endl; - throw std::runtime_error(oss.str()); + oss << "Could not write to raster file: " << block_path.path() << std::endl; + throw ApplicationException(oss.str()); } } } else { - auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, (void*)data.data(), cellCols, + auto err = band->RasterIO(GF_Write, 0, 0, cellCols, cellRows, timestepData.second.data(), cellCols, cellRows, gdal_type(_dataType), 0, 0); if (err != CE_None) { std::ostringstream oss; - oss << "Could not write to raster file: " << block_path.string() << std::endl; - throw std::runtime_error(oss.str()); + oss << "Could not write to raster file: " << block_path.path() << std::endl; + throw ApplicationException(oss.str()); } } itPrev = it; diff --git a/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp b/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp index dc8ca09..5f3710a 100644 --- a/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp +++ b/Source/moja.modules.libpq/src/providerrelationallibpqpostgresql.cpp @@ -1,5 +1,7 @@ #include "moja/modules/libpq/providerrelationallibpqpostgresql.h" +#include + #include #include @@ -44,7 +46,7 @@ class PostgresConnection { PQerrorMessage(_conn) % connInfo) .str(); MOJA_LOG_ERROR << "PostgresConnection: " << msg; - throw std::runtime_error("Error Postgres connection failed: " + msg); + BOOST_THROW_EXCEPTION(datarepository::ConnectionFailedException() << datarepository::ConnectionError(msg)); } PQsetNoticeReceiver(_conn, noticeReceiver, nullptr); PQsetNoticeProcessor(_conn, noticeProcessor, nullptr); @@ -327,7 +329,8 @@ class ProviderRelationalLibpqPostgreSQL::impl { PQerrorMessage(_conn) % _dbConnString) .str(); MOJA_LOG_ERROR << "ProviderRelationalLibpqPostgreSQL: " << msg; - throw std::runtime_error("Error " + msg); + BOOST_THROW_EXCEPTION(datarepository::QueryException() + << datarepository::SQL(query) << datarepository::SQLError(msg)); break; } case PGRES_COPY_BOTH: From 9f64a6fc54b1d2613a1338ae77fb6d31b878d95f Mon Sep 17 00:00:00 2001 From: Max Fellows Date: Wed, 11 Aug 2021 09:26:55 -0700 Subject: [PATCH 40/40] Bug fixes and GCBM compatibility: - option to clear fluxes after accumulating records - uncertainty aggregators now correctly get the land unit area in all types of simulation - uncertainty variable now sets value directly instead of using setter for compatibility with read-only variable types (ExternalVariable) --- .../moja/flint/aggregatoruncertainty.h | 1 + .../flint/aggregatoruncertaintylandunit.h | 1 + .../include/moja/flint/recordaccumulator.h | 2 ++ .../moja.flint/src/aggregatoruncertainty.cpp | 22 +++++++++++++--- .../src/aggregatoruncertaintylandunit.cpp | 25 +++++++++++++++---- .../src/uncertaintylandunitsqlitewriter.cpp | 4 +-- Source/moja.flint/src/uncertaintyvariable.cpp | 3 +-- 7 files changed, 45 insertions(+), 13 deletions(-) diff --git a/Source/moja.flint/include/moja/flint/aggregatoruncertainty.h b/Source/moja.flint/include/moja/flint/aggregatoruncertainty.h index eacddf9..0eb7b4c 100644 --- a/Source/moja.flint/include/moja/flint/aggregatoruncertainty.h +++ b/Source/moja.flint/include/moja/flint/aggregatoruncertainty.h @@ -56,6 +56,7 @@ class FLINT_API AggregatorUncertainty : public ModuleBase { bool module_info_on_; bool aggregate_sink_and_source_; bool aggregate_stock_; + bool clear_fluxes_after_recording_; // -- Land Unit level Collections // -- these collections will be kep for each land unit and put into greater aggregation on LU success. diff --git a/Source/moja.flint/include/moja/flint/aggregatoruncertaintylandunit.h b/Source/moja.flint/include/moja/flint/aggregatoruncertaintylandunit.h index 4971a77..6283af5 100644 --- a/Source/moja.flint/include/moja/flint/aggregatoruncertaintylandunit.h +++ b/Source/moja.flint/include/moja/flint/aggregatoruncertaintylandunit.h @@ -27,6 +27,7 @@ class FLINT_API AggregatorUncertaintyLandUnitSharedData { bool cell_index_on; bool do_stock; bool output_month_12_only; + bool clear_fluxes_after_recording; }; typedef Poco::Tuple runStatDataRecord; diff --git a/Source/moja.flint/include/moja/flint/recordaccumulator.h b/Source/moja.flint/include/moja/flint/recordaccumulator.h index f2f6a62..80f8474 100644 --- a/Source/moja.flint/include/moja/flint/recordaccumulator.h +++ b/Source/moja.flint/include/moja/flint/recordaccumulator.h @@ -553,6 +553,8 @@ class RecordAccumulatorMap2 { const rec_accu_map& getRecords() const { return _records; } + rec_accu_map& getRecords() { return _records; } + std::vector getPersistableCollection() const { std::vector persistables; persistables.reserve(_records.size()); diff --git a/Source/moja.flint/src/aggregatoruncertainty.cpp b/Source/moja.flint/src/aggregatoruncertainty.cpp index ba10193..92195b9 100644 --- a/Source/moja.flint/src/aggregatoruncertainty.cpp +++ b/Source/moja.flint/src/aggregatoruncertainty.cpp @@ -1,5 +1,6 @@ #include "moja/flint/aggregatoruncertainty.h" +#include "moja/flint/spatiallocationinfo.h" #include "moja/flint/ilandunitdatawrapper.h" #include "moja/flint/ioperationresult.h" #include "moja/flint/ioperationresultflux.h" @@ -37,6 +38,10 @@ void AggregatorUncertainty::configure(const DynamicObject& config) { if (config.contains("aggregate_stock")) { aggregate_stock_ = config["aggregate_stock"]; } + clear_fluxes_after_recording_ = false; + if (config.contains("clear_fluxes_after_recording")) { + clear_fluxes_after_recording_ = config["clear_fluxes_after_recording"]; + } } void AggregatorUncertainty::subscribe(NotificationCenter& notificationCenter) { @@ -110,10 +115,15 @@ void AggregatorUncertainty::onTimingInit() { simulation_unit_data_->lu_count_processing_unit++; simulation_unit_data_->lu_count_local_domain++; - if (_landUnitData->hasVariable("landUnitArea")) - simulation_unit_data_->land_unit_area = _landUnitData->getVariable("landUnitArea")->value(); - else - simulation_unit_data_->land_unit_area = 1.0; + if (_landUnitData->hasVariable("spatialLocationInfo")) { + auto spatialLocationInfo = std::static_pointer_cast( + _landUnitData->getVariable("spatialLocationInfo")->value().extract>()); + simulation_unit_data_->land_unit_area = spatialLocationInfo->getProperty("landUnitArea"); + } else if (_landUnitData->hasVariable("landUnitArea")) { + simulation_unit_data_->land_unit_area = _landUnitData->getVariable("landUnitArea")->value(); + } else { + simulation_unit_data_->land_unit_area = 1.0; + } // Clear the LU data sets in prep for this LU processing fluxes_lu_.clear(); @@ -202,6 +212,10 @@ void AggregatorUncertainty::record_flux_set() { flux_data_lu{ date_record_id, module_info_id, src_ix, dst_ix, flux_value}); } } + + if (clear_fluxes_after_recording_) { + _landUnitData->clearLastAppliedOperationResults(); + } } void AggregatorUncertainty::record_stock_set() { diff --git a/Source/moja.flint/src/aggregatoruncertaintylandunit.cpp b/Source/moja.flint/src/aggregatoruncertaintylandunit.cpp index 78f8ac6..20cd79d 100644 --- a/Source/moja.flint/src/aggregatoruncertaintylandunit.cpp +++ b/Source/moja.flint/src/aggregatoruncertaintylandunit.cpp @@ -60,6 +60,10 @@ void AggregatorUncertaintyLandUnit::configure(const DynamicObject& config) { } else { classifier_set_var_name_ = "classifier_set"; } + aggregator_land_unit_shared_data_.clear_fluxes_after_recording = false; + if (config.contains("clear_fluxes_after_recording")) { + aggregator_land_unit_shared_data_.clear_fluxes_after_recording = config["clear_fluxes_after_recording"]; + } } void AggregatorUncertaintyLandUnit::subscribe(NotificationCenter& notificationCenter) { @@ -176,10 +180,15 @@ void AggregatorUncertaintyLandUnit::onTimingInit() { simulation_unit_data_->lu_count_processing_unit++; simulation_unit_data_->lu_count_local_domain++; - if (_landUnitData->hasVariable("landUnitArea")) - simulation_unit_data_->land_unit_area = _landUnitData->getVariable("landUnitArea")->value(); - else - simulation_unit_data_->land_unit_area = 1.0; + if (_landUnitData->hasVariable("spatialLocationInfo")) { + auto spatialLocationInfo = std::static_pointer_cast( + _landUnitData->getVariable("spatialLocationInfo")->value().extract>()); + simulation_unit_data_->land_unit_area = spatialLocationInfo->getProperty("landUnitArea"); + } else if (_landUnitData->hasVariable("landUnitArea")) { + simulation_unit_data_->land_unit_area = _landUnitData->getVariable("landUnitArea")->value(); + } else { + simulation_unit_data_->land_unit_area = 1.0; + } constructed_tile_id_ = 0; // TileIdx [24 bits], blockIdx [12 bits], cellIdx [28 bits] @@ -282,7 +291,9 @@ void AggregatorUncertaintyLandUnit::onTimingShutdown() { stocks_lu_.clear(); } -void AggregatorUncertaintyLandUnit::onOutputStep() { recordStockSet(); } +void AggregatorUncertaintyLandUnit::onOutputStep() { + recordStockSet(); +} void AggregatorUncertaintyLandUnit::onError(std::string msg) { fluxes_lu_.clear(); @@ -358,6 +369,10 @@ void AggregatorUncertaintyLandUnit::recordFluxSet() { fluxes_lu_.emplace_back(fluxDataLU{date_record_id, module_info_id, srcIx, dstIx, fluxValue}); } } + + if (aggregator_land_unit_shared_data_.clear_fluxes_after_recording) { + _landUnitData->clearLastAppliedOperationResults(); + } } } // namespace moja::flint diff --git a/Source/moja.flint/src/uncertaintylandunitsqlitewriter.cpp b/Source/moja.flint/src/uncertaintylandunitsqlitewriter.cpp index bd0b6b1..12e69f4 100644 --- a/Source/moja.flint/src/uncertaintylandunitsqlitewriter.cpp +++ b/Source/moja.flint/src/uncertaintylandunitsqlitewriter.cpp @@ -477,7 +477,7 @@ double _st_dev(const std::vector& data) { void UncertaintyLandUnitSQLiteWriter::calculate_stdev() { { - auto& records = simulation_unit_data_->land_unit_stock_results.get_records(); + auto& records = simulation_unit_data_->land_unit_stock_results.getRecords(); for (auto& rec : records) { auto& values = rec.second.values; rec.second.stdev = _st_dev(values); @@ -487,7 +487,7 @@ void UncertaintyLandUnitSQLiteWriter::calculate_stdev() { } } { - auto& records = simulation_unit_data_->land_unit_flux_results.get_records(); + auto& records = simulation_unit_data_->land_unit_flux_results.getRecords(); for (auto& rec : records) { auto& values = rec.second.fluxes; rec.second.stdev = _st_dev(values); diff --git a/Source/moja.flint/src/uncertaintyvariable.cpp b/Source/moja.flint/src/uncertaintyvariable.cpp index e08e8ee..cb8d70f 100644 --- a/Source/moja.flint/src/uncertaintyvariable.cpp +++ b/Source/moja.flint/src/uncertaintyvariable.cpp @@ -72,8 +72,7 @@ const DynamicVar& UncertaintyVariable::value() const { } } } -// value_ = val; - variable_->set_value(val); + value_ = val; } } else { auto& val = value_;