Restart netCDF file on checkpoint restart (NGWPC-10459)

include/core/Layer.hpp

@@ -132,7 +132,7 @@ namespace ngen
         std::string unit_name() const;
         virtual std::vector<std::string> required_checkpoint_units() const;
 
-        virtual std::map<std::string, std::string> get_catchment_output_data_for_timestep();
+        virtual const std::map<std::string, std::string>& get_catchment_output_data_for_timestep();
         virtual void set_simulations_output_format(std::vector<std::string> out_formats);
         virtual std::vector<std::string> get_simulations_output_format();
         protected:

include/core/NgenSimulation.hpp

@@ -101,7 +101,7 @@ class NgenSimulation
      * @param merge_all_ranks Whether to also get the units from other MPI ranks. This should only be `true` when calling blocking MPI processes is safe for the program.
      */
     std::vector<std::string> required_checkpoint_units(bool merge_all_ranks) const;
-    void create_netcdf_writer(std::shared_ptr<realization::Formulation_Manager> manager, std::string nc_output_file_name);
+    void create_netcdf_writer(std::shared_ptr<realization::Formulation_Manager> manager, std::string nc_output_file_name, bool create_new_file);
 
 private:
     void advance_models_one_output_step();

include/netcdf/NetCDFFile.hpp

@@ -16,10 +16,17 @@
 #endif
 #include "NetCDFVar.hpp"
 
+
+enum class NetCDFOpenMode {
+    OPEN_READ,
+    OPEN_WRITE,
+    CREATE
+};
+
 class NetCDFFile {
 public:
 
-    NetCDFFile(const std::string& filename, bool write_only, bool is_mpi);
+    NetCDFFile(const std::string& filename, NetCDFOpenMode open_mode, bool is_mpi);
     ~NetCDFFile();
 
     void load_attributes(std::shared_ptr<NetCDFVar> nc_var) ;

include/netcdf/NetCDFManager.hpp

@@ -28,10 +28,10 @@ class NetCDFManager
 {
 public:
      NetCDFManager(std::shared_ptr<realization::Formulation_Manager> manager, 
-        const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs);
+        const std::string& output_name, Simulation_Time const& sim_time, bool create_new_file, int mpi_rank, int mpi_num_procs);
 
     // Constructor for read-only NetCDF (no MPI needed)
-    NetCDFManager(const std::string& filename, bool read_only);
+    NetCDFManager(const std::string& filename, NetCDFOpenMode open_mode);
 
     // Default constructor for mdframe tests 
     NetCDFManager();
@@ -43,9 +43,6 @@ class NetCDFManager
 
     void gather_all_catchments(const std::vector<int64_t>& catchments_in_proc);
 
-    //set up netcdf dimensions and variables
-    void define_catchment_netcdf_components();
-
     // List variable names
     std::vector<std::string> list_variables() const;
 
@@ -70,19 +67,21 @@ class NetCDFManager
     void add_output_variable_data_from_formulation();
 
     // Add catchment output data to the file (for writing)
-    void write_simulations_response_from_formulation(size_t time_index, std::map<std::string, std::string> catchment_output_values);
+    void write_simulations_response_from_formulation(size_t time_index, const std::map<std::string, std::string>& catchment_output_values);
     void primary_netcdf_writer(size_t time_index, const std::map<int64_t, std::string>& catchment_output_values);
     void secondary_netcdf_worker(const std::map<int64_t, std::string>& catchment_output_values);
 
     ~NetCDFManager();
 
 private:
-    bool read_only_;
+    /* Set up netcdf dimensions and variables.
+       Note: A copy of Simulation_Time is passed because the object is expected to be modified and discarded at the end of the function. */
+    void define_catchment_netcdf_components(Simulation_Time sim_time);
+    NetCDFOpenMode open_mode_;
     std::string nc_filename_;
     std::unique_ptr<NetCDFFile> nc_file_;
     std::vector<NetCDFVar> vars_;
     std::shared_ptr<realization::Formulation_Manager> manager_;
-    std::shared_ptr<Simulation_Time> sim_time_;
     size_t num_timesteps_;
     int num_catchments_ = 0;
     std::vector<int64_t> catchments_;

src/NGen.cpp

@@ -714,11 +714,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) {
                                                        std::move(nexus_indexes),
                                                        mpi_rank,
                                                        mpi_num_procs);
-    #if NGEN_WITH_NETCDF
-        if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()){
-            simulation->create_netcdf_writer(manager, "catchment_output");
-        }
-    #endif //NGEN_WITH_NETCDF
     auto time_done_init                             = std::chrono::steady_clock::now();
     std::chrono::duration<double> time_elapsed_init = time_done_init - time_start;
     LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO);
@@ -732,6 +727,9 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) {
         }
     }
 
+#if NGEN_WITH_NETCDF
+    bool create_new_netcdf_file = true;
+#endif // NGEN_WITH_NETCDF
     { // optionally load a checkpoint if configured
         auto checkpoint_loader = state_saving_config.checkpoint_loader();
         if (checkpoint_loader) {
@@ -740,8 +738,18 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) {
             std::shared_ptr<State_Snapshot_Loader> snapshot_loader
                 = checkpoint_loader->initialize_checkpoint_snapshot(required_units);
             simulation->load_checkpoint(snapshot_loader);
+#if NGEN_WITH_NETCDF
+            create_new_netcdf_file = false;
+#endif //NGEN_WITH_NETCDF
         }
     }
+#if NGEN_WITH_NETCDF
+    if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()) {
+        if (!create_new_netcdf_file)
+            LOG(LogLevel::INFO, "Attempting to open the output of the prior checkpoint run's netCDF output file. The prior file must be located in the default output location to be correctly read.");
+        simulation->create_netcdf_writer(manager, "catchment_output", create_new_netcdf_file);
+    }
+#endif //NGEN_WITH_NETCDF
 
     if (state_saving_config.has_checkpoint_saver()) {
         int checkpoint_frequency;

src/core/Layer.cpp

@@ -188,7 +188,7 @@ void ngen::Layer::load_hot_start(std::shared_ptr<State_Snapshot_Loader> snapshot
     }
 }
 
-std::map<std::string, std::string> ngen::Layer::get_catchment_output_data_for_timestep(){
+const std::map<std::string, std::string>& ngen::Layer::get_catchment_output_data_for_timestep(){
     return catchment_output_values;
 }
 

src/core/NgenSimulation.cpp

@@ -220,8 +220,8 @@ void NgenSimulation::advance_models_one_output_step()
                     std::vector<std::string> output_formats = layer->get_simulations_output_format();
                     if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()){
                         produce_netcdf_format_ = true;
-                        std::map<std::string, std::string> catchment_output_vals = layer->get_catchment_output_data_for_timestep();
-                        nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals);
+                        const std::map<std::string, std::string> &catchment_output_vals = layer->get_catchment_output_data_for_timestep();
+                        nc_manager_->write_simulations_response_from_formulation(simulation_step_, catchment_output_vals);
                     }
                 #endif //NGEN_WITH_NETCDF
 
@@ -524,10 +524,10 @@ void NgenSimulation::serialize(Archive& ar, const unsigned int version) {
 #endif //NGEN_WITH_NEXUSES
 }
 
-void NgenSimulation::create_netcdf_writer(std::shared_ptr<realization::Formulation_Manager> manager, std::string nc_output_file_name)
+void NgenSimulation::create_netcdf_writer(std::shared_ptr<realization::Formulation_Manager> manager, std::string nc_output_file_name, bool create_new_file)
 {
 #if NGEN_WITH_NETCDF
-    this->nc_manager_ = std::make_unique<NetCDFManager>(manager, nc_output_file_name, *sim_time_, mpi_rank_, mpi_num_procs_);
+    this->nc_manager_ = std::make_unique<NetCDFManager>(manager, nc_output_file_name, *sim_time_, create_new_file, mpi_rank_, mpi_num_procs_);
 #endif
 }
 

src/forcing/NetCDFPerFeatureDataProvider.cpp

@@ -53,7 +53,7 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat
     //nc_set_chunk_cache(sizep, nelemsp, preemptionp);
 
     //open the file
-    nc_manager = std::make_shared<NetCDFManager>(input_path, true);
+    nc_manager = std::make_shared<NetCDFManager>(input_path, NetCDFOpenMode::OPEN_READ);
     nc_manager->open_file();
 
     //nc_get_chunk_cache(&sizep, &nelemsp, &preemptionp);

src/netcdf/NetCDFFile.cpp

@@ -23,19 +23,27 @@
     } while (0)
 
 
-NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, bool is_mpi)
+NetCDFFile::NetCDFFile(const std::string& filename, NetCDFOpenMode open_mode, bool is_mpi)
     : nc_file_name_(filename), is_mpi_(is_mpi)
 {
-    int mode = NC_NETCDF4;
-    if(write_only){
-        read_only_ = false;
-        NC_CHECK(nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_), "Creating NetCDF file failed");
+    switch (open_mode) {
+        case NetCDFOpenMode::OPEN_READ:
+            read_only_ = true;
+            NC_CHECK(nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_), "Opening NetCDF file in read-only mode failed.");
+            break;
+        case NetCDFOpenMode::OPEN_WRITE:
+            read_only_ = false;
+            NC_CHECK(nc_open(nc_file_name_.c_str(), NC_WRITE, &ncid_), "Opening NetCDF file in write mode failed.");
+            break;
+        case NetCDFOpenMode::CREATE:
+            read_only_ = false;
+            NC_CHECK(nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_), "Creating NetCDF file failed");
+            break;
+        default:
+            std::string err_msg = "Invalid open mode for NetCDFFile";
+            LOG(LogLevel::FATAL, err_msg);
+            throw std::runtime_error(err_msg);
     }
-    else{
-        read_only_ = true;
-        NC_CHECK(nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_), "Opening NetCDF file failed");
-    }
-//    }
     if(read_only_){
         load_variables(); //load all netcdf data to objects.
     } 

src/netcdf/NetCDFManager.cpp

@@ -10,10 +10,9 @@
 #include <NGenConfig.h>
 
 NetCDFManager::NetCDFManager(std::shared_ptr<realization::Formulation_Manager> manager, 
-        const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs)
+        const std::string& output_name, Simulation_Time const& sim_time, bool create_new_file, int mpi_rank, int mpi_num_procs)
+    : manager_{manager}, open_mode_{create_new_file ? NetCDFOpenMode::CREATE : NetCDFOpenMode::OPEN_WRITE}
 {
-    manager_ = manager;
-    sim_time_ = std::make_shared<Simulation_Time>(sim_time);
     std::filesystem::path check_path(output_name);
     if(check_path.is_absolute()){
         nc_filename_ = output_name;
@@ -47,23 +46,24 @@ NetCDFManager::NetCDFManager(std::shared_ptr<realization::Formulation_Manager> m
     }
     gather_all_catchments(catchments_in_proc);
     if (rank_ == 0){
-        nc_file_ = std::make_unique<NetCDFFile>(nc_filename_, true, is_mpi_);
-        define_catchment_netcdf_components();
+        nc_file_ = std::make_unique<NetCDFFile>(nc_filename_, this->open_mode_, is_mpi_);
+        if (create_new_file)
+            define_catchment_netcdf_components(sim_time);
     }
 #if NGEN_WITH_MPI
     if (comm_ != MPI_COMM_NULL) //This check is important if the user runs MPI with a single process.
         MPI_Barrier(comm_);
 #endif
 }
 
-NetCDFManager::NetCDFManager(const std::string& filename, bool read_only)
-    : read_only_(true)
+NetCDFManager::NetCDFManager(const std::string& filename, NetCDFOpenMode open_mode)
+    : open_mode_{open_mode}
 {
-    if (read_only_){
-        nc_file_ = std::make_unique<NetCDFFile>(filename, !read_only, false);
+    if (open_mode_ == NetCDFOpenMode::OPEN_READ) {
+        nc_file_ = std::make_unique<NetCDFFile>(filename, open_mode, false);
     }
     else{
-        throw std::runtime_error("Write only non-MPI function not implemented.");
+        throw std::runtime_error("Write non-MPI function not implemented.");
     }
 #if NGEN_WITH_MPI
     comm_ = MPI_COMM_NULL;
@@ -81,13 +81,13 @@ int NetCDFManager::create_file(const std::string& filename)
     if (num_procs_ > 1) {
         // MPI-enabled NetCDF
         is_mpi_ = true;
-        nc_file_ = std::make_unique<NetCDFFile>(filename, true, is_mpi_);
+        nc_file_ = std::make_unique<NetCDFFile>(filename, NetCDFOpenMode::CREATE, is_mpi_);
     }else{
         is_mpi_ = false;
     }
 #endif
     nc_filename_ = filename;
-    nc_file_ = std::make_unique<NetCDFFile>(nc_filename_, true, is_mpi_);
+    nc_file_ = std::make_unique<NetCDFFile>(nc_filename_, NetCDFOpenMode::CREATE, is_mpi_);
     if (!nc_file_) {
         throw std::runtime_error("Failed to create NetCDF file: " + filename);
     }
@@ -130,7 +130,7 @@ void NetCDFManager::gather_all_catchments(const std::vector<int64_t>& catchments
 #endif
 }
 
-void NetCDFManager::define_catchment_netcdf_components()
+void NetCDFManager::define_catchment_netcdf_components(Simulation_Time sim_time)
 {
     std::string name;
     int dim_id;
@@ -141,19 +141,19 @@ void NetCDFManager::define_catchment_netcdf_components()
     try
     {
         name = "time";
-        int num_timesteps = sim_time_->get_total_output_times();
+        int num_timesteps = sim_time.get_total_output_times();
         dim_id = add_dimension(name, num_timesteps);
         dim_ids = {dim_id};
         names = {name};
         add_variable(name, NC_INT, dim_ids, names);
 
         //add timestep values and attributes for time
         std::vector<int> time_epoch_seconds(num_timesteps);
-        time_epoch_seconds[0] = sim_time_->get_current_epoch_time();
+        time_epoch_seconds[0] = sim_time.get_current_epoch_time();
         for(int time_index = 1; time_index < num_timesteps; time_index++)
         {
-            sim_time_->advance_timestep();
-            time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time();
+            sim_time.advance_timestep();
+            time_epoch_seconds[time_index] = sim_time.get_current_epoch_time();
         }
         nc_file_->write_variable_data(name, time_epoch_seconds);
         nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00");
@@ -240,7 +240,7 @@ static std::vector<double> string_split(std::string str, char delimiter)
     return res;
 }
 
-void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, std::map<std::string, std::string> catchment_output_values)
+void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, const std::map<std::string, std::string>& catchment_output_values)
 {
     std::map<int64_t, std::string> output_values;
     try{

test/core/NetCDFCreatorTest.cpp

@@ -213,7 +213,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers)
 {
     std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc";
     std::string file_path = full_file_path.string();
-    nc_manager = std::make_unique<NetCDFManager>(manager_, file_path, *sim_time_, 0, 1);
+    nc_manager = std::make_unique<NetCDFManager>(manager_, file_path, *sim_time_, true, 0, 1);
     NetCDFFile* nc_file = nc_manager->get_file_handle();
     std::shared_ptr<NetCDFVar> catchments_var = nc_file->get_ncvar("catchments");
     size_t len = catchments_var->get_dim_size("catchments");
@@ -239,7 +239,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentOutputValues)
 {
     std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc";
     std::string file_path = full_file_path.string();
-    nc_manager = std::make_unique<NetCDFManager>(manager_, file_path, *sim_time_, 0, 1);
+    nc_manager = std::make_unique<NetCDFManager>(manager_, file_path, *sim_time_, true, 0, 1);
     NetCDFFile* nc_file = nc_manager->get_file_handle();
     std::map<std::string, std::string> catchment_output_values;
     auto c_form = std::dynamic_pointer_cast<realization::Catchment_Formulation>(manager_->get_formulation("cat-52"));