Partial fixes for multisite ensembles

CHANGELOG.md

@@ -38,6 +38,7 @@ section for the next release.
 
 ### Changed
 
+- Ensemble and sensitivity analyses run with now assign an ensemble ID if one is not specified in the XML, even when running with no DB (#3654).
 - `download.ERA5_cds` now uses the R package ecmwfr (replacing python dependency of cdsapi via reticulate), enabling direct NetCDF downloads; and made flexible for both reanalysis and ensemble data product.
 - `extract_soil_gssurgo` now supports spatial sampling using a grid of user-defined size and spacing. And supports ensemble simulation of soil organic carbon (SOC) stocks, using area-weighted aggregation
 - The ERA5 NC extraction function can now handle multi-site instead of one

base/workflow/R/run.write.configs.R

@@ -164,7 +164,7 @@ run.write.configs <- function(settings, ensemble.size, input_design, write = TRU
   ### Sensitivity Analysis
   if ("sensitivity.analysis" %in% names(settings)) {
     ### Write out SA config files
-    PEcAn.logger::logger.info("\n ----- Writing model run config files ----")
+    PEcAn.logger::logger.info("\n ----- Writing model config files for sensitivity run ----")
     sa.runs <- PEcAn.uncertainty::write.sa.configs(
       defaults = settings$pfts,
       quantile.samples = sa.samples,

modules/uncertainty/NEWS.md

@@ -1,6 +1,12 @@
 # PEcAn.uncertainty 1.8.2
 
-* Plotting sensitivity now makes less noise in the console and once again produces a one-page PDF as intended.
+* Plotting sensitivity now makes less noise in the console and once again
+    produces a one-page PDF as intended.
+* `write.ensemble.configs` and `write.sa.configs` now generate an ensemble id
+    if one is not provided in a DB-free run.
+    Runs with DB continue to always generate a new id.
+    Note that multi-site runs with no id provided will now get a separate
+    ensemble ID (and thus generate separate analyses) for each site.
 * Documented that `runModule.run.sensitivity.analysis` does not yet work with
     multisite settings.
     This will be fixed in a future release.

modules/uncertainty/R/ensemble.R

@@ -217,37 +217,34 @@ get.ensemble.samples <- function( ensemble.size, pft.samples, env.samples,
 ##' @return list, containing $runs = data frame of runids, $ensemble.id = the ensemble ID for these runs and $samples with ids and samples used for each tag.  Also writes sensitivity analysis configuration files as a side effect
 ##' @details The restart functionality is developed using model specific functions by calling write_restart.modelname function. First, you need to make sure that this function is already exist for your desired model.See here \url{https://pecanproject.github.io/pecan-documentation/latest/pecan-models.html}
 ##' new state is a dataframe with a different column for each state variable. The number of the rows in this dataframe needs to be the same as the ensemble size.
-##' State variables that you can use for setting up the intial conditions differs for different models. You may check the documentation of the write_restart.modelname your model.
+##' State variables that you can use for setting up the initial conditions differs for different models. You may check the documentation of the write_restart.modelname your model.
 ##' The units for the state variables need to be in the PEcAn standard units which can be found in \link{standard_vars}.
 ##' new.params also has similar structure to ensemble.samples which is sent as an argument.
 ##'
 ##' @importFrom dplyr %>%
 ##' @importFrom rlang .data
 ##' @export
 ##' @author David LeBauer, Carl Davidson, Hamze Dokoohaki
-write.ensemble.configs <- function(input_design , ensemble.size, defaults, ensemble.samples, settings, model, 
+write.ensemble.configs <- function(input_design, ensemble.size, defaults, ensemble.samples, settings, model,
                                    clean = FALSE, write.to.db = TRUE, restart = NULL, samples = NULL, rename = FALSE) {
-
-
-  # Check if there are NO inputs
-
-for (input_tag in names(settings$run$inputs)) {
-  input <- settings$run$inputs[[input_tag]]
-  input_paths <- input$path
-
+
   # Check for required paths
-  if (is.null(input_paths) || length(input_paths) == 0) {
-     PEcAn.logger::logger.error("Input", sQuote(input_tag), "has no paths specified")
-  }
-
-  # Check for unsampled multi-path inputs
-  if (length(input_paths) > 1 && 
-     !(input_tag %in% names(settings$ensemble$samplingspace))) {
-    PEcAn.logger::logger.error(
-      "Input", sQuote(input_tag), "has", length(input_paths), "paths but no sampling method.",
-      "Add <samplingspace> for this input in pecan.xml")
+  for (input_tag in names(settings$run$inputs)) {
+    input <- settings$run$inputs[[input_tag]]
+    input_paths <- input$path
+    if (is.null(input_paths) || length(input_paths) == 0) {
+      PEcAn.logger::logger.error("Input", sQuote(input_tag), "has no paths specified")
+    }
+    # Check for unsampled multi-path inputs
+    if (length(input_paths) > 1 &&
+          !(input_tag %in% names(settings$ensemble$samplingspace))) {
+      PEcAn.logger::logger.error(
+        "Input", sQuote(input_tag), "has", length(input_paths),
+        "paths but no sampling method.",
+        "Add <samplingspace> for this input in pecan.xml"
+      )
+    }
   }
-}
 
 
 
@@ -281,14 +278,12 @@ for (input_tag in names(settings$run$inputs)) {
 
 
   # Get the workflow id
-  if (!is.null(settings$workflow$id)) {
-    workflow.id <- settings$workflow$id
-  } else {
-    workflow.id <- -1
-  }
+  workflow.id <- settings$workflow$id %||% -1
+
   #------------------------------------------------- if this is a new fresh run------------------  
   if (is.null(restart)){
     # create an ensemble id
+    # Note: this ignores any existing settings$ensemble$id
     if (!is.null(con) && write.to.db) {
       # write ensemble first
       ensemble.id <- PEcAn.DB::db.query(paste0(
@@ -302,7 +297,10 @@ for (input_tag in names(settings$run$inputs)) {
           "values (", pft$posteriorid, ", ", ensemble.id, ")"), con = con)
       }
     } else {
-      ensemble.id <- NA
+      # Use existing id if provided, or an arbitrary unique value if not
+      # Note: Since write.ensemble.configs is called separately for each site,
+      # a multisite run with no ID provided gives each site its own ensemble id!
+      ensemble.id <- settings$ensemble$id %||% rlang::hash(settings)
     }
     #-------------------------generating met/param/soil/veg/... for all ensembles----
     if (!is.null(con)){

modules/uncertainty/R/run.sensitivity.analysis.R

@@ -76,6 +76,9 @@ run.sensitivity.analysis <- function(settings,
   if (is.null(pfts)) {
     #extract just pft names
     pfts <- purrr::map_chr(settings$pfts, "name")
+    if (!is.null(settings$run$site$site.pft)) {
+      pfts <- pfts[pfts %in% settings$run$site$site.pft]
+    }
   } else {
     # validate pfts argument
     if (!is.character(pfts)) {
@@ -104,18 +107,12 @@ run.sensitivity.analysis <- function(settings,
 
     # Can specify ensemble ids manually. If not, look in settings.
     # If none there, will use the most recent, which was loaded with samples.Rdata
-    if (!is.null(ensemble.id)) {
-      fname <- sensitivity.filename(settings, "sensitivity.samples", "Rdata",
-                                    ensemble.id = ensemble.id,
-                                    all.var.yr = TRUE)
-    } else if (!is.null(settings$sensitivity.analysis$ensemble.id)) {
-      ensemble.id <- settings$sensitivity.analysis$ensemble.id
-      fname <- sensitivity.filename(settings, "sensitivity.samples", "Rdata",
-                                    ensemble.id = ensemble.id,
-                                    all.var.yr = TRUE)
-    } else {
-      ensemble.id <- NULL
-    }
+    ensemble.id <- ensemble.id %||%
+      settings$sensitivity.analysis$ensemble.id %||%
+      rlang::hash(settings)
+    fname <- sensitivity.filename(settings, "sensitivity.samples", "Rdata",
+                                  ensemble.id = ensemble.id,
+                                  all.var.yr = TRUE)
     if (file.exists(fname)) {
       load(fname, envir = samples)
     }

modules/uncertainty/R/sensitivity.R

@@ -162,6 +162,7 @@ write.sa.configs <- function(defaults, quantile.samples, settings, model,
   names(median.samples) <- names(quantile.samples)
 
   if (!is.null(con)) {
+    # Note: ignores any existing run or ensemble ids in settings
     ensemble.id <- PEcAn.DB::db.query(paste0(
       "INSERT INTO ensembles (runtype, workflow_id) ",
       "VALUES ('sensitivity analysis', ", format(workflow.id, scientific = FALSE), ") ",
@@ -206,8 +207,11 @@ write.sa.configs <- function(defaults, quantile.samples, settings, model,
       }
     }
   } else {
-    run.id <- PEcAn.utils::get.run.id("SA", "median")
-    ensemble.id <- NA
+    run.id <- PEcAn.utils::get.run.id("SA", "median", site.id = settings$run$site$id)
+    # Use SA ensemble id if provided, or an arbitrary unique value if not
+    # Note: Since write.sa.configs is called separately for each site,
+    # a multisite run with no ID provided gives each site its own ensemble id!
+    ensemble.id <- settings$sensitivity.analysis$ensemble.id %||% rlang::hash(settings)
   }
   medianrun <- run.id
 
@@ -339,7 +343,8 @@ write.sa.configs <- function(defaults, quantile.samples, settings, model,
               run.type = "SA",
               index = round(quantile, 3),
               trait = trait,
-              pft.name = names(trait.samples)[i]
+              pft.name = names(trait.samples)[i],
+              site.id = settings$run$site$id
             )
           }
           runs[[pftname]][quantile.str, trait] <- run.id