From e1d6957dcb5699fa1592dca263da94697003ae4f Mon Sep 17 00:00:00 2001 From: emmamendelsohn Date: Fri, 10 Nov 2023 15:25:54 -0500 Subject: [PATCH 1/6] draft of checks to spat4cast_submit() --- R/spat4cast_submit.R | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/R/spat4cast_submit.R b/R/spat4cast_submit.R index 4bb1551..454cfda 100644 --- a/R/spat4cast_submit.R +++ b/R/spat4cast_submit.R @@ -1,8 +1,22 @@ -spat4cast_submit <- function(dir, theme = "lai_recovery") { - # upload anonymously to submissions bucket +spat4cast_submit <- function(dir, theme = "lai_recovery", fire = "august_complex") { + + submitted_files <- list.files(dir, full = TRUE) + + # Check file type + assertthat::assert_that(n_distinct(tools::file_ext(submitted_files)) == 1) + assertthat::assert_that(unique(tools::file_ext(submitted_files)) == "tif", msg = "Error: The file extension in the directory is not 'tif'. Please check the file format.") + + # Check bounding box for files + ## can we use the `fire` argument to pull a template target file for comparison? + ## if we are to enforce these attributes, then we should modify `ingest_planetary_data` to remove dy, dx, dt, srs arguments (and have them enforced according to the fire of intered) + res(rast(submitted_files)) + origin(rast(submitted_files)) + ext(rast(submitted_files)) + + ## Upload anonymously to submissions bucket minioclient::mc_alias_set("efi", "data.ecoforecast.org", "", "") minioclient::mc_cp(dir, - paste0("efi/spat4cast-submissions/", theme), + paste0("efi/spat4cast-submissions/", theme, "/", fire), # should we have separate buckets for each fire? recursive = TRUE) } \ No newline at end of file From f23d69d8cfb9233c3e17435d776764d3512a38de Mon Sep 17 00:00:00 2001 From: emmamendelsohn Date: Fri, 10 Nov 2023 16:00:19 -0500 Subject: [PATCH 2/6] basic script to transfer from submissions to forecasts --- process_submissions.R | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 process_submissions.R diff --git a/process_submissions.R b/process_submissions.R new file mode 100644 index 0000000..1c5ebb1 --- /dev/null +++ b/process_submissions.R @@ -0,0 +1,5 @@ +## Send from submissions to forecasts +minioclient::mc_alias_set("efi", "data.ecoforecast.org", "", "") +minioclient::mc_cp(from = paste0("efi/spat4cast-submissions/", theme), + to = paste0("efi/spat4cast-forecasts/", theme), + recursive = TRUE) \ No newline at end of file From 049b75fbc5aaaf73c0a10f052811273857d986f4 Mon Sep 17 00:00:00 2001 From: emmamendelsohn Date: Tue, 14 Nov 2023 12:05:41 -0500 Subject: [PATCH 3/6] respond to comments from PR#13, including saving files in hive structure and adding mode_id. adding roxygen documentation. --- R/spat4cast_submit.R | 59 +++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 17 deletions(-) diff --git a/R/spat4cast_submit.R b/R/spat4cast_submit.R index 454cfda..b4fedd9 100644 --- a/R/spat4cast_submit.R +++ b/R/spat4cast_submit.R @@ -1,22 +1,47 @@ - -spat4cast_submit <- function(dir, theme = "lai_recovery", fire = "august_complex") { +#' Submit spatial forecast +#' +#' @param dir A directory of .tif files for scoring +#' @param model_id A unique ID representing your group and model +#' @param variable The data source being forecasted. Currently "lai_recovery" is the only option. +#' @param site_id The side ID of the area being forecasted. Currently "august_complex" is the only option. +#' +#' @return +#' @export +#' +#' @examples +exported <- function(x) { + x + 1 +} +spat4cast_submit <- function(dir, + model_id, + variable = c("lai_recovery"), + site_id = c("august_complex")) { - submitted_files <- list.files(dir, full = TRUE) + # Check variable and site_id + variable <- match.arg(variable) + site_id <- match.arg(site_id) # Check file type - assertthat::assert_that(n_distinct(tools::file_ext(submitted_files)) == 1) - assertthat::assert_that(unique(tools::file_ext(submitted_files)) == "tif", msg = "Error: The file extension in the directory is not 'tif'. Please check the file format.") + submitted_files <- list.files(dir, full = TRUE) + assertthat::assert_that(n_distinct(tools::file_ext(submitted_files)) == 1, msg = "Error: One or more file extension in the submitted directory is not 'tif'. Please check file formats.") + assertthat::assert_that(unique(tools::file_ext(submitted_files)) == "tif", msg = "Error: The file extension in the submitted directory is not 'tif'. Please check file formats.") + + # Check metadata for files + ## can we use the `variable` and `site_id` arguments to pull a template target file for comparison? + ## if we are to enforce these attributes, then we should modify `ingest_planetary_data` to remove dy, dx, dt, srs arguments (and have them enforced according to site_id) + submitted_files_rast <- rast(submitted_files) + res(submitted_files_rast) + origin(submitted_files_rast) + ext(submitted_files_rast) + crs(submitted_files_rast) + + # Upload anonymously to submissions bucket, using hive partitioning for file structure + out <- glue::glue("efi/spat4cast-submissions/variable={variable}/site_id={site_id}/model_id={model_id}") + + minioclient::mc_alias_set("efi", "data.ecoforecast.org", access_key = "", secret_key = "") # keys not needed for public bucket submission + submit <- minioclient::mc_cp(dir, out, recursive = TRUE) + + # Return submit message (or maybe something more succinct?) + return(submit) - # Check bounding box for files - ## can we use the `fire` argument to pull a template target file for comparison? - ## if we are to enforce these attributes, then we should modify `ingest_planetary_data` to remove dy, dx, dt, srs arguments (and have them enforced according to the fire of intered) - res(rast(submitted_files)) - origin(rast(submitted_files)) - ext(rast(submitted_files)) - - ## Upload anonymously to submissions bucket - minioclient::mc_alias_set("efi", "data.ecoforecast.org", "", "") - minioclient::mc_cp(dir, - paste0("efi/spat4cast-submissions/", theme, "/", fire), # should we have separate buckets for each fire? - recursive = TRUE) } \ No newline at end of file From b18e1ff231d9bcd9824110e65f417013bd314773 Mon Sep 17 00:00:00 2001 From: emmamendelsohn Date: Tue, 14 Nov 2023 12:12:15 -0500 Subject: [PATCH 4/6] move instead of copy submissions from top-level submissions bucket --- process_submissions.R | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/process_submissions.R b/process_submissions.R index 1c5ebb1..2270b53 100644 --- a/process_submissions.R +++ b/process_submissions.R @@ -1,5 +1,10 @@ ## Send from submissions to forecasts -minioclient::mc_alias_set("efi", "data.ecoforecast.org", "", "") -minioclient::mc_cp(from = paste0("efi/spat4cast-submissions/", theme), - to = paste0("efi/spat4cast-forecasts/", theme), - recursive = TRUE) \ No newline at end of file + +minioclient::mc_alias_set(alias = "efi", + endpoint = "data.ecoforecast.org", + access_key = Sys.getenv("AWS_ACCESS_KEY_ID"), + secret_key = Sys.getenv("AWS_SECRET_ACCESS_KEY")) + +transfer_output <- minioclient::mc_mv(from = "efi/spat4cast-submissions/", + to = "efi/spat4cast-forecasts/", + recursive = TRUE) From 69a356438ab0ac2388d29a57cd68c73423491dc0 Mon Sep 17 00:00:00 2001 From: emmamendelsohn Date: Tue, 14 Nov 2023 12:33:12 -0500 Subject: [PATCH 5/6] somewhat roundabout way to prevent copying directory name into hive structure --- R/spat4cast_submit.R | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/R/spat4cast_submit.R b/R/spat4cast_submit.R index b4fedd9..c335731 100644 --- a/R/spat4cast_submit.R +++ b/R/spat4cast_submit.R @@ -5,13 +5,11 @@ #' @param variable The data source being forecasted. Currently "lai_recovery" is the only option. #' @param site_id The side ID of the area being forecasted. Currently "august_complex" is the only option. #' -#' @return +#' @return message from minio submission #' @export #' -#' @examples -exported <- function(x) { - x + 1 -} +#' @examples spat4cast_submit(dir = "climatology", model_id = "team1-climatology", variable = "lai_recovery", site_id = "august_complex") + spat4cast_submit <- function(dir, model_id, variable = c("lai_recovery"), @@ -29,17 +27,19 @@ spat4cast_submit <- function(dir, # Check metadata for files ## can we use the `variable` and `site_id` arguments to pull a template target file for comparison? ## if we are to enforce these attributes, then we should modify `ingest_planetary_data` to remove dy, dx, dt, srs arguments (and have them enforced according to site_id) - submitted_files_rast <- rast(submitted_files) - res(submitted_files_rast) - origin(submitted_files_rast) - ext(submitted_files_rast) - crs(submitted_files_rast) + submitted_files_rast <- terra::rast(submitted_files) + terra::res(submitted_files_rast) + terra::origin(submitted_files_rast) + terra::ext(submitted_files_rast) + terra::crs(submitted_files_rast) # Upload anonymously to submissions bucket, using hive partitioning for file structure - out <- glue::glue("efi/spat4cast-submissions/variable={variable}/site_id={site_id}/model_id={model_id}") - minioclient::mc_alias_set("efi", "data.ecoforecast.org", access_key = "", secret_key = "") # keys not needed for public bucket submission - submit <- minioclient::mc_cp(dir, out, recursive = TRUE) + + file_names <- basename(submitted_files) + out <- glue::glue("efi/spat4cast-submissions/variable={variable}/site_id={site_id}/model_id={model_id}/") + + submit <- purrr::walk(file_names, ~withr::with_dir(dir, minioclient::mc_cp(from = ., out, recursive = TRUE))) # Return submit message (or maybe something more succinct?) return(submit) From 2b3f09e1eb2518742a7a156f9448767f607fddfc Mon Sep 17 00:00:00 2001 From: Carl Boettiger Date: Sun, 26 Nov 2023 09:53:53 -0800 Subject: [PATCH 6/6] Update spat4cast_submit.R add reference date as @rqthomas suggests --- R/spat4cast_submit.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/spat4cast_submit.R b/R/spat4cast_submit.R index c335731..be2943c 100644 --- a/R/spat4cast_submit.R +++ b/R/spat4cast_submit.R @@ -37,11 +37,11 @@ spat4cast_submit <- function(dir, minioclient::mc_alias_set("efi", "data.ecoforecast.org", access_key = "", secret_key = "") # keys not needed for public bucket submission file_names <- basename(submitted_files) - out <- glue::glue("efi/spat4cast-submissions/variable={variable}/site_id={site_id}/model_id={model_id}/") + out <- glue::glue("efi/spat4cast-submissions/variable={variable}/site_id={site_id}/model_id={model_id}/reference_date={reference_date}") submit <- purrr::walk(file_names, ~withr::with_dir(dir, minioclient::mc_cp(from = ., out, recursive = TRUE))) # Return submit message (or maybe something more succinct?) return(submit) -} \ No newline at end of file +}