diff --git a/DESCRIPTION b/DESCRIPTION index 5c2be9f..618ec1d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: dpi Type: Package Title: Data Product Programming Interface -Version: 0.3.0 +Version: 0.4.0 Authors@R: c( person("Afshin", "Mashadi-Hossein", email = "amashadihossein@gmail.com", role = c("aut", "cre")), @@ -11,7 +11,7 @@ License: GPL-3 + file LICENSE Encoding: UTF-8 LazyData: true Roxygen: list(markdown = TRUE) -RoxygenNote: 7.3.2 +RoxygenNote: 7.3.3 Depends: R (>= 3.6.0) Imports: diff --git a/NEWS.md b/NEWS.md index f445587..76a6ce8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# dpi 0.4.0 + +* Add "prefix" option to `board_params_set_s3()` so that a subfolder can be used as a storage location for s3 daaps + # dpi 0.3.0 * Fully deprecated `board_alias` (#42) argument, which is deprecated in `pinsLabkey` v0.2.0 diff --git a/R/board_params_set_s3.R b/R/board_params_set_s3.R index b85eb7f..48fe6f4 100644 --- a/R/board_params_set_s3.R +++ b/R/board_params_set_s3.R @@ -1,27 +1,49 @@ #' @title Create formatted parameters that specify an s3 bucket board #' #' @description Build a data frame that contains all of the parameters -#' needed to specify an s3 bucket pin board. +#' needed to specify an s3 bucket pin board. The function validates the +#' provided parameters and returns a standardized data frame for use with +#' other daapr functions. #' #' @param bucket_name The name of the s3 bucket where the data product will be -#' stored. +#' stored. Must be a non-empty string. +#' @param prefix The prefix within this bucket where the data product will be +#' stored. Typically ends with `/` for S3 directory handling. Optional, defaults +#' to NULL. #' @param region The AWS region where the s3 bucket was created, e.g. -#' "us-east-1" or "us-west-1". +#' "us-east-1" or "us-west-1". Must be a non-empty string. The function will +#' check if the provided region is in the list of known AWS availability zones +#' and issue a warning if it's not. #' @param board_alias A short name for the board. -#' `r lifecycle::badge("deprecated")` this argument is deprecated with -#' pins ≥ 1.0. +#' `r lifecycle::badge("deprecated")` This argument is deprecated with +#' pins ≥ 1.0 and will be removed in a future version. Using this parameter +#' will result in an error. #' -#' @return A data.frame with class "s3_board" and a column for each param. +#' @return A data.frame with class "s3_board" and columns for: +#' \itemize{ +#' \item board_type: Always set to "s3_board" +#' \item bucket_name: The provided S3 bucket name +#' \item prefix: The provided prefix (or NULL if not specified) +#' \item region: The provided AWS region +#' } #' #' @examples #' \dontrun{ +#' # Basic usage with required parameters #' board_params_set_s3( -#' bucket_name = "bucket_name", +#' bucket_name = "my-bucket-name", +#' region = "us-east-1" +#' ) +#' +#' # With optional prefix parameter +#' board_params_set_s3( +#' bucket_name = "my-bucket-name", +#' prefix = "data-products/", #' region = "us-east-1" #' ) #' } #' @export -board_params_set_s3 <- function(bucket_name, region, board_alias = deprecated()) { +board_params_set_s3 <- function(bucket_name, prefix = NULL, region, board_alias = deprecated()) { if (lifecycle::is_present(board_alias)) { lifecycle::deprecate_stop("0.1.0", "board_params_set_s3(board_alias)", details = downgrade_message()) @@ -45,9 +67,20 @@ board_params_set_s3 <- function(bucket_name, region, board_alias = deprecated()) ))) } + # If prefix is provided, check that it ends with a trailing slash + if (!is.null(prefix) && nchar(prefix) > 0 && !endsWith(prefix, "/")) { + stop(cli::format_warning(glue::glue( + "prefix '{prefix}' must end with a trailing slash ", + "to avoid issues with S3 directory handling." + )), call. = FALSE) + } + + # Create the data frame with explicit lengths to avoid row count issues board_params <- data.frame( board_type = "s3_board", - bucket_name = bucket_name, region = region, + bucket_name = bucket_name, + prefix = if (is.null(prefix)) NA_character_ else prefix, + region = region, stringsAsFactors = FALSE ) diff --git a/R/dp_connect.R b/R/dp_connect.R index f1d90e8..9f02fad 100644 --- a/R/dp_connect.R +++ b/R/dp_connect.R @@ -43,9 +43,9 @@ dp_connect <- function(board_params, creds, ...) { #' @export dp_connect.s3_board <- function(board_params, creds, ...) { args <- list(...) - board_subdir <- "daap/" + board_subdir <- paste0(ifelse(is.na(board_params$prefix), "", board_params$prefix), "daap/") if (length(args$board_subdir) > 0) { - board_subdir <- args$board_subdir + board_subdir <- paste0(ifelse(is.na(board_params$prefix), "", board_params$prefix), args$board_subdir) } aws_creds <- creds diff --git a/man/board_params_set_s3.Rd b/man/board_params_set_s3.Rd index 8701694..cb414c5 100644 --- a/man/board_params_set_s3.Rd +++ b/man/board_params_set_s3.Rd @@ -4,30 +4,58 @@ \alias{board_params_set_s3} \title{Create formatted parameters that specify an s3 bucket board} \usage{ -board_params_set_s3(bucket_name, region, board_alias = deprecated()) +board_params_set_s3( + bucket_name, + prefix = NULL, + region, + board_alias = deprecated() +) } \arguments{ \item{bucket_name}{The name of the s3 bucket where the data product will be -stored.} +stored. Must be a non-empty string.} + +\item{prefix}{The prefix within this bucket where the data product will be +stored. Typically ends with \code{/} for S3 directory handling. Optional, defaults +to NULL.} \item{region}{The AWS region where the s3 bucket was created, e.g. -"us-east-1" or "us-west-1".} +"us-east-1" or "us-west-1". Must be a non-empty string. The function will +check if the provided region is in the list of known AWS availability zones +and issue a warning if it's not.} \item{board_alias}{A short name for the board. -\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#deprecated}{\figure{lifecycle-deprecated.svg}{options: alt='[Deprecated]'}}}{\strong{[Deprecated]}} this argument is deprecated with -pins ≥ 1.0.} +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#deprecated}{\figure{lifecycle-deprecated.svg}{options: alt='[Deprecated]'}}}{\strong{[Deprecated]}} This argument is deprecated with +pins ≥ 1.0 and will be removed in a future version. Using this parameter +will result in an error.} } \value{ -A data.frame with class "s3_board" and a column for each param. +A data.frame with class "s3_board" and columns for: +\itemize{ +\item board_type: Always set to "s3_board" +\item bucket_name: The provided S3 bucket name +\item prefix: The provided prefix (or NULL if not specified) +\item region: The provided AWS region +} } \description{ Build a data frame that contains all of the parameters -needed to specify an s3 bucket pin board. +needed to specify an s3 bucket pin board. The function validates the +provided parameters and returns a standardized data frame for use with +other daapr functions. } \examples{ \dontrun{ +# Basic usage with required parameters +board_params_set_s3( + bucket_name = "my-bucket-name", + region = "us-east-1" +) + +# With optional prefix parameter board_params_set_s3( - bucket_name = "bucket_name", + bucket_name = "my-bucket-name", + prefix = "data-products/", region = "us-east-1" ) } diff --git a/man/dpi-package.Rd b/man/dpi-package.Rd index 0521d3c..2db7a5b 100644 --- a/man/dpi-package.Rd +++ b/man/dpi-package.Rd @@ -6,6 +6,8 @@ \alias{dpi-package} \title{dpi: Data Product Programming Interface} \description{ +\if{html}{\figure{logo.png}{options: style='float: right' alt='logo' width='120'}} + This package provides an interface to data product. } \seealso{ diff --git a/tests/testthat/_snaps/board_params_set_s3.md b/tests/testthat/_snaps/board_params_set_s3.md index 80fe691..fd51572 100644 --- a/tests/testthat/_snaps/board_params_set_s3.md +++ b/tests/testthat/_snaps/board_params_set_s3.md @@ -3,6 +3,15 @@ Code board_params_set_s3(bucket_name = "bucket_name", region = "us-east-1") Output - board_type bucket_name region - 1 s3_board bucket_name us-east-1 + board_type bucket_name prefix region + 1 s3_board bucket_name us-east-1 + +# s3 board params are built properly with prefix + + Code + board_params_set_s3(bucket_name = "bucket_name", prefix = "testPrefix/", + region = "us-east-1") + Output + board_type bucket_name prefix region + 1 s3_board bucket_name testPrefix/ us-east-1 diff --git a/tests/testthat/test-board_params_set_s3.R b/tests/testthat/test-board_params_set_s3.R index 982222f..0d07e9d 100644 --- a/tests/testthat/test-board_params_set_s3.R +++ b/tests/testthat/test-board_params_set_s3.R @@ -3,6 +3,12 @@ test_that("s3 board params are built properly", { board_params_set_s3(bucket_name = "bucket_name", region = "us-east-1") ) }) + +test_that("s3 board params are built properly with prefix", { + expect_snapshot( + board_params_set_s3(bucket_name = "bucket_name", prefix = "testPrefix/", region = "us-east-1") + ) +}) test_that("board_alias raises a deprecation error", { expect_error( @@ -33,3 +39,19 @@ test_that("Empty region raises error", { ) }) +test_that("Prefix without trailing slash raises warning", { + expect_error( + board_params_set_s3(bucket_name = "bucket_name", prefix = "data-products", region = "us-east-1"), + regexp = "trailing slash" + ) + + # No warning when prefix has trailing slash + expect_no_error( + board_params_set_s3(bucket_name = "bucket_name", prefix = "data-products/", region = "us-east-1") + ) + + # No warning when prefix is NULL + expect_no_error( + board_params_set_s3(bucket_name = "bucket_name", prefix = NULL, region = "us-east-1") + ) +}) \ No newline at end of file