diff --git a/NAMESPACE b/NAMESPACE index cb9858f7..c70e3c3f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,11 +3,11 @@ export(crop_to_extent) export(download_cgiar_tables) export(download_flo1k_tables) -export(download_future_climate_tables) export(download_hydrography90m_tables) export(download_landcover_tables) export(download_merit_dem_tables) -export(download_present_climate_tables) +export(download_observed_climate_tables) +export(download_projected_climate_tables) export(download_soil_tables) export(download_test_data) export(download_tiles) diff --git a/R/download_env90m_tables.R b/R/download_env90m_tables.R index dd678ae0..52772e19 100644 --- a/R/download_env90m_tables.R +++ b/R/download_env90m_tables.R @@ -104,12 +104,12 @@ #' sub-catchment scale for freshwater biodiversity modeling, in prep. #' Please contact the authors for more up-to-date citation info. #' -#' @name download-env90m-tables +#' @name download_env90m_tables NULL -#' @describeIn download-env90m-tables Download SOILGRIDS tables +#' @describeIn download_env90m_tables Download SOILGRIDS tables #' (soilgrids250m_v2_0) #' @examples #' @@ -166,7 +166,7 @@ download_soil_tables <- function(subset = NULL, -#' @describeIn download-env90m-tables Download flow tables (flo1k_v1_0) +#' @describeIn download_env90m_tables Download flow tables (flo1k_v1_0) #' @examples #' #' ### Flow (mean): flo1k_v1_0 ### @@ -225,7 +225,7 @@ download_flo1k_tables <- function(subset = NULL, -#' @describeIn download-env90m-tables Download CGIAR-CSI tables (cgiar_csi_v3) +#' @describeIn download_env90m_tables Download CGIAR-CSI tables (cgiar_csi_v3) #' @examples #' #' ### CGIAR CSI dataset: cgiar_csi_v3 ### @@ -280,7 +280,7 @@ download_cgiar_tables <- function(subset = NULL, } -#' @describeIn download-env90m-tables Download MERIT-DEM tables +#' @describeIn download_env90m_tables Download MERIT-DEM tables #' (Multi-Error-Removed Improved-Terrain Digital Elevation Model, #' merit_dem_v1_0_3) #' @examples @@ -333,7 +333,7 @@ download_merit_dem_tables <- function(subset = NULL, return(download_simple_tables( "merit_dem_v1_0_3", - "env90m_merit_dem_paths_file_sizes.txt", + "env90m_meritdem_paths_file_sizes.txt", subset, tile_ids, ignore_missing, download, download_dir, file_format, delete_zips, tempdir, quiet )) @@ -341,7 +341,7 @@ download_merit_dem_tables <- function(subset = NULL, -#' @describeIn download-env90m-tables Download Hydrography90m tables +#' @describeIn download_env90m_tables Download Hydrography90m tables #' (hydrography90m_v1_0) #' @examples #' @@ -397,27 +397,26 @@ download_hydrography90m_tables <- function(subset = NULL, } -#' @describeIn download-env90m-tables Download CHELSA bioclimatic variables tables, -#' except for future simulations (Climatologies at high resolution for the +#' @describeIn download_env90m_tables Download CHELSA bioclimatic variables +#' tables, except for projections (Climatologies at high resolution for the #' earth’s land surface areas, chelsa_bioclim_v2_1) +#' #' @examples -#' #' ### Bioclimatic Variables: chelsa_bioclim_v2_1 ### -#' ### (excluding future simulations) ### -#' +#' ### (excluding projections) ### #' # Show all available bioclim variable names -#' # (excluding future simulations): -#' download_present_climate_tables() +#' # (excluding projections): +#' download_observed_climate_tables() #' #' # Compute download size of all bioclim variables, for one tile: -#' download_present_climate_tables( +#' download_observed_climate_tables( #' subset = "ALL", #' tile_ids = c("h00v04"), #' download = FALSE) #' #' # Download one bioclim variable (Annual mean temperature), for two tiles: #' \dontrun{ -#' download_present_climate_tables( +#' download_observed_climate_tables( #' subset = c("bio1"), #' tile_ids = c("h00v04", "h10v04"), #' download = TRUE, @@ -428,7 +427,7 @@ download_hydrography90m_tables <- function(subset = NULL, #' # Download one bioclim variable (Annual mean temperature), for one tile, #' # unzip, and delete the zips: #' \dontrun{ -#' download_present_climate_tables( +#' download_observed_climate_tables( #' subset = c("bio1"), #' tile_ids = c("h00v04"), #' download = TRUE, @@ -438,60 +437,60 @@ download_hydrography90m_tables <- function(subset = NULL, #' } #' #' @export -download_present_climate_tables <- function(subset = NULL, - tile_ids = NULL, - download = FALSE, - download_dir = ".", - file_format = "txt", - delete_zips = TRUE, - ignore_missing = FALSE, - tempdir = NULL, - quiet = FALSE) { +download_observed_climate_tables <- function(subset = NULL, + tile_ids = NULL, + download = FALSE, + download_dir = ".", + file_format = "txt", + delete_zips = TRUE, + ignore_missing = FALSE, + tempdir = NULL, + quiet = FALSE) { return(download_simple_tables( "chelsa_bioclim_v2_1", - "env90m_presentclimate_paths_file_sizes.txt", + "env90m_observedclimate_paths_file_sizes.txt", subset, tile_ids, ignore_missing, download, download_dir, file_format, delete_zips, tempdir, quiet )) } -#' @describeIn download-env90m-tables Download CHELSA bioclimatic variables tables, -#' future simulations only (Climatologies at high resolution for the +#' @describeIn download_env90m_tables Download CHELSA bioclimatic variables, +#' tables, projections only (Climatologies at high resolution for the #' earth’s land surface areas, chelsa_bioclim_v2_1) -#' @param base_vars (Only in `download_future_climate_tables()` and +#' @param base_vars (Only in `download_projected_climate_tables()` and #' `download_landcover_tables()`) Vector of the desired base variables, e.g. #' the landcover variable "c20_1992" can be expressed as base variable "c20" #' and year "1992". -#' @param time_periods (Only in `download_future_climate_...`) Vector of the +#' @param time_periods (Only in `download_projected_climate_...`) Vector of the #' desired time periods (leave `NULL` or specify `"ALL"` for all available #' time periods). -#' @param models (Only in `download_future_climate_...`) Vector of the desired +#' @param models (Only in `download_projected_climate_...`) Vector of the desired #' models (leave `NULL` or specify `"ALL"` for all available models). -#' @param scenarios (Only in `download_future_climate_...`) Vector of the +#' @param scenarios (Only in `download_projected_climate_...`) Vector of the #' desired scenarios (leave `NULL` or specify `"ALL"` for all available #' scenarios). -#' @param versions (Only in `download_future_climate_...`) Vector of the +#' @param versions (Only in `download_projected_climate_...`) Vector of the #' desired versions (leave `NULL` or specify `"ALL"` for all available #' versions). As of January 2025, the only available version is "V.2.1". #' #' @examples #' #' ### Bioclimatic Variables: chelsa_bioclim_v2_1 ### -#' ### (future simulations) ### -#' # Show all available future bioclim variable names -#' download_future_climate_tables() +#' ### (projections only) ### +#' # Show all available projected bioclim variable names +#' download_projected_climate_tables() #' #' # Compute download size of all variables, for one tile: -#' download_future_climate_tables( +#' download_projected_climate_tables( #' subset = "ALL", #' tile_ids = c("h00v04"), #' download = FALSE) #' -#' # Download one hy90m variable (Annual mean temperature), for two tiles: +#' # Download one variable (Annual mean temperature), for two tiles: #' \dontrun{ -#' download_future_climate_tables( +#' download_projected_climate_tables( #' subset = c("bio1"), #' tile_ids = c("h00v04", "h10v04"), #' download = TRUE, @@ -499,10 +498,10 @@ download_present_climate_tables <- function(subset = NULL, #' file_format = "zip") #' } #' -#' # Download one hy90m variable (Annual mean temperature), for one tile, +#' # Download one variable (Annual mean temperature), for one tile, #' # unzip, and delete the zips: #' \dontrun{ -#' download_future_climate_tables( +#' download_projected_climate_tables( #' subset = c("bio1"), #' tile_ids = c("h00v04"), #' download = TRUE, @@ -512,27 +511,27 @@ download_present_climate_tables <- function(subset = NULL, #' } #' #' @export -download_future_climate_tables <- function(base_vars = NULL, - time_periods = NULL, - models = NULL, - scenarios = NULL, - versions = NULL, - subset = NULL, - tile_ids = NULL, - download = FALSE, - download_dir = ".", - file_format = "txt", - delete_zips = TRUE, - ignore_missing = FALSE, - tempdir = NULL, - quiet = FALSE) { +download_projected_climate_tables <- function(base_vars = NULL, + time_periods = NULL, + models = NULL, + scenarios = NULL, + versions = NULL, + subset = NULL, + tile_ids = NULL, + download = FALSE, + download_dir = ".", + file_format = "txt", + delete_zips = TRUE, + ignore_missing = FALSE, + tempdir = NULL, + quiet = FALSE) { ######################################### ### Extract info from file size table ### ######################################### # Use file_size_table from tempdir or download it: - file_name <- "env90m_futureclimate_paths_file_sizes.txt" + file_name <- "env90m_projectedclimate_paths_file_sizes.txt" file_size_table <- get_file_size_table(file_name, tempdir, quiet) # Info: @@ -858,7 +857,7 @@ download_future_climate_tables <- function(base_vars = NULL, variables$tile_ids=tile_ids } - # Download the future climate data if requested: + # Download the projected climate data if requested: if (download) { if (is.null(tile_ids)) { msg <- "To download, please specify parameter 'tile_ids'." @@ -889,10 +888,10 @@ download_future_climate_tables <- function(base_vars = NULL, } -#' @describeIn download-env90m-tables Download ESA Land Cover tables +#' @describeIn download_env90m_tables Download ESA Land Cover tables #' (esa_cci_landcover_v2_1_1) #' -#' @param base_vars (Only in `download_future_climate_tables()` and +#' @param base_vars (Only in `download_projected_climate_tables()` and #' `download_landcover_tables()`) Vector of the desired base variables, e.g. #' the landcover variable "c20_1992" can be expressed as base variable "c20" #' and year "1992". diff --git a/R/get_predict_table.R b/R/get_predict_table.R index 50f19391..56c7e520 100644 --- a/R/get_predict_table.R +++ b/R/get_predict_table.R @@ -4,17 +4,9 @@ #' an specific subset of subcatchments. #' #' @param variable character vector of variable names. Possible values are: -#' c("bio1", "bio10", "bio11", "bio12", "bio13","bio14", "bio15", "bio16", -#' "bio17", "bio18","bio19", "bio2", "bio3", "bio4", "bio5","bio6", "bio7", -#' "bio8", "bio9", "c100", "c10", "c20", "c30", "c40", "c50","c60", "c70", -#' "c80", "c90", "chancurv","chandistdwseg", "chandistupcel", "chandistupseg", -#' "chanelvdwcel", "chanelvdwseg", "chanelvupcel","chanelvupseg", -#' "changraddwseg", "changradupcel", "changradupseg", "elev", "flow", -#' "flowpos", "gradient", "length", "out", "outdiffdwbasin", "outdiffdwscatch", -#' "outdistdwbasin", "outdistdwscatch", "outlet", "slopdiff", "slopgrad", -#' "soil", "source", "strdiffdwnear", "strdiffupfarth", "strdiffupnear", -#' "strdistdwnear", "strdistprox", "strdistupfarth", "strdistupnear", -#' "stright"). +#' all variables in the Env90m dataset, which can bew viewed by calling +#' 'download__tables()'. For more details, see +#' '?download_env90m_tables'. #' @param statistics character vector of statistics names. Possible values are #' "sd", "mean", "range" or "ALL". Default "ALL". #' @param tile_id character. The IDs of the tiles of interest. @@ -28,6 +20,8 @@ #' If FALSE, the table is only stored on disk. Default is TRUE. #' @param quiet logical. If FALSE, the standard output will be printed. #' Default is TRUE. +#' @param tempdir String. Path to the directory where to store/look for the +#' file size table. If not passed, defaults to the output of [base::tempdir()]. #' @importFrom processx run #' @importFrom stringi stri_rand_strings #' @importFrom stringi stri_rand_strings @@ -80,58 +74,50 @@ get_predict_table <- function(variable, out_file_path, n_cores = NULL, read = TRUE, - quiet = TRUE) { + quiet = TRUE, + tempdir = NULL) { + + # Define tempdir: + if (is.null(tempdir)) { + tempdir <- tempdir() + } - # Check variable name is one of the accepted values - # TODO replace this list by call to function get_env_vars... - accepted_vars <- c("bio1", "bio10", "bio11", "bio12", "bio13", - "bio14", "bio15", "bio16", "bio17", "bio18", - "bio19", "bio2", "bio3", "bio4", "bio5", - "bio6", "bio7", "bio8", "bio9", "c100", - "c10", "c20", "c30", "c40", "c50", - "c60", "c70", "c80", "c90", "c110", "c120", "c130", - "c140", "c150", "c160", "c170", "c180", "c190", "c200", - "c210", "c220", "chancurv","chandistdwseg", - "chandistupcel", "chandistupseg","chanelvdwcel", - "chanelvdwseg", "chanelvupcel","chanelvupseg", - "changraddwseg", "changradupcel","changradupseg", "elev", - "flow", "flowpos","gradient", "length", "out", - "outdiffdwbasin","outdiffdwscatch", "outdistdwbasin", - "outdistdwscatch","outlet", "slopdiff", "slopgrad", "soil", - "source","strdiffdwnear", "strdiffupfarth", "strdiffupnear", - "strdistdwnear", "strdistprox", "strdistupfarth", - "strdistupnear", "stright", "changradupcel", - "changradupseg", "elev_drop", "flow_accum", "gradient", - "length", "out_dist", "out_drop", "outdiffdwbasin", - "outlet_elev", "soil_ACDWRB", "soil_AWCtS", "soil_BDRICM", - "soil_BDRLOG", "soil_BLDFIE", "soil_CECSOL", "soil_CLYPPT", - "soil_CRFVOL", "soil_HISTPR", "soil_ORCDRC", "soil_PHIHOX", - "soil_SLGWRB", "soil_SLTPPT", "soil_SNDPPT", "soil_TEXMHT", - "soil_WWP", "source_elev", "strdistprox") #Check if one of the arguments is missing if (missing(variable)) stop(paste0('Variable is missing. Please provide at least the name of one variable. - Possible names are: ', paste0(accepted_vars, collapse=", "))) + You may use any of the >1000 variables of the Environment90m dataset,', + ' which you can view using e.g. download_soil_tables(). Please check', + ' ?download_env90m_tables for more details.')) if (missing(tile_id)) - stop("Please provide at least one tile ID") + stop("Please provide at least one tile ID (parameter \"tile_id\").") if (missing(input_var_path)) stop("Please provide a path to the table with environmental variables for - the entire tiles") + the entire tiles (parameter \"input_var_path\").") if (missing(subcatch_id)) - stop("Please provide at least one subcatchment ID") + stop("Please provide at least one subcatchment ID (parameter \"subcatch_id\").") if (missing(out_file_path)) - stop("Please provide a path to the output file") + stop("Please provide a path to the output file (parameter \"out_file_path\").") # Check if paths exists if (!file.exists(input_var_path)) stop(paste0("Path: ", input_var_path, " does not exist.")) - if (any(!variable %in% accepted_vars)) - stop("Please provide a valid variable name") + # Check if input tables exist: + # This only works if the files have this name pattern! + for (ivar in variable) { + for (itile in tile_id) { + ipath <- file.path(input_var_path, paste0(ivar, "_", itile, ".txt")) + if (file.exists(ipath)) { + if (!quiet) message("Input table exists: ", ipath) + } else { + stop(paste0("Input table: does not exist: ", ipath)) + } + } + } # Check if statistics name provided is one of the accepted values if (any(!(statistics %in% "ALL"))) { @@ -140,7 +126,6 @@ get_predict_table <- function(variable, sd, mean, range or ALL") } - # Check if n_cores is numeric if (!is.numeric(n_cores)) stop(paste0("n_cores: Has to be numeric.")) @@ -153,6 +138,33 @@ get_predict_table <- function(variable, if (!is.logical(read)) stop("read: Has to be TRUE or FALSE.") + # Now do the longer check, which needs to download some files first (unless they are + # already present in temp): + # Check variable name is one of the accepted values + #if (!quiet) message("Checking the variable names against the list(s) of allowed variable names...") + if (!quiet) message(paste("Downloading the list(s) of allowed variable names,", + "unless they were already downloaded to your temp directory...")) + # TODO: This download is potentially noisy. Make it (possibly) quiet? Really quiet? + accepted_vars <- c( + download_observed_climate_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_projected_climate_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_hydrography90m_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_soil_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_landcover_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_flo1k_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_cgiar_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names, + download_merit_dem_tables(download=FALSE, quiet=TRUE, tempdir=tempdir)$variable_names + ) + if (any(!variable %in% accepted_vars)) { + which_invalid <- variable[!variable %in% accepted_vars] + stop(paste0("These variables are not valid: ", + paste0(which_invalid, collapse=", "), + ". Please provide a valid variable name.", + "\nYou may use any of the >1000 variables of the Environment90m dataset,", + " which you can view using e.g. download_soil_tables(). Please check", + " ?download_env90m_tables for more details.")) + } + # Check operating system sys_os <- get_os() @@ -161,9 +173,11 @@ get_predict_table <- function(variable, tmp <- paste0("/tmp_", rand_string) # Path to the tmp directory + # TODO: Should we do this in the tempdir? tmp_dir <- paste0(getwd(), tmp) # Create temporary output directory + #if (!quiet) message("Creating temp directory for GRASS: ", tmp_dir) dir.create(tmp_dir, showWarnings = FALSE) # Format variable, statistics and tile_id vectors so they can be read in bash @@ -173,7 +187,7 @@ get_predict_table <- function(variable, # Make bash scripts executable make_sh_exec() - + if (!quiet) message("Running bash script...") if (sys_os == "linux" || sys_os == "osx") { # Call external bash script processx::run(system.file("sh", "get_predict_table.sh", @@ -186,7 +200,7 @@ get_predict_table <- function(variable, out_file_path, tmp_dir, n_cores), - echo = FALSE) + echo = !quiet) } else { # Check if WSL and Ubuntu are installed check_wsl() @@ -213,12 +227,14 @@ get_predict_table <- function(variable, wsl_sh_file)) } + if (!quiet) message("Running bash script: Done.") + # Delete temporary output directory unlink(tmp_dir, recursive = TRUE) if (read == TRUE) { - # Read predict table + if (!quiet) message("Reading result table from ", out_file_path) predict_table <- fread(out_file_path) return(predict_table) } diff --git a/inst/sh/get_predict_table.sh b/inst/sh/get_predict_table.sh index 3c195f63..98f7e129 100644 --- a/inst/sh/get_predict_table.sh +++ b/inst/sh/get_predict_table.sh @@ -32,6 +32,7 @@ VAR=( $(echo $1 | tr "/" "\n") ) [[ "${#VAR[@]}" -eq 1 ]] && var=($(echo $1)) || var=("${VAR[@]}") export var +echo "bash: Variables: "$var # select summary statistics # ALL @@ -39,31 +40,38 @@ export var ss=( $(echo $2 | tr "/" "\n") ) [[ "${#ss[@]}" -eq 1 ]] && SS=($(echo $2)) || SS=("${ss[@]}") export SS +echo "bash: Summary stats: "$SS # tiles of interest #export tiles=( h18v02 h18v04 h20v02 h20v04 ) TT=($(echo "$3" | tr "/" "\n")) [[ "${#TT[@]}" -eq 1 ]] && tiles=($(echo $3)) || tiles=("${TT[@]}") export tiles +echo "bash: Tiles: "$tiles -# path to environmental tables for each tile +# path to environmental tables for each tile export ENVTB=$4 +#echo "bash: Directory for input files: "$ENVTB # file with the list of subcatchments IDs export SUBCIDS=$5 #export SUBCIDS=/data/marquez/vignette/out/subc_IDs.txt +#echo "bash: Table containing subcatchment ids: "$SUBCIDS # output file export OUTFILE=$6 #export OUTFILE=/data/marquez/vignette/out/projectionTB2.csv +#echo "bash: Result will be written to: "$OUTFILE # folder to store temporal files: this folder is defined within the R function export TMP=$7 #export TMP=/data/marquez/vignette/out +#echo "bash: Temp directory: "$TMP # number of cores to run the extraction of information (rows) from tile tables # (n.tiles * n.variables) export NCORES=$8 +#echo "bash: Num cores for parallel computing: "$NCORES #export NCORES=16 @@ -84,16 +92,20 @@ subsetTB(){ } export -f subsetTB -parallel -j $NCORES subsetTB ::: ${tiles[@]} ::: ${var[@]} +echo "bash: Running in parallel: Treating tiles and variables. This will write tables 'ENV_...txt' into "$TMP +parallel -j $NCORES subsetTB ::: ${tiles[@]} ::: ${var[@]} +echo "bash: Running in parallel: Done." ################## # Subset the tables if user is only interested in few statistics (e.g., mean) if [[ "${SS[@]}" != 'ALL' ]] # run only if user do not select ALL then + echo "bash: Iterating over ENV_..." for TB in $(find $TMP -name "ENV_*.txt") do + echo "bash: Iteration: "$TB NR=$(awk 'NR == 1 {print NF}' $TB) [[ "$NR" != 6 ]] && continue @@ -119,7 +131,9 @@ then > $TMP/ENV_${RAND_STRING}.txt mv $TMP/ENV_${RAND_STRING}.txt $TB + #echo "bash: Moved: "$TMP/ENV_${RAND_STRING}.txt" to "$TB done + echo "bash: Iterating: Done." fi @@ -154,6 +168,7 @@ if [[ "${elem[@]}" -gt 0 ]]; then mv $TMP/aggreg_${X}_tmp2.txt $TMP/aggreg_${X}.txt fi +# Comment this to keep temp files: rm $TMP/aggreg_${X}_tmp*.txt ' _ @@ -197,13 +212,11 @@ fi ######################### -# remove temporal files +# remove temporal files (comment this to keep temp files): rm $TMP/aggreg* rm $TMP/ENV* rm $TMP/all_var_trim.csv rm $TMP/all_var_full.txt - - exit diff --git a/inst/test_data/env90m_observedclimate_paths_file_sizes.txt b/inst/test_data/env90m_observedclimate_paths_file_sizes.txt new file mode 100644 index 00000000..38363aa1 --- /dev/null +++ b/inst/test_data/env90m_observedclimate_paths_file_sizes.txt @@ -0,0 +1,4 @@ +file_path;file_id;file_size +chelsa_bioclim_v2_1/1981-2010_observed/bio17/bio17_h10v04.zip;notneeded_legacy;161 +chelsa_bioclim_v2_1/1981-2010_observed/bio17/bio17_h10v06.zip;notneeded_legacy;110526834 +chelsa_bioclim_v2_1/1981-2010_observed/bio17/bio17_h10v08.zip;notneeded_legacy;404897845 diff --git a/inst/test_data/env90m_projectedclimate_paths_file_sizes.txt b/inst/test_data/env90m_projectedclimate_paths_file_sizes.txt new file mode 100644 index 00000000..1ea34a80 --- /dev/null +++ b/inst/test_data/env90m_projectedclimate_paths_file_sizes.txt @@ -0,0 +1,4 @@ +file_path;file_id;file_size +chelsa_bioclim_v2_1/2071-2100_projected/bio11/bio11_2071-2100_ipsl-cm6a-lr_ssp370_V.2.1_h12v00.zip;notneeded_legacy;54529544 +chelsa_bioclim_v2_1/2071-2100_projected/bio11/bio11_2071-2100_ipsl-cm6a-lr_ssp370_V.2.1_h12v02.zip;notneeded_legacy;18035725 +chelsa_bioclim_v2_1/2071-2100_projected/bio11/bio11_2071-2100_ipsl-cm6a-lr_ssp370_V.2.1_h12v04.zip;notneeded_legacy;628 diff --git a/inst/test_data/hydrography90m_paths_file_sizes.txt b/inst/test_data/hydrography90m_paths_file_sizes.txt new file mode 100644 index 00000000..2d6f3bee --- /dev/null +++ b/inst/test_data/hydrography90m_paths_file_sizes.txt @@ -0,0 +1,18 @@ +file_path;file_id;file_size +README.txt;1a8chlzeTFdis2rfapX8JcveazoxzlEpW;21666 +Regular_tiles_basins.pdf;1ZccvYXy-acYzZaFmw_vXcW-ZMLcJafag;2191916 +global/stream_dist_dw_near_ovr.tif;1k3tEopzCAbY_NO6c1dl_O_gfkj-0cZ2r;13301240047 +global/order_topo_ovr.tif;1zoGjZzmBblvNYQELMTIU5TLLpWF5syW-;6125520067 +global/order_strahler_ovr.tif;1KWoe__JizH5CZI8LUeTEcoS0pEzNj8f8;3092207698 +r.stream.order/order_topo_tiles20d/order_topo_h18v06.tif;1gWGY_XbQnPFSy-0x3LXhfrctycOXED-T;242328629 +r.stream.order/order_topo_tiles20d/order_topo_h20v06.tif;16IoDckMf4tbeFayAWy1-e_wk0MUZYx_b;234857568 +r.stream.order/order_topo_tiles20d/order_topo_h30v10.tif;1TXyRV1ehYc_LP8vdAXBJ6WqyVIgrSiQp;203637690 +r.stream.order/order_topo_tiles20d/order_topo_h16v02.tif;1FBhfu9l218QJx4pO-eIWaNgsWnxxR4FT;27330052 +r.stream.order/order_topo_tiles20d/order_topo_h20v08.tif;1IirP6C8WB8zhbxa9FfO4nXHyktEgccLZ;237232764 +r.stream.order/order_topo_tiles20d/order_topo_h26v04.tif;1JnTzBhWC-fwXuqRRLWL6Rk3ZcM2enzQu;202099042 +r.stream.order/order_vect_tiles20d/order_vect_segment_h12v02.gpkg;1-CQQIcokcKZ5FABCa5VK9NFEDaN3rCLR;900820992 +r.stream.order/order_vect_tiles20d/order_vect_segment_h32v12.gpkg;10cnDllxtrgLXST2mAYxuDhsNIhMil_Jj;721108992 +r.stream.order/order_vect_tiles20d/order_vect_segment_h16v02.gpkg;1nwC_o5z8ikDKyy2S2yCvbScsQki8gDuR;813932544 +r.watershed/regional_unit/regional_unit_9.tif;1JsRDPdIbSLE5BxV9x9r29dx8FzpI920A;1561224 +r.watershed/regional_unit/regional_unit_99.tif;1rC87TJ36HAr3dIlIt4PQSG0w3xkeQmJb;2925608 +r.watershed/regional_unit/regional_unit_190.tif;1S0_nAmFHVn4KMMeWyhsU79uQW-TTXa4h;780854 diff --git a/man/download-env90m-tables.Rd b/man/download_env90m_tables.Rd similarity index 88% rename from man/download-env90m-tables.Rd rename to man/download_env90m_tables.Rd index 6f187062..fae107f1 100644 --- a/man/download-env90m-tables.Rd +++ b/man/download_env90m_tables.Rd @@ -1,14 +1,14 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/download_env90m_tables.R -\name{download-env90m-tables} -\alias{download-env90m-tables} +\name{download_env90m_tables} +\alias{download_env90m_tables} \alias{download_soil_tables} \alias{download_flo1k_tables} \alias{download_cgiar_tables} \alias{download_merit_dem_tables} \alias{download_hydrography90m_tables} -\alias{download_present_climate_tables} -\alias{download_future_climate_tables} +\alias{download_observed_climate_tables} +\alias{download_projected_climate_tables} \alias{download_landcover_tables} \title{Download Environment90m tables} \usage{ @@ -72,7 +72,7 @@ download_hydrography90m_tables( quiet = FALSE ) -download_present_climate_tables( +download_observed_climate_tables( subset = NULL, tile_ids = NULL, download = FALSE, @@ -84,7 +84,7 @@ download_present_climate_tables( quiet = FALSE ) -download_future_climate_tables( +download_projected_climate_tables( base_vars = NULL, time_periods = NULL, models = NULL, @@ -156,23 +156,23 @@ functions. If not passed, defaults to the output of [base::tempdir()].} \item{quiet}{logical. If FALSE, informative messages will be printed. Default is FALSE.} -\item{base_vars}{(Only in `download_future_climate_tables()` and +\item{base_vars}{(Only in `download_projected_climate_tables()` and `download_landcover_tables()`) Vector of the desired base variables, e.g. the landcover variable "c20_1992" can be expressed as base variable "c20" and year "1992".} -\item{time_periods}{(Only in `download_future_climate_...`) Vector of the +\item{time_periods}{(Only in `download_projected_climate_...`) Vector of the desired time periods (leave `NULL` or specify `"ALL"` for all available time periods).} -\item{models}{(Only in `download_future_climate_...`) Vector of the desired +\item{models}{(Only in `download_projected_climate_...`) Vector of the desired models (leave `NULL` or specify `"ALL"` for all available models).} -\item{scenarios}{(Only in `download_future_climate_...`) Vector of the +\item{scenarios}{(Only in `download_projected_climate_...`) Vector of the desired scenarios (leave `NULL` or specify `"ALL"` for all available scenarios).} -\item{versions}{(Only in `download_future_climate_...`) Vector of the +\item{versions}{(Only in `download_projected_climate_...`) Vector of the desired versions (leave `NULL` or specify `"ALL"` for all available versions). As of January 2025, the only available version is "V.2.1".} @@ -253,12 +253,12 @@ merit_dem_v1_0_3) \item \code{download_hydrography90m_tables()}: Download Hydrography90m tables (hydrography90m_v1_0) -\item \code{download_present_climate_tables()}: Download CHELSA bioclimatic variables tables, -except for future simulations (Climatologies at high resolution for the +\item \code{download_observed_climate_tables()}: Download CHELSA bioclimatic variables +tables, except for projections (Climatologies at high resolution for the earth’s land surface areas, chelsa_bioclim_v2_1) -\item \code{download_future_climate_tables()}: Download CHELSA bioclimatic variables tables, -future simulations only (Climatologies at high resolution for the +\item \code{download_projected_climate_tables()}: Download CHELSA bioclimatic variables, +tables, projections only (Climatologies at high resolution for the earth’s land surface areas, chelsa_bioclim_v2_1) \item \code{download_landcover_tables()}: Download ESA Land Cover tables @@ -278,15 +278,18 @@ download_soil_tables( download = FALSE) # Download one soil variable (Clay content), for two tiles: +\dontrun{ download_soil_tables( subset = c("clyppt"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} # Download one soil variable (Clay content), for one tile, # unzip, and delete the zips: +\dontrun{ download_soil_tables( subset = c("clyppt"), tile_ids = c("h00v04"), @@ -294,6 +297,7 @@ download_soil_tables( download_dir = ".", file_format = "txt", delete_zips = TRUE) +} ### Flow (mean): flo1k_v1_0 ### @@ -302,21 +306,26 @@ download_flo1k_tables() # Compute download size of the only flo1k_v1_0 variable (mean flow), # for one tile: +\dontrun{ download_flo1k_tables( subset = "ALL", tile_ids = c("h00v04"), download = FALSE) +} # Download the only flo1k_v1_0 variable (flo1k), for two tiles: +\dontrun{ download_soil_tables( subset = c("flo1k"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} -# Download the only flo1k_v1_0 variable (flo1k), for one tile: +# Download the only flo1k_v1_0 variable (flo1k), for one tile, # unzip, and delete the zips: +\dontrun{ download_soil_tables( subset = c("flo1k"), tile_ids = c("h00v04"), @@ -324,6 +333,7 @@ download_soil_tables( download_dir = ".", file_format = "txt", delete_zips = TRUE) +} ### CGIAR CSI dataset: cgiar_csi_v3 ### @@ -338,14 +348,17 @@ download_cgiar_tables( # Download one cgiar variable (Global Aridity Index), for two tiles: download_cgiar_tables( +\dontrun{ subset = c("garid"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} # Download two cgiar variables (Global Aridity Index, Potential # Evapotranspiration), for one tile, unzip, and delete the zips: +\dontrun{ download_cgiar_tables( subset = c("garid", "gevapt"), tile_ids = c("h00v04"), @@ -353,6 +366,7 @@ download_cgiar_tables( download_dir = ".", file_format = "txt", delete_zips = TRUE) +} ### Digital Elevation Model: merit_dem_v1_0_3 ### @@ -369,15 +383,18 @@ download_merit_dem_tables( # Download the only merit_dem_v1_0_3 variable (mean elevation), # for two tiles: +\dontrun{ download_merit_dem_tables( subset = c("elev"), # or "ALL" tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} # Download the only merit_dem_v1_0_3 variable (mean elevation), # for one tile, unzip, and delete the zips: +\dontrun{ download_merit_dem_tables( subset = c("elev"), tile_ids = c("h00v04"), @@ -385,6 +402,7 @@ download_merit_dem_tables( download_dir = ".", file_format = "txt", delete_zips = TRUE) +} ### Hydrography90m: hydrography90m_v1_0 ### @@ -398,15 +416,18 @@ download_hydrography90m_tables( download = FALSE) # Download one hy90m variable (Strahler’s stream order), for two tiles: +\dontrun{ download_hydrography90m_tables( subset = c("stream_strahler"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} # Download one hy90m variable (Strahler’s stream order), for one tile, # unzip, and delete the zips: +\dontrun{ download_hydrography90m_tables( subset = c("stream_strahler"), tile_ids = c("h00v04"), @@ -414,67 +435,75 @@ download_hydrography90m_tables( download_dir = ".", file_format = "txt", delete_zips = TRUE) - +} ### Bioclimatic Variables: chelsa_bioclim_v2_1 ### -### (excluding future simulations) ### +### (excluding projections) ### # Show all available bioclim variable names -# (excluding future simulations): -download_present_climate_tables() +# (excluding projections): +download_observed_climate_tables() # Compute download size of all bioclim variables, for one tile: -download_present_climate_tables( +download_observed_climate_tables( subset = "ALL", tile_ids = c("h00v04"), download = FALSE) # Download one bioclim variable (Annual mean temperature), for two tiles: -download_present_climate_tables( +\dontrun{ +download_observed_climate_tables( subset = c("bio1"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} # Download one bioclim variable (Annual mean temperature), for one tile, # unzip, and delete the zips: -download_present_climate_tables( +\dontrun{ +download_observed_climate_tables( subset = c("bio1"), tile_ids = c("h00v04"), download = TRUE, download_dir = ".", file_format = "txt", - delete_zips = TRUE) + delete_zips = TRUE) +} ### Bioclimatic Variables: chelsa_bioclim_v2_1 ### -### (future simulations) ### -# Show all available future bioclim variable names -download_future_climate_tables() +### (projections only) ### +# Show all available projected bioclim variable names +download_projected_climate_tables() # Compute download size of all variables, for one tile: -download_future_climate_tables( +download_projected_climate_tables( subset = "ALL", tile_ids = c("h00v04"), download = FALSE) -# Download one hy90m variable (Annual mean temperature), for two tiles: -download_future_climate_tables( +# Download one variable (Annual mean temperature), for two tiles: +\dontrun{ +download_projected_climate_tables( subset = c("bio1"), tile_ids = c("h00v04", "h10v04"), download = TRUE, download_dir = ".", file_format = "zip") +} -# Download one hy90m variable (Annual mean temperature), for one tile, +# Download one variable (Annual mean temperature), for one tile, # unzip, and delete the zips: -download_future_climate_tables( +\dontrun{ +download_projected_climate_tables( subset = c("bio1"), tile_ids = c("h00v04"), download = TRUE, download_dir = ".", file_format = "txt", delete_zips = TRUE) +} ### Landcover: esa_cci_landcover_v2_1_1 ### @@ -489,6 +518,7 @@ download_landcover_tables() tile_ids="ALL") # Download two base variables and one year, for two tiles: +\dontrun{ vars <- download_landcover_tables( base_vars=c("c10", "c130"), years=c(1992), @@ -497,6 +527,7 @@ download_landcover_tables() download_dir="/tmp", file_format="zip", delete_zips=FALSE) +} } \references{ diff --git a/tests/testthat/test-check_tiles_filesize.R b/tests/testthat/test-check_tiles_filesize.R index e55b4e55..c121fdfc 100644 --- a/tests/testthat/test-check_tiles_filesize.R +++ b/tests/testthat/test-check_tiles_filesize.R @@ -20,9 +20,10 @@ ######################### tests_quiet=TRUE - -# NOT COMMIT: -tests_quiet=FALSE +use_dummy_tables=TRUE +# TODO: After testing, the dummy tables are in the temp dir and might +# affect later function runs where the real tables are needed. Remove +# them? Or put in separate test dir? # Where to store and download files: if (! exists("tmpdir")){ @@ -39,12 +40,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -74,12 +76,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -109,12 +112,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -144,12 +148,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -179,12 +184,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -215,12 +221,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -250,12 +257,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -289,12 +297,13 @@ test_that(testname, { # before even checking whether 140 exists (hint: it doesn't...) # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -324,12 +333,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -360,12 +370,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -393,12 +404,13 @@ if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - # Get file size table (code from: download_tiles.R) + # Get file size table from inst/test_data, or from the server: fname <- 'hydrography90m_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/agciopgzXjWswF4/download?path=%2FREADME/" - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -423,20 +435,20 @@ test_that(testname, { }) -testname = "9.1: Test with another file size file for the env90m case (present climate)" +testname = "9.1: Test with another file size file for the env90m case (observedclimate)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Prepare: - # Get file size table (code from: download_tiles.R) - fname <- 'env90m_presentclimate_paths_file_sizes.txt' + # Get file size table from inst/test_data, or from the server: + fname <- 'env90m_observedclimate_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/zw56kEd25NsQqcQ/download?path=%2FREADME/" # Note: base_url not the same as for hydrography90m! - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") @@ -448,7 +460,7 @@ test_that(testname, { # Run: bytes <- check_tiles_filesize( - 'bio5', + 'bio17', file_format = "zip", tile_id = "h10v04", h90m_varnames = all_varnames, @@ -457,21 +469,23 @@ test_that(testname, { file_size_table = file_size_table) # Check: - expect_equal(bytes, 159) + expect_equal(bytes, 161) }) -testname = "9.2: Test with another file size file for the env90m case (future climate)" +testname = "9.2: Test with another file size file for the env90m case (projectedclimate)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Prepare: - fname <- 'env90m_futureclimate_paths_file_sizes.txt' + # Get file size table from inst/test_data, or from the server: + fname <- 'env90m_projectedclimate_paths_file_sizes.txt' file_size_file <- file.path(tmpdir, fname) - if (!(file.exists(file_size_file))) { + if (use_dummy_tables) { + file.copy(system.file(paste0("test_data/", fname), package="hydrographr"), tmpdir, overwrite=FALSE) + } else if (!(file.exists(file_size_file))) { base_url <- "https://public.igb-berlin.de/index.php/s/zw56kEd25NsQqcQ/download?path=%2FREADME/" # Note: base_url not the same as for hydrography90m! - skip_if_offline("public.igb-berlin.de") download.file(paste0(base_url, fname), destfile = file_size_file, mode = "wb") } file_size_table <- fread(file_size_file, sep = ";") diff --git a/tests/testthat/test-download_env90m_tables.R b/tests/testthat/test-download_env90m_tables.R index 18fbbe43..b7b23b2a 100644 --- a/tests/testthat/test-download_env90m_tables.R +++ b/tests/testthat/test-download_env90m_tables.R @@ -36,7 +36,7 @@ test_that(testname, { # Run: tab <- get_file_size_table( - file_name="env90m_presentclimate_paths_file_sizes.txt", + file_name="env90m_observedclimate_paths_file_sizes.txt", quiet=tests_quiet) # Check whether the specified table was loaded @@ -496,19 +496,19 @@ test_that(testname, { }) -testname = "3.1 futureclimate: show variable names (all)" +testname = "3.1 projectedclimate: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run three times, each should yield the same result! - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( quiet=FALSE) - vars2 <- download_future_climate_tables( + vars2 <- download_projected_climate_tables( quiet=FALSE, subset="ALL") - vars3 <- download_future_climate_tables( + vars3 <- download_projected_climate_tables( quiet=FALSE, time_periods="ALL", scenarios="ALL", @@ -539,7 +539,7 @@ test_that(testname, { }) -testname = "3.2.1 futureclimate: show variable names (of a subset, specified: subset)" +testname = "3.2.1 projectedclimate: show variable names (of a subset, specified: subset)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -547,7 +547,7 @@ test_that(testname, { mysubset <- c("bio1_2041-2070_ipsl-cm6a-lr_ssp126_V.2.1", "bio1_2071-2100_ipsl-cm6a-lr_ssp126_V.2.1") # Run: - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( quiet=FALSE, subset=mysubset) @@ -557,7 +557,7 @@ test_that(testname, { }) -testname = "3.2.2 futureclimate: show variable names (of a subset, specified: various components)" +testname = "3.2.2 projectedclimate: show variable names (of a subset, specified: various components)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -569,7 +569,7 @@ test_that(testname, { myversions <- c("V.2.1") # Run: - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( quiet=FALSE, base_vars=mybasevars, models=mymodels, @@ -579,7 +579,7 @@ test_that(testname, { # Run another time, this time, two components are specified "ALL", # which should lead to the same result! - vars2 <- download_future_climate_tables( + vars2 <- download_projected_climate_tables( quiet=FALSE, base_vars=mybasevars, models=mymodels, @@ -600,7 +600,7 @@ test_that(testname, { }) -testname = "3.3.1 futureclimate: show download size (specified: subset / various components)" +testname = "3.3.1 projectedclimate: show download size (specified: subset / various components)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -608,13 +608,13 @@ test_that(testname, { mytiles <- c("h02v02", "h04v02") # Run: - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( quiet=FALSE, tile_ids=mytiles, subset=c("bio1_2041-2070_ipsl-cm6a-lr_ssp126_V.2.1", "bio1_2071-2100_ipsl-cm6a-lr_ssp126_V.2.1")) # Run (same subset, specified using separate base_vars and years) - vars2 <- download_future_climate_tables( + vars2 <- download_projected_climate_tables( quiet=FALSE, tile_ids=mytiles, base_vars=c("bio1"), @@ -633,12 +633,12 @@ test_that(testname, { }) -testname = "3.3.2 futureclimate: show download size (test passing subset=ALL)" +testname = "3.3.2 projectedclimate: show download size (test passing subset=ALL)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( subset="ALL", tile_ids=c("h02v02"), quiet=FALSE) @@ -650,12 +650,12 @@ test_that(testname, { }) -testname = "3.3.3 futureclimate: show download size (test passing tile_ids=ALL)" +testname = "3.3.3 projectedclimate: show download size (test passing tile_ids=ALL)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( subset=c("bio1_2041-2070_ipsl-cm6a-lr_ssp126_V.2.1"), tile_ids="ALL", quiet=FALSE) @@ -667,13 +667,13 @@ test_that(testname, { }) -testname = "3.4.1 futureclimate: download subset" +testname = "3.4.1 projectedclimate: download subset" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: skip_if(R_SKIP_DOWNLOAD, 'R_SKIP_DOWNLOAD: This test downloads 68 MB, so we skip it.') - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( tile_ids=c("h02v02"), subset=c("bio1_2041-2070_ipsl-cm6a-lr_ssp126_V.2.1", "bio1_2071-2100_ipsl-cm6a-lr_ssp126_V.2.1"), download=TRUE, @@ -693,15 +693,15 @@ test_that(testname, { }) -testname = "3.4.2 futureclimate: download all variables (for one tile)" +testname = "3.4.2 projectedclimate: download all variables (for one tile)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - skip_if(R_SKIP_HUGE_DOWNLOAD, "R_SKIP_HUGE_DOWNLOAD: This test download 12.9 GB, so we skip it.") + skip_if(R_SKIP_HUGE_DOWNLOAD, "R_SKIP_HUGE_DOWNLOAD: This test downloads 12.9 GB, so we skip it.") # Note: Test 3.3.2 (computing download size of subset=ALL) tests quite similar behaviour, # just without the actual download in the end. - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( subset="ALL", tile_ids=c("h10v04"), download=TRUE, @@ -716,15 +716,15 @@ test_that(testname, { }) -testname = "3.4.3 futureclimate: download all tiles (for one variable)" +testname = "3.4.3 projectedclimate: download all tiles (for one variable)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - skip_if(R_SKIP_HUGE_DOWNLOAD, "R_SKIP_HUGE_DOWNLOAD: This test download 10.5 GB, so we skip it.") + skip_if(R_SKIP_HUGE_DOWNLOAD, "R_SKIP_HUGE_DOWNLOAD: This test downloads 10.5 GB, so we skip it.") # Note: Test 3.3.3 (computing download size of tile_ids=ALL) tests quite similar behaviour, # just without the actual download in the end. - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( subset=c("bio1_2041-2070_ipsl-cm6a-lr_ssp126_V.2.1", "bio1_2071-2100_ipsl-cm6a-lr_ssp126_V.2.1"), tile_ids="ALL", download=TRUE, @@ -739,7 +739,7 @@ test_that(testname, { }) -testname = "3.5.1 futureclimate: failure: model not available" +testname = "3.5.1 projectedclimate: failure: model not available" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -748,7 +748,7 @@ test_that(testname, { # Run: expect_error( - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( models=c("ipsl-cm6a-lr", "xyz"), base_vars=c("bio1"), time_periods=c("2041-2070", "2071-2100"), @@ -761,7 +761,7 @@ test_that(testname, { }) -testname = "3.5.2 futureclimate: failure: scenarios not available" +testname = "3.5.2 projectedclimate: failure: scenarios not available" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -770,7 +770,7 @@ test_that(testname, { # Run: expect_error( - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( scenarios=c("ssp126", "xyz", "abc"), base_vars=c("bio1"), models=c("ipsl-cm6a-lr"), @@ -783,7 +783,7 @@ test_that(testname, { }) -testname = "3.5.3 futureclimate: failure: version not available" +testname = "3.5.3 projectedclimate: failure: version not available" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -792,7 +792,7 @@ test_that(testname, { # Run: expect_error( - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( versions=c("V.2.1", "xyz"), base_vars=c("bio1"), models=c("ipsl-cm6a-lr"), @@ -805,7 +805,7 @@ test_that(testname, { }) -testname = "3.5.4 futureclimate: failure: base_var not available" +testname = "3.5.4 projectedclimate: failure: base_var not available" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -821,7 +821,7 @@ test_that(testname, { # Run: expect_error( - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( tempdir=tmpdir, quiet=FALSE, base_vars=c("bio1", "xyz", "abc", "def"), @@ -833,7 +833,7 @@ test_that(testname, { ) }) -testname = "3.5.5 futureclimate: failure: no subset passed" +testname = "3.5.5 projectedclimate: failure: no subset passed" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -842,7 +842,7 @@ test_that(testname, { # Run: expect_warning( - vars <- download_future_climate_tables( + vars <- download_projected_climate_tables( tile_ids=c("h02v02", "h04v02"), download=TRUE, tempdir=tmpdir, @@ -852,13 +852,13 @@ test_that(testname, { }) -testname = "4.1 presentclimate: show variable names (all)" +testname = "4.1 observedclimate: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run twice, should yield the same result! - vars <- download_present_climate_tables(quiet=FALSE) - vars2 <- download_present_climate_tables(quiet=FALSE, subset="ALL") + vars <- download_observed_climate_tables(quiet=FALSE) + vars2 <- download_observed_climate_tables(quiet=FALSE, subset="ALL") # Check that the correct variable names are listed: expect_equal(vars, vars2) @@ -870,12 +870,12 @@ test_that(testname, { }) -testname = "4.2 presentclimate (or other simple tables): show variable names (of a subset)" +testname = "4.2 observedclimate (or other simple tables): show variable names (of a subset)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio1", "bio10"), quiet=FALSE) @@ -885,12 +885,12 @@ test_that(testname, { }) -testname = "4.3.1 presentclimate (or other simple tables): show download size (specified: subset)" +testname = "4.3.1 observedclimate (or other simple tables): show download size (specified: subset)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio10", "bio13"), tile_ids=c("h00v04", "h10v04"), quiet=FALSE) @@ -902,12 +902,12 @@ test_that(testname, { }) -testname = "4.3.2 presentclimate (or other simple tables): show download size (test passing subset=ALL)" +testname = "4.3.2 observedclimate (or other simple tables): show download size (test passing subset=ALL)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset="ALL", tile_ids=c("h10v04"), download=FALSE, @@ -919,12 +919,12 @@ test_that(testname, { }) -testname = "4.3.3 presentclimate (or other simple tables): show download size (test passing tile_ids=ALL)" +testname = "4.3.3 observedclimate (or other simple tables): show download size (test passing tile_ids=ALL)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio5"), tile_ids="ALL", download=FALSE, @@ -936,13 +936,13 @@ test_that(testname, { }) -testname = "4.4.1 presentclimate (or other simple tables): download subset" +testname = "4.4.1 observedclimate (or other simple tables): download subset" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { # Run: skip_if_offline("public.igb-berlin.de") - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio10", "bio13"), tile_ids=c("h00v04", "h10v04"), download=TRUE, @@ -961,7 +961,7 @@ test_that(testname, { }) -testname = "4.4.2 presentclimate (or other simple tables): download all variables (for one tile)" +testname = "4.4.2 observedclimate (or other simple tables): download all variables (for one tile)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -969,7 +969,7 @@ test_that(testname, { skip_if_offline("public.igb-berlin.de") # Note: Test 4.3.2 (computing download size of subset=ALL) tests quite similar behaviour, # just without the actual download in the end. - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset="ALL", tile_ids=c("h10v04"), download=TRUE, @@ -988,7 +988,7 @@ test_that(testname, { }) -testname = "4.4.3 presentclimate (or other simple tables): download all tiles (for one variable)" +testname = "4.4.3 observedclimate (or other simple tables): download all tiles (for one variable)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -996,7 +996,7 @@ test_that(testname, { skip_if(R_SKIP_HUGE_DOWNLOAD, "R_SKIP_HUGE_DOWNLOAD: This test downloads 6.9 GB, so we skip.") # Note: Test 4.3.3 (computing download size of tile_ids=ALL) tests quite similar behaviour, # just without the actual download in the end. - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio5"), tile_ids="ALL", download=TRUE, @@ -1015,7 +1015,7 @@ test_that(testname, { }) -testname = "4.5.1 presentclimate (or other simple tables): failure: variable not available" +testname = "4.5.1 observedclimate (or other simple tables): failure: variable not available" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -1026,7 +1026,7 @@ test_that(testname, { # Run: expect_error( - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( subset=c("bio1", "bio10", "xyz"), tile_ids=c("h02v02", "h04v02"), quiet=FALSE), @@ -1034,7 +1034,7 @@ test_that(testname, { ) }) -testname = "4.5.2 presentclimate (or other simple tables): failure: no subset provided" +testname = "4.5.2 observedclimate (or other simple tables): failure: no subset provided" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { @@ -1043,7 +1043,7 @@ test_that(testname, { # Run: expect_warning( - vars <- download_present_climate_tables( + vars <- download_observed_climate_tables( tile_ids=c("h02v02", "h04v02"), download=TRUE, quiet=FALSE), @@ -1055,7 +1055,7 @@ test_that(testname, { testname = "5 soil: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Note: All other soil download functionality is tested by testing the present climate + # Note: All other soil download functionality is tested by testing the observed climate # download functionality - they all use the same download_simple_tables function! # Run twice, should yield the same result! @@ -1075,7 +1075,7 @@ test_that(testname, { testname = "6 hydro90m: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Note: All other hydro90m download functionality is tested by testing the present climate + # Note: All other hydro90m download functionality is tested by testing the observed climate # download functionality - they all use the same download_simple_tables function! # Run twice, should yield the same result! @@ -1104,7 +1104,7 @@ test_that(testname, { testname = "7 cgiar: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Note: All other cgiar download functionality is tested by testing the present climate + # Note: All other cgiar download functionality is tested by testing the observed climate # download functionality - they all use the same download_simple_tables function! # Run twice, should yield the same result! @@ -1121,7 +1121,7 @@ test_that(testname, { testname = "8 flo1k: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Note: All other flo1k download functionality is tested by testing the present climate + # Note: All other flo1k download functionality is tested by testing the observed climate # download functionality - they all use the same download_simple_tables function! # Run twice, should yield the same result! @@ -1138,7 +1138,7 @@ test_that(testname, { testname = "9 merit_dem: show variable names (all)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - # Note: All other merit_dem download functionality is tested by testing the present climate + # Note: All other merit_dem download functionality is tested by testing the observed climate # download functionality - they all use the same download_simple_tables function! # Run twice, should yield the same result! diff --git a/tests/testthat/test-download_tiles.R b/tests/testthat/test-download_tiles.R index c6ee5bd8..c1a2a01c 100644 --- a/tests/testthat/test-download_tiles.R +++ b/tests/testthat/test-download_tiles.R @@ -42,7 +42,7 @@ if (! exists("download_dir")){ testname = "1 - global" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - skip_if(R_SKIP_HUGE_DOWNLOAD, 'R_SKIP_HUGE_DOWNLOAD: Downloading this is 7431.1 MB, so we skip it this time...') + skip_if(R_SKIP_HUGE_DOWNLOAD, 'R_SKIP_HUGE_DOWNLOAD: This test downloads 7.4 GB, so we skip it.') # Prepare: download_dir <- file.path(tmpdir, "test_download_tiles_1") diff --git a/tests/testthat/test-download_tiles_base.R b/tests/testthat/test-download_tiles_base.R index 804dec55..2579f54a 100644 --- a/tests/testthat/test-download_tiles_base.R +++ b/tests/testthat/test-download_tiles_base.R @@ -39,7 +39,7 @@ if (!(tests_quiet)) print("Testing: download_tiles_base") testname = "1: Global case: 'direction_ovr.tif' (from IGB)" if (!(tests_quiet)) print(paste("TEST: ", testname)) test_that(testname, { - skip_if(R_SKIP_HUGE_DOWNLOAD, 'R_SKIP_HUGE_DOWNLOAD: This test downloads 7431.1 MB, so we skip it.') + skip_if(R_SKIP_HUGE_DOWNLOAD, 'R_SKIP_HUGE_DOWNLOAD: This test downloads 7.4 GB, so we skip it.') # Prepare: download_dir <- file.path(tmpdir, "test_download_tiles_base_1")