diff --git a/DESCRIPTION b/DESCRIPTION index e136c9eb..4fd4d406 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,7 +22,7 @@ Imports: curl, DBI, gargle (>= 1.5.0), - httr, + httr2, jsonlite, lifecycle, methods, diff --git a/NAMESPACE b/NAMESPACE index 8dfaf332..b3a7de58 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -156,18 +156,21 @@ import(methods) import(rlang, except = unbox) importFrom(bit64,integer64) importFrom(gargle,token_fetch) -importFrom(httr,DELETE) -importFrom(httr,GET) -importFrom(httr,PATCH) -importFrom(httr,POST) -importFrom(httr,PUT) -importFrom(httr,add_headers) -importFrom(httr,config) -importFrom(httr,content) -importFrom(httr,headers) -importFrom(httr,http_status) -importFrom(httr,parse_media) -importFrom(httr,status_code) +importFrom(httr2,req_auth_bearer_token) +importFrom(httr2,req_body_json) +importFrom(httr2,req_body_raw) +importFrom(httr2,req_error) +importFrom(httr2,req_method) +importFrom(httr2,req_perform) +importFrom(httr2,req_url_path_append) +importFrom(httr2,req_url_query) +importFrom(httr2,req_user_agent) +importFrom(httr2,request) +importFrom(httr2,resp_body_json) +importFrom(httr2,resp_body_raw) +importFrom(httr2,resp_body_string) +importFrom(httr2,resp_content_type) +importFrom(httr2,resp_status) importFrom(jsonlite,unbox) importFrom(lifecycle,deprecated) importFrom(tibble,tibble) diff --git a/NEWS.md b/NEWS.md index 8ecfd063..e4c7632f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -9,6 +9,8 @@ * bigrquery now requires R 4.0, in line with our version support principles. +* bigrquery now uses `httr2` internally instead of `httr`. + # bigrquery 1.5.1 * Forward compatibility with upcoming dbplyr release (#601). diff --git a/R/bq-auth.R b/R/bq-auth.R index 44839536..77bc7514 100644 --- a/R/bq-auth.R +++ b/R/bq-auth.R @@ -92,6 +92,12 @@ bq_auth <- function(email = gargle::gargle_oauth_email(), "Try calling {.fun bq_auth} directly with necessary specifics." )) } + # Take a tip from httr2::oauth_token() and store the expiry time. That way + # we know when we need to refresh credentials before making a request. + expires_in <- cred$credentials$expires_in + if (!is.null(expires_in)) { + cred$credentials$expires_at <- as.numeric(Sys.time()) + expires_in + } .auth$set_cred(cred) .auth$set_auth_active(TRUE) @@ -131,7 +137,7 @@ bq_deauth <- function() { #' Produce configured token #' #' @eval gargle:::PREFIX_token_description(gargle_lookup_table) -#' @eval gargle:::PREFIX_token_return() +#' @returns An OAuth bearer token. #' #' @family low-level API functions #' @export @@ -143,7 +149,19 @@ bq_token <- function() { if (!bq_has_token()) { bq_auth() } - httr::config(token = .auth$cred) + # Opportunistically refresh the token, if possible. We are doing something + # close to httr2:::token_has_expired() here, but melded to gargle's refresh + # implementation. + token <- .auth$cred$credentials + if (is.null(token$expires_at) || !.auth$cred$can_refresh()) { + return(token$access_token) + } + deadline <- as.integer(Sys.time()) + 5 + if (deadline > token$expires_at) { + .auth$cred$refresh() + token <- .auth$cred$credentials + } + token$access_token } #' Is there a token on hand? @@ -229,7 +247,7 @@ bq_oauth_client <- function() { #' } bq_user <- function() { if (bq_has_token()) { - gargle::token_email(bq_token()) + gargle::token_email(.auth$cred) } else { NULL } diff --git a/R/bq-download.R b/R/bq-download.R index 14d12a66..4c18761b 100644 --- a/R/bq-download.R +++ b/R/bq-download.R @@ -391,13 +391,13 @@ bq_download_chunk_handle <- function(x, begin = 0L, max_results = 1e4) { ) url <- paste0(base_url, bq_path(x$project, dataset = x$dataset, table = x$table, data = "")) - url <- httr::modify_url(url, query = prepare_bq_query(query)) + url <- httr2::url_parse(url) + url$query <- prepare_bq_query(query) + url <- httr2::url_build(url) if (bq_has_token()) { - token <- .auth$get_cred() - signed <- token$sign("GET", url) - url <- signed$url - headers <- signed$headers + # TODO: Did we break non-header clients here? + headers <- c("Authorization" = paste("Bearer", bq_token())) } else { headers <- list() } @@ -416,12 +416,14 @@ bq_download_callback <- function(path, progress = NULL, call = caller_env()) { function(result) { if (!is.null(progress)) cli::cli_progress_update(id = progress) - bq_check_response( - status = result$status_code, - type = curl::parse_headers_list(result$headers)[["content-type"]], - content = result$content, - call = call - ) + if (result$status_code >= 400) { + resp <- httr2::response( + status_code = result$status_code, + headers = result$headers, + body = result$content + ) + cli::cli_abort(bq_error_body(resp), call = call) + } con <- file(path, open = "wb") defer(close(con)) diff --git a/R/bq-request.R b/R/bq-request.R index 457cbba8..ec729a86 100644 --- a/R/bq-request.R +++ b/R/bq-request.R @@ -31,7 +31,7 @@ bq_ua <- function() { "bigrquery/", utils::packageVersion("bigrquery"), " ", "(GPN:RStudio; )", " ", "gargle/", utils::packageVersion("gargle"), " ", - "httr/", utils::packageVersion("httr") + "httr2/", utils::packageVersion("httr2") ) } @@ -40,33 +40,29 @@ bq_body <- function(body, ...) { utils::modifyList(body, user) } - -#' @importFrom httr GET config +#' @importFrom httr2 req_perform resp_body_json resp_body_raw bq_get <- function(url, ..., query = NULL, raw = FALSE, token = bq_token()) { - req <- GET( - paste0(base_url, url), - token, - httr::user_agent(bq_ua()), - ..., - query = prepare_bq_query(query) - ) - process_request(req, raw = raw) + req <- bq_request(url, "GET", token, query = query) + resp <- req_perform(req) + if (!raw) { + resp_body_json(resp) + } else { + resp_body_raw(resp) + } } -bq_exists <- function(url, ..., query = NULL, token = bq_token()) { - req <- GET( - paste0(base_url, url), - token, - httr::user_agent(bq_ua()), - ..., - query = prepare_bq_query(query) - ) - status_code(req) >= 200 && status_code(req) < 300 +bq_exists <- function(url, query = NULL, token = bq_token()) { + req <- bq_request(url, "GET", token, query = query) + resp <- req_perform(req) + # A 404 is not an error here. + req <- req_error(req, is_error = function(resp) { + resp_status(resp) != 404 && resp_status(resp) >= 400 + }) + resp <- req_perform(req) + resp_status(resp) != 404 } - -#' @importFrom httr GET config -bq_get_paginated <- function(url, ..., query = NULL, token = bq_token(), +bq_get_paginated <- function(url, query = NULL, token = bq_token(), page_size = 50, max_pages = Inf, warn = TRUE) { check_number_whole(max_pages, min = 1, allow_infinite = TRUE) @@ -78,14 +74,14 @@ bq_get_paginated <- function(url, ..., query = NULL, token = bq_token(), query <- utils::modifyList(list(maxResults = page_size), query %||% list()) pages <- list() - page <- bq_get(url, ..., query = query, token = token) + page <- bq_get(url, query = query, token = token) i <- 1 pages[[i]] <- page page_token <- page$nextPageToken while (!is.null(page_token) && i < max_pages) { query$pageToken <- page_token - page <- bq_get(url, ..., query = query, token = token) + page <- bq_get(url, query = query, token = token) i <- i + 1 pages[[i]] <- page @@ -103,137 +99,81 @@ bq_get_paginated <- function(url, ..., query = NULL, token = bq_token(), pages } - -#' @importFrom httr DELETE config -bq_delete <- function(url, ..., query = NULL, token = bq_token()) { - req <- DELETE( - paste0(base_url, url), - token, - httr::user_agent(bq_ua()), - ..., - query = prepare_bq_query(query) - ) - process_request(req) +#' @importFrom httr2 req_perform +bq_delete <- function(url, query = NULL, token = bq_token()) { + req <- bq_request(url, "DELETE", token, query = query) + req_perform(req) + invisible(NULL) } -#' @importFrom httr POST add_headers config -bq_post <- function(url, body, ..., query = NULL, token = bq_token()) { - json <- jsonlite::toJSON(body, pretty = TRUE, auto_unbox = TRUE) - - req <- POST( - paste0(base_url, url), - body = json, - httr::user_agent(bq_ua()), - token, - add_headers("Content-Type" = "application/json"), - ..., - query = prepare_bq_query(query) - ) - invisible(process_request(req)) +#' @importFrom httr2 req_body_json req_perform resp_body_json +bq_post <- function(url, body, query = NULL, token = bq_token()) { + req <- bq_request(url, "POST", token, query = query) + req <- req_body_json(req, body) + resp <- req_perform(req) + invisible(resp_body_json(resp)) } -#' @importFrom httr PATCH add_headers config -bq_patch <- function(url, body, ..., query = NULL, token = bq_token()) { - json <- jsonlite::toJSON(body, pretty = TRUE, auto_unbox = TRUE) - req <- PATCH( - paste0(base_url, url), - body = json, - httr::user_agent(bq_ua()), - token, - add_headers("Content-Type" = "application/json"), - ..., - query = prepare_bq_query(query) - ) - process_request(req) +#' @importFrom httr2 req_body_json req_perform resp_body_json +bq_patch <- function(url, body, query = NULL, token = bq_token()) { + req <- bq_request(url, "PATCH", token, query = query) + req <- req_body_json(req, body) + resp <- req_perform(req) + resp_body_json(resp) } -#' @importFrom httr POST PUT add_headers headers config status_code +#' @importFrom httr2 req_body_json req_body_raw req_perform resp_body_json # https://cloud.google.com/bigquery/docs/reference/api-uploads bq_upload <- function(url, metadata, media, query = list(), token = bq_token()) { query <- utils::modifyList(list(fields = "jobReference",uploadType = "resumable"), query) - config <- add_headers("Content-Type" = metadata[["type"]]) - - req <- POST( - paste0(upload_url, url), - body = metadata[["content"]], - httr::user_agent(bq_ua()), - token, - config, - query = query - ) - if (status_code(req) == 200) { + req <- bq_request(url, "POST", token, query = query, base_url = upload_url) + req <- req_body_json(req, metadata[["content"]]) + resp <- req_perform(req) - config <- add_headers("Content-Type" = media[["type"]]) - - req <- PUT( - headers(req)$location, - body = media[["content"]], - httr::user_agent(bq_ua()), - token, - config - ) - - } - - process_request(req) + # Note: We only get here if the request above is successful. + session_uri <- httr2::resp_header(resp, "Location") + req <- bq_request(session_uri, "PUT", token, query = query) + req <- req_body_raw(req, media[["content"]], type = media[["type"]]) + resp <- req_perform(req) + resp_body_json(resp) } - -#' @importFrom httr http_status content parse_media status_code -process_request <- function(req, raw = FALSE, call = caller_env()) { - status <- status_code(req) - # No content -> success - if (status == 204) return(TRUE) - - type <- req$headers$`Content-type` - content <- content(req, "raw") - - bq_check_response( - status = status, - type = type, - content = content, - call = call - ) - - if (raw) { - content - } else { - jsonlite::fromJSON(rawToChar(content), simplifyVector = FALSE) +#' @importFrom httr2 request req_user_agent req_url_path_append req_method req_auth_bearer_token req_url_query req_error req_perform +bq_request <- function(url, + method, + token, + query = NULL, + base_url = base_url, + call = caller_env()) { + req <- request(base_url) + req <- req_user_agent(req, bq_ua()) + req <- req_url_path_append(req, url) + req <- req_method(req, method) + req <- req_auth_bearer_token(req, token) + if (!is.null(query)) { + req <- req_url_query(req, !!!prepare_bq_query(query)) } + req_error(req, body = bq_error_body) } -bq_check_response <- function(status, type, content, call = caller_env()) { - if (status >= 200 && status < 300) { - return() - } - - type <- httr::parse_media(type) - text <- rawToChar(content) - - if (type$complete == "application/json") { - json <- jsonlite::fromJSON(text, simplifyVector = FALSE) - gargle_abort( - reason = json$error$errors[[1L]]$reason, - message = json$error$message, - status = status, - call = call - ) - } else { - message <- paste0("HTTP error [", status, "]\n", text) - gargle_abort( - reason = NULL, - message = message, - status = status, - call = call +#' @importFrom httr2 resp_content_type resp_status resp_body_json resp_body_string +bq_error_body <- function(resp) { + # Generic error message for non-JSON responses. + if (resp_content_type(resp) != "application/json") { + message <- paste0( + "HTTP error [", + resp_status(resp), + "]\n", + resp_body_string(resp) ) + return(message) } -} - -gargle_abort <- function(reason, message, status, call = caller_env()) { - class <- paste0("bigrquery_http_", status) + body <- resp_body_json(resp) + reason <- body$error$errors[[1L]]$reason + message <- body$error$message if (!is.null(reason)) { advice <- NULL if (reason == "responseTooLarge") { @@ -255,9 +195,6 @@ gargle_abort <- function(reason, message, status, call = caller_env()) { paste0(message, " [", reason, "] "), i = advice ) - class <- c(paste0("bigrquery_", reason), class) } - - cli::cli_abort(message, class = class, call = call) + message } - diff --git a/R/bq-table.R b/R/bq-table.R index fb744287..41ccfaf4 100644 --- a/R/bq-table.R +++ b/R/bq-table.R @@ -116,7 +116,8 @@ bq_table_exists <- function(x) { bq_table_delete <- function(x) { x <- as_bq_table(x) url <- bq_path(x$project, x$dataset, x$table) - invisible(bq_delete(url)) + bq_delete(url) + invisible(x) } #' @export diff --git a/R/gs-object.R b/R/gs-object.R index 89f0af48..0116e666 100644 --- a/R/gs-object.R +++ b/R/gs-object.R @@ -27,8 +27,12 @@ gs_object_delete <- function(x, token = bq_token()) { x$bucket, x$object ) - req <- httr::DELETE(url, token, httr::user_agent(bq_ua())) - process_request(req) + req <- request(url) + req <- req_method(req, "DELETE") + req <- req_user_agent(req, bq_ua()) + req <- req_auth_bearer_token(req, token$auth_token) + req_perform(req) + invisible(NULL) } gs_object_exists <- function(x, token = bq_token()) { @@ -37,6 +41,13 @@ gs_object_exists <- function(x, token = bq_token()) { x$bucket, x$object ) - req <- httr::GET(url, token, httr::user_agent(bq_ua())) - req$status_code != 404 + req <- request(url) + req <- req_user_agent(req, bq_ua()) + req <- req_auth_bearer_token(req, token$auth_token) + # A 404 is not an error here. + req <- req_error(req, is_error = function(resp) { + resp_status(resp) != 404 && resp_status(resp) >= 400 + }) + resp <- req_perform(req) + resp_status(resp) != 404 } diff --git a/man/bq_token.Rd b/man/bq_token.Rd index ca2df952..d8ccb607 100644 --- a/man/bq_token.Rd +++ b/man/bq_token.Rd @@ -7,7 +7,7 @@ bq_token() } \value{ -A \code{request} object (an S3 class provided by \link[httr:httr-package]{httr}). +An OAuth bearer token. } \description{ For internal use or for those programming around the BigQuery API.