diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml index 9e7aa575d5cd..8255915b76b6 100644 --- a/bindings/dotnet/Cargo.toml +++ b/bindings/dotnet/Cargo.toml @@ -34,7 +34,7 @@ doc = false # this crate won't be published, we always use the local version opendal = { version = ">=0", path = "../../core", features = [ "blocking", - "reqwest-rustls-tls", + "http-transport-reqwest", "executors-tokio", # enabled layers diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index e88f5a348b71..b5b8070afcb9 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -35,7 +35,7 @@ jni = { version = "0.22.4" } # opendal-java won't be published to crates.io, we always use the local version opendal = { version = ">=0", path = "../../core", default-features = false, features = [ "blocking", - "reqwest-rustls-tls", + "http-transport-reqwest-rustls", "executors-tokio", # enabled layers diff --git a/core/Cargo.lock b/core/Cargo.lock index b5b4ed5b6551..9af901922770 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -4654,6 +4654,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.9.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -9390,11 +9406,13 @@ dependencies = [ "http-body-util", "hyper 1.9.0", "hyper-rustls 0.27.9", + "hyper-tls", "hyper-util", "js-sys", "log", "mime", "mime_guess", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -9405,6 +9423,7 @@ dependencies = [ "serde_json", "sync_wrapper 1.0.2", "tokio", + "tokio-native-tls", "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.3", diff --git a/core/Cargo.toml b/core/Cargo.toml index abd30f49de1f..78dc695274e8 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -87,7 +87,6 @@ blocking = ["opendal-core/blocking"] default = [ "auto-register-services", "http-transport-reqwest", - "reqwest-rustls-tls", "executors-tokio", "layers-concurrent-limit", "layers-logging", @@ -95,7 +94,25 @@ default = [ "layers-timeout", ] executors-tokio = ["opendal-core/executors-tokio"] -http-transport-reqwest = ["dep:opendal-http-transport-reqwest"] +# Default http-transport-reqwest transport feature, used for: +# - install default http transport +# - feature gate for various of bindings dependency +http-transport-reqwest = ["http-transport-reqwest-rustls"] +# Use reqwest with platform TLS via native-tls. +http-transport-reqwest-native-tls = [ + "dep:opendal-http-transport-reqwest", + "opendal-http-transport-reqwest/native-tls", +] +# Use reqwest with Rustls, with default aws-lc-rs crypto provider. +http-transport-reqwest-rustls = [ + "dep:opendal-http-transport-reqwest", + "opendal-http-transport-reqwest/rustls", +] +# Use reqwest with Rustls and no built-in crypto provider. +http-transport-reqwest-rustls-no-provider = [ + "dep:opendal-http-transport-reqwest", + "opendal-http-transport-reqwest/rustls-no-provider", +] internal-tokio-rt = ["opendal-core/internal-tokio-rt"] layers-async-backtrace = ["dep:opendal-layer-async-backtrace"] layers-await-tree = ["dep:opendal-layer-await-tree"] @@ -121,14 +138,13 @@ layers-tail-cut = ["dep:opendal-layer-tail-cut"] layers-throttle = ["dep:opendal-layer-throttle"] layers-timeout = ["dep:opendal-layer-timeout"] layers-tracing = ["dep:opendal-layer-tracing"] +# Deprecated since 0.58; Will delete in 0.59. reqwest-rustls-no-provider-tls = [ "http-transport-reqwest", - "opendal-http-transport-reqwest/rustls-no-provider", -] -reqwest-rustls-tls = [ - "http-transport-reqwest", - "opendal-http-transport-reqwest/rustls", + "http-transport-reqwest-rustls-no-provider", ] +# Deprecated since 0.58; Will delete in 0.59. +reqwest-rustls-tls = ["http-transport-reqwest", "http-transport-reqwest-rustls"] services-aliyun-drive = ["dep:opendal-service-aliyun-drive"] services-alluxio = ["dep:opendal-service-alluxio"] services-azblob = ["dep:opendal-service-azblob"] diff --git a/core/http-transports/reqwest/Cargo.toml b/core/http-transports/reqwest/Cargo.toml index 1b72144e2f24..687bf02d1838 100644 --- a/core/http-transports/reqwest/Cargo.toml +++ b/core/http-transports/reqwest/Cargo.toml @@ -31,8 +31,20 @@ version = { workspace = true } all-features = true [features] -default = [] +default = ["rustls"] +# Use platform TLS via reqwest's native-tls feature. +native-tls = ["reqwest/native-tls"] +# Use Rustls with reqwest's default crypto provider and platform certificate verification. rustls = ["reqwest/rustls"] +# Use Rustls without a built-in crypto provider. +# +# `rustls` builds `aws_lc_rs` crypto provider by default. When using other TLS backends, +# consider features like: +# - ring +# - fips +# Also read more about `rustls-native-certs` crate. +# +# To use WebPKI roots, read more about `webpki_roots` crate as well. rustls-no-provider = ["reqwest/rustls-no-provider"] [dependencies] diff --git a/core/http-transports/reqwest/README.md b/core/http-transports/reqwest/README.md new file mode 100644 index 000000000000..ca312a970935 --- /dev/null +++ b/core/http-transports/reqwest/README.md @@ -0,0 +1,196 @@ +# opendal-http-transport-reqwest + +Reqwest-based HTTP transport for [Apache OpenDAL](https://opendal.apache.org). + +This crate provides `ReqwestTransport`, an implementation of OpenDAL's +`HttpTransport` trait backed by [reqwest](https://crates.io/crates/reqwest). + +## TLS configuration + +OpenDAL does not add another TLS backend selector on top of reqwest. Pick the +Cargo feature set you need, build a `reqwest::Client` with reqwest's +`ClientBuilder`, and pass that client to `ReqwestTransport::new`. + +The default feature is `rustls`. When `ReqwestTransport::default()` is used on +native targets, this crate explicitly asks reqwest for the Rustls backend if the +`rustls` feature is compiled in. + +If this crate is built with only `rustls-no-provider`, install a Rustls default +crypto provider before using `ReqwestTransport::default()`, or build a +preconfigured `reqwest::Client` yourself and pass it to `ReqwestTransport::new`. + +### Reqwest TLS resolution + +Reqwest has its own TLS resolution rules. For OpenDAL users, the practical +resolution order is: + +- Explicit reqwest client configuration wins. Call + `ClientBuilder::tls_backend_rustls()`, + `ClientBuilder::tls_backend_native()`, or + `ClientBuilder::tls_backend_preconfigured()`, but reqwest documents that API + as semver-unstable because the concrete TLS config type must match reqwest's + dependency versions. +- Otherwise, reqwest's automatic TLS backend selection applies. The reqwest + `default-tls` feature takes precedence if any crate in the dependency tree + enables it. Disable reqwest defaults with `default-features = false` when you + need deterministic TLS features. +- Cargo features are additive. If multiple TLS backends are compiled by feature + unification, use explicit client configuration instead of relying on automatic + selection. + +See reqwest's TLS documentation: + +- [reqwest TLS module](https://docs.rs/reqwest/latest/reqwest/tls/index.html) +- [ClientBuilder::tls_backend_rustls](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_rustls) +- [ClientBuilder::tls_backend_native](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_native) +- [ClientBuilder::tls_backend_preconfigured](https://docs.rs/reqwest/latest/reqwest/struct.ClientBuilder.html#method.tls_backend_preconfigured) + +### Feature matrix + +| Feature | Crypto provider | Certificate roots | Use when | +|---------|-----------------|-------------------|----------| +| `rustls` (default) | reqwest default | Platform verifier | Pure-Rust TLS with the system trust store | +| `rustls-no-provider` | You provide | You provide | BYO crypto provider, roots, or FIPS module | +| `native-tls` | OS library | OS trust store | You want platform TLS | + +## Cargo features + +Most users depend on the `opendal` facade crate: + +```toml +[dependencies] +# Default: reqwest transport with rustls. +opendal = { version = "0.57" } +``` + +To select one transport TLS feature explicitly, disable default features: + +```toml +[dependencies] +opendal = { version = "0.57", default-features = false, features = [ + "http-transport-reqwest-rustls", +] } +``` + +## Build different TLS backends + +### Rustls + +Use reqwest's Rustls backend explicitly when you build the client: + +```toml +[dependencies] +opendal = { version = "0.57", default-features = false, features = [ + "http-transport-reqwest-rustls", +] } +reqwest = { version = "0.13.4", default-features = false, features = [ + "rustls", + "stream", +] } +``` + +```rust +use std::time::Duration; + +use opendal::HttpTransporter; +use opendal::ReqwestTransport; + +fn build_transport() -> Result> { + let client = reqwest::Client::builder() + .tls_backend_rustls() + .connect_timeout(Duration::from_secs(10)) + .build()?; + + Ok(HttpTransporter::new(ReqwestTransport::new(client))) +} +``` + +### Native TLS + +Use `native-tls` when you want reqwest to delegate TLS to the platform stack: + +```toml +[dependencies] +opendal = { version = "0.57", default-features = false, features = [ + "http-transport-reqwest-native-tls", +] } +reqwest = { version = "0.13.4", default-features = false, features = [ + "native-tls", + "stream", +] } +``` + +```rust +use opendal::HttpTransporter; +use opendal::ReqwestTransport; + +fn build_transport() -> Result> { + let client = reqwest::Client::builder() + .tls_backend_native() + .build()?; + + Ok(HttpTransporter::new(ReqwestTransport::new(client))) +} +``` + +### rustls-no-provider + +When using `wekpki-roots`, you could use `rustls-no-provider`. + +Use `rustls-no-provider` when the application owns the Rustls provider and +certificate roots. This example only configures Mozilla roots from +`webpki-roots`; install or configure the Rustls crypto provider in application +startup. + +```toml +[dependencies] +opendal = { version = "0.57", default-features = false, features = [ + "http-transport-reqwest-rustls-no-provider", +] } +reqwest = { version = "0.13.4", default-features = false, features = [ + "rustls-no-provider", + "stream", +] } +rustls = { version = "0.23", default-features = false, features = ["std"] } +webpki-roots = "1" +``` + +```rust +use opendal::HttpTransporter; +use opendal::ReqwestTransport; + +fn build_transport() -> Result> { + let root_store = + rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + + let tls_config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + let client = reqwest::Client::builder() + .tls_backend_preconfigured(tls_config) + .build()?; + + Ok(HttpTransporter::new(ReqwestTransport::new(client))) +} +``` + +To use `ring` crate, please refer `rustls` documentation. + +## WASM targets + +On `wasm32-unknown-unknown` and `wasm32-none`, reqwest switches to its WASM +client implementation. It uses the browser or host Fetch API instead of hyper. +TLS is provided by the browser or host environment, so the effective choice is +the system TLS stack exposed to that environment. + +This means `native-tls`, `rustls`, `rustls-no-provider`, custom root stores, +and custom Rustls crypto providers do not select the TLS implementation for +wasm requests. See [reqwest's WASM documentation](https://docs.rs/reqwest/latest/reqwest/#wasm) +for the target-specific limitations. + +## License and Trademarks + +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 + +Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. diff --git a/core/http-transports/reqwest/src/lib.rs b/core/http-transports/reqwest/src/lib.rs index 6f7c49b3a4e1..11a0ea0a1157 100644 --- a/core/http-transports/reqwest/src/lib.rs +++ b/core/http-transports/reqwest/src/lib.rs @@ -16,14 +16,58 @@ // under the License. //! Reqwest based HTTP transport for Apache OpenDAL. +//! +//! # TLS backends +//! +//! Enable one of the following Cargo features to compile reqwest TLS support. +//! The `default` feature enables `rustls`. +//! +//! - **`native-tls`** — Platform TLS links against the OS TLS library: +//! - Windows: SChannel +//! - macOS: Secure Transport +//! - Linux: OpenSSL, requires system development packages +//! +//! - **`rustls`** — [Rustls](https://crates.io/crates/rustls) configured by +//! reqwest with its default crypto provider and platform certificate +//! verification. Pure-Rust TLS stack, no system TLS dependency. +//! +//! - **`rustls-no-provider`** — Rustls without a built-in crypto provider. +//! You must install a [`rustls::crypto::CryptoProvider`] before building a +//! client. Use this when you want to bring your own provider +//! (e.g., a FIPS-certified module). +//! +//! In application or language binding builds, prefer selecting a single backend +//! feature. In workspace or `--all-features` builds, Cargo may enable multiple +//! backend features via feature unification; build a [`reqwest::Client`] with +//! reqwest's TLS configuration methods and pass it to [`ReqwestTransport::new`] +//! to force the backend you want. +//! +//! On wasm targets, reqwest uses the Fetch API instead of hyper. TLS is provided +//! by the host environment, so native TLS backend features do not select a TLS +//! stack there. +//! +//! # Custom reqwest client +//! +//! ``` +//! use std::time::Duration; +//! +//! use opendal_core::HttpTransporter; +//! use opendal_http_transport_reqwest::ReqwestTransport; +//! +//! # fn build() -> Result<(), Box> { +//! let client = reqwest::Client::builder() +//! .tls_backend_rustls() +//! .connect_timeout(Duration::from_secs(10)) +//! .build()?; +//! let transport = ReqwestTransport::new(client); +//! let _transport = HttpTransporter::new(transport); +//! # Ok(()) +//! # } +//! ``` -#![deny(missing_docs)] - -use std::fmt::Debug; -use std::fmt::Formatter; +use std::fmt::{Debug, Formatter}; use std::future; use std::mem; -use std::str::FromStr; use std::sync::LazyLock; use futures::TryStreamExt; @@ -40,11 +84,7 @@ use opendal_core::raw::parse_content_length; static DEFAULT_REQWEST_CLIENT: LazyLock = LazyLock::new(reqwest::Client::new); -/// A [`reqwest::Client`] backed HTTP transport. -/// -/// # Notes -/// -/// Reqwest must be configured with a TLS feature before sending HTTPS requests. +/// A HTTP transport with [`reqwest::Client`]. #[derive(Clone)] pub struct ReqwestTransport { client: reqwest::Client, @@ -84,7 +124,7 @@ impl HttpTransport for ReqwestTransport { let (parts, body) = req.into_parts(); - let url = reqwest::Url::from_str(&uri.to_string()).map_err(|err| { + let url = reqwest::Url::parse(&uri.to_string()).map_err(|err| { Error::new(ErrorKind::Unexpected, "request url is invalid") .with_operation("reqwest::fetch") .with_context("url", uri.to_string()) @@ -202,3 +242,22 @@ impl http_body::Body for HttpBufferBody { http_body::SizeHint::with_exact(self.0.len() as u64) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(any(feature = "rustls", feature = "native-tls"))] + #[test] + fn test_default_transport_succeeds() { + let transport = ReqwestTransport::default(); + assert_eq!(format!("{:?}", transport), "ReqwestTransport"); + } + + #[test] + fn test_from_reqwest_client() { + let client = reqwest::Client::new(); + let transport = ReqwestTransport::from(client); + assert_eq!(format!("{:?}", transport), "ReqwestTransport"); + } +} diff --git a/core/services/hf/Cargo.toml b/core/services/hf/Cargo.toml index c159da107843..205d0cd7f295 100644 --- a/core/services/hf/Cargo.toml +++ b/core/services/hf/Cargo.toml @@ -43,8 +43,6 @@ serde_json = { workspace = true } [dev-dependencies] base64 = { workspace = true } futures = { workspace = true } -opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.57.0", features = [ - "rustls", -] } +opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.57.0" } serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }