-
Notifications
You must be signed in to change notification settings - Fork 784
feat(http-transport-reqwest): allow tls options #7798
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
6c8970a
dede66d
2676e10
0a3522a
2de79e8
57f4c67
76cc17d
1d887a0
fed18c9
4487b59
09d5e0c
dc4aa7e
b84e44e
e8dce1c
ce37d8a
2f82fc6
1e6d1e8
eab6d50
9aee9e8
d615f06
c46ac0f
3eaf192
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -31,9 +31,25 @@ version = { workspace = true } | |
| all-features = true | ||
|
|
||
| [features] | ||
| default = [] | ||
| rustls = ["reqwest/rustls"] | ||
| default = ["native-tls"] | ||
| # Use platform TLS via reqwest's native-tls feature. | ||
| native-tls = ["reqwest/native-tls"] | ||
| # Use Rustls with the aws-lc-rs crypto provider and platform certificate verification. | ||
| rustls = [ | ||
| "reqwest/rustls-no-provider", | ||
| "dep:rustls", | ||
| "rustls/aws-lc-rs", | ||
| "dep:rustls-platform-verifier", | ||
| ] | ||
| # Use Rustls without a built-in crypto provider. | ||
| rustls-no-provider = ["reqwest/rustls-no-provider"] | ||
| # Use Rustls with the aws-lc-rs crypto provider and bundled Mozilla root certificates. | ||
| webpki-roots = [ | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should this be rustls-wekpki-roots? |
||
| "reqwest/rustls-no-provider", | ||
| "dep:rustls", | ||
| "rustls/aws-lc-rs", | ||
| "dep:webpki-roots", | ||
| ] | ||
|
|
||
| [dependencies] | ||
| bytes = { workspace = true } | ||
|
|
@@ -44,3 +60,6 @@ opendal-core = { path = "../../core", version = "0.57.0", default-features = fal | |
| reqwest = { version = "0.13.4", features = [ | ||
| "stream", | ||
| ], default-features = false } | ||
| rustls = { version = "0.23", optional = true, default-features = false } | ||
|
Xuanwo marked this conversation as resolved.
Outdated
|
||
| rustls-platform-verifier = { version = "0.7", optional = true, default-features = false } | ||
| webpki-roots = { version = "1", optional = true } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| # opendal-http-transport-reqwest | ||
|
|
||
| Reqwest-based HTTP transport for [Apache OpenDAL](https://opendal.apache.org). | ||
|
|
||
| This crate provides `ReqwestTransport`, an implementation of OpenDAL's | ||
| `HttpTransport` trait backed by [reqwest](https://crates.io/crates/reqwest). | ||
|
|
||
| ## TLS configuration | ||
|
|
||
| When using Rustls, TLS configuration has two independent axes: | ||
|
|
||
| | Axis | What it decides | Options | | ||
| |------|----------------|---------| | ||
| | **Crypto provider** | Who performs the cryptographic operations (key exchange, symmetric ciphers, hashing) | `aws-lc-rs` (default in `rustls`/`webpki-roots`), `ring`, or any custom `CryptoProvider` | | ||
| | **Certificate verification** | How the server's TLS certificate chain is validated | Platform verifier (default in `rustls`), bundled Mozilla roots (`webpki-roots`), or custom | | ||
|
|
||
| The `native-tls` feature sidesteps both axes by delegating everything to | ||
| the OS TLS library (SChannel / Secure Transport / OpenSSL). | ||
|
|
||
| ### Feature matrix | ||
|
Xuanwo marked this conversation as resolved.
|
||
|
|
||
| | Feature | Crypto provider | Certificate roots | Use when | | ||
| |---------|----------------|-------------------|----------| | ||
| | `native-tls` (default) | OS library | OS trust store | You want zero Rust-side TLS config | | ||
|
Xuanwo marked this conversation as resolved.
Outdated
|
||
| | `rustls` | aws-lc-rs | Platform verifier | Pure-Rust TLS with OS trust store | | ||
| | `webpki-roots` | aws-lc-rs | Bundled Mozilla roots | Fully self-contained, no OS dependency | | ||
| | `rustls-no-provider` | **you provide** | **you provide** | BYO crypto (ring, FIPS module, etc.) | | ||
|
|
||
| ### Usage via the `opendal` facade crate | ||
|
|
||
| Most users depend on `opendal` rather than this crate directly. The facade | ||
| installs this transport when any `http-transport-reqwest-*` feature is enabled. | ||
|
|
||
| ```toml | ||
| # Default — reqwest transport with native-tls | ||
| opendal = { version = "0.57" } | ||
| ``` | ||
|
|
||
| To select a different TLS backend, disable default features and enable the | ||
| one you need: | ||
|
|
||
| ```toml | ||
| opendal = { version = "0.57", default-features = false, features = ["http-transport-reqwest-rustls"] } | ||
| ``` | ||
|
|
||
| ### Feature usage with `rustls` | ||
|
|
||
| ```toml | ||
| [dependencies] | ||
| opendal-http-transport-reqwest = { version = "0.57", default-features = false, features = ["rustls"] } | ||
| ``` | ||
|
|
||
| ```rust | ||
| use std::time::Duration; | ||
|
|
||
| use opendal_http_transport_reqwest::ReqwestTlsBackend; | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I didn't expect we need to implement this. Is there a reason why we didn't just use things from reqwest?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is mainly a consideration of what features do we want to compile and ship. e.g.:
Bindings will use this crate mostly so compilation size is also a concern.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I realized that I'm writing a library instead of an app. I'll fix this soon. |
||
| use opendal_http_transport_reqwest::ReqwestTransport; | ||
| use opendal::HttpTransporter; | ||
|
|
||
| // You can configure reqwest dynamically and select a compiled TLS backend. | ||
| let tls_backend = "rustls".parse::<ReqwestTlsBackend>().unwrap(); | ||
| let transport = ReqwestTransport::builder() | ||
| .tls_backend(tls_backend) | ||
| .configure(|builder| builder.connect_timeout(Duration::from_secs(10))) | ||
| .build() | ||
| .unwrap(); | ||
| ``` | ||
|
|
||
| ### Bringing your own reqwest client | ||
|
|
||
| When you need full control over the TLS stack — custom `ClientConfig`, | ||
| client certificates, proxy settings, or connection pool tuning — build a | ||
| `reqwest::Client` yourself and wrap it: | ||
|
|
||
| ```toml | ||
| [dependencies] | ||
| opendal = { version = "0.57", default-features = false, features = [ | ||
| "services-s3", | ||
| "http-transport-reqwest-rustls-no-provider", | ||
| ] } | ||
| opendal-http-transport-reqwest = { version = "0.57", default-features = false, features = ["rustls-no-provider"] } | ||
| rustls = { version = "0.23", features = ["ring"], default-features = false } | ||
| webpki-roots = "1" | ||
| ``` | ||
|
|
||
| ```rust | ||
| use std::time::Duration; | ||
|
|
||
| use opendal::HttpTransporter; | ||
| use opendal::OperationContext; | ||
| use opendal_http_transport_reqwest::ReqwestTransport; | ||
|
|
||
| fn main() { | ||
| // 1. Configure your crypto provider and certificate roots. | ||
| let root_store = | ||
| rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); | ||
|
|
||
| let tls_config = rustls::ClientConfig::builder_with_provider( | ||
| rustls::crypto::ring::default_provider().into(), | ||
| ) | ||
| .with_safe_default_protocol_versions() | ||
| .unwrap() | ||
| .with_root_certificates(root_store) | ||
| .with_no_client_auth(); | ||
|
|
||
| // 2. Build a reqwest client with your TLS config. | ||
| let client = reqwest::Client::builder() | ||
| .tls_backend_preconfigured(tls_config) | ||
| .connect_timeout(Duration::from_secs(10)) | ||
| .pool_max_idle_per_host(20) | ||
| .build() | ||
| .unwrap(); | ||
|
|
||
| // 3. Wrap it as a ReqwestTransport and attach to an operator. | ||
| let transport = HttpTransporter::new(ReqwestTransport::new(client)); | ||
|
|
||
| let op = opendal::Operator::via_iter("s3", [ | ||
| ("bucket".to_string(), "my-bucket".to_string()), | ||
| ("region".to_string(), "us-east-1".to_string()), | ||
| ]) | ||
| .expect("failed to build operator") | ||
| .with_context(OperationContext::new().with_http_transport(transport)); | ||
| } | ||
| ``` | ||
|
|
||
| This approach gives you complete ownership over TLS and client. | ||
|
|
||
| ## License and Trademarks | ||
|
|
||
| Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation. | ||
Uh oh!
There was an error while loading. Please reload this page.