Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bindings/dotnet/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ doc = false
# this crate won't be published, we always use the local version
opendal = { version = ">=0", path = "../../core", features = [
"blocking",
"reqwest-rustls-tls",
"http-transport-reqwest-rustls",
"executors-tokio",

# enabled layers
Expand Down
2 changes: 1 addition & 1 deletion bindings/java/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ jni = { version = "0.22.4" }
# opendal-java won't be published to crates.io, we always use the local version
opendal = { version = ">=0", path = "../../core", default-features = false, features = [
"blocking",
"reqwest-rustls-tls",
"http-transport-reqwest-rustls",
"executors-tokio",

# enabled layers
Expand Down
22 changes: 22 additions & 0 deletions core/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

32 changes: 22 additions & 10 deletions core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -86,16 +86,36 @@ auto-register-services = ["dep:ctor"]
blocking = ["opendal-core/blocking"]
default = [
"auto-register-services",
"http-transport-reqwest",
"reqwest-rustls-tls",
Comment thread
Xuanwo marked this conversation as resolved.
"http-transport-reqwest-native-tls",
"executors-tokio",
"layers-concurrent-limit",
"layers-logging",
"layers-retry",
"layers-timeout",
]
executors-tokio = ["opendal-core/executors-tokio"]
# Enable the reqwest HTTP transport without selecting a TLS backend.
http-transport-reqwest = ["dep:opendal-http-transport-reqwest"]
# Use reqwest with platform TLS via native-tls.
http-transport-reqwest-native-tls = [
"http-transport-reqwest",
"opendal-http-transport-reqwest/native-tls",
]
# Use reqwest with Rustls, the aws-lc-rs crypto provider, and platform certificate verification.
http-transport-reqwest-rustls = [
"http-transport-reqwest",
"opendal-http-transport-reqwest/rustls",
]
# Use reqwest with Rustls and no built-in crypto provider.
http-transport-reqwest-rustls-no-provider = [
"http-transport-reqwest",
"opendal-http-transport-reqwest/rustls-no-provider",
]
# Use reqwest with Rustls, the aws-lc-rs crypto provider, and bundled Mozilla root certificates.
http-transport-reqwest-webpki-roots = [
"http-transport-reqwest",
"opendal-http-transport-reqwest/webpki-roots",
]
internal-tokio-rt = ["opendal-core/internal-tokio-rt"]
layers-async-backtrace = ["dep:opendal-layer-async-backtrace"]
layers-await-tree = ["dep:opendal-layer-await-tree"]
Expand All @@ -121,14 +141,6 @@ layers-tail-cut = ["dep:opendal-layer-tail-cut"]
layers-throttle = ["dep:opendal-layer-throttle"]
layers-timeout = ["dep:opendal-layer-timeout"]
layers-tracing = ["dep:opendal-layer-tracing"]
reqwest-rustls-no-provider-tls = [
Comment thread
Xuanwo marked this conversation as resolved.
"http-transport-reqwest",
"opendal-http-transport-reqwest/rustls-no-provider",
]
reqwest-rustls-tls = [
"http-transport-reqwest",
"opendal-http-transport-reqwest/rustls",
]
services-aliyun-drive = ["dep:opendal-service-aliyun-drive"]
services-alluxio = ["dep:opendal-service-alluxio"]
services-azblob = ["dep:opendal-service-azblob"]
Expand Down
23 changes: 21 additions & 2 deletions core/http-transports/reqwest/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,25 @@ version = { workspace = true }
all-features = true

[features]
default = []
rustls = ["reqwest/rustls"]
default = ["native-tls"]
# Use platform TLS via reqwest's native-tls feature.
native-tls = ["reqwest/native-tls"]
# Use Rustls with the aws-lc-rs crypto provider and platform certificate verification.
rustls = [
"reqwest/rustls-no-provider",
"dep:rustls",
"rustls/aws-lc-rs",
"dep:rustls-platform-verifier",
]
# Use Rustls without a built-in crypto provider.
rustls-no-provider = ["reqwest/rustls-no-provider"]
# Use Rustls with the aws-lc-rs crypto provider and bundled Mozilla root certificates.
webpki-roots = [

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be rustls-wekpki-roots?

"reqwest/rustls-no-provider",
"dep:rustls",
"rustls/aws-lc-rs",
"dep:webpki-roots",
]

[dependencies]
bytes = { workspace = true }
Expand All @@ -44,3 +60,6 @@ opendal-core = { path = "../../core", version = "0.57.0", default-features = fal
reqwest = { version = "0.13.4", features = [
"stream",
], default-features = false }
rustls = { version = "0.23", optional = true, default-features = false }
Comment thread
Xuanwo marked this conversation as resolved.
Outdated
rustls-platform-verifier = { version = "0.7", optional = true, default-features = false }
webpki-roots = { version = "1", optional = true }
132 changes: 132 additions & 0 deletions core/http-transports/reqwest/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# opendal-http-transport-reqwest

Reqwest-based HTTP transport for [Apache OpenDAL](https://opendal.apache.org).

This crate provides `ReqwestTransport`, an implementation of OpenDAL's
`HttpTransport` trait backed by [reqwest](https://crates.io/crates/reqwest).

## TLS configuration

When using Rustls, TLS configuration has two independent axes:

| Axis | What it decides | Options |
|------|----------------|---------|
| **Crypto provider** | Who performs the cryptographic operations (key exchange, symmetric ciphers, hashing) | `aws-lc-rs` (default in `rustls`/`webpki-roots`), `ring`, or any custom `CryptoProvider` |
| **Certificate verification** | How the server's TLS certificate chain is validated | Platform verifier (default in `rustls`), bundled Mozilla roots (`webpki-roots`), or custom |

The `native-tls` feature sidesteps both axes by delegating everything to
the OS TLS library (SChannel / Secure Transport / OpenSSL).

### Feature matrix
Comment thread
Xuanwo marked this conversation as resolved.

| Feature | Crypto provider | Certificate roots | Use when |
|---------|----------------|-------------------|----------|
| `native-tls` (default) | OS library | OS trust store | You want zero Rust-side TLS config |
Comment thread
Xuanwo marked this conversation as resolved.
Outdated
| `rustls` | aws-lc-rs | Platform verifier | Pure-Rust TLS with OS trust store |
| `webpki-roots` | aws-lc-rs | Bundled Mozilla roots | Fully self-contained, no OS dependency |
| `rustls-no-provider` | **you provide** | **you provide** | BYO crypto (ring, FIPS module, etc.) |

### Usage via the `opendal` facade crate

Most users depend on `opendal` rather than this crate directly. The facade
installs this transport when any `http-transport-reqwest-*` feature is enabled.

```toml
# Default — reqwest transport with native-tls
opendal = { version = "0.57" }
```

To select a different TLS backend, disable default features and enable the
one you need:

```toml
opendal = { version = "0.57", default-features = false, features = ["http-transport-reqwest-rustls"] }
```

### Feature usage with `rustls`

```toml
[dependencies]
opendal-http-transport-reqwest = { version = "0.57", default-features = false, features = ["rustls"] }
```

```rust
use std::time::Duration;

use opendal_http_transport_reqwest::ReqwestTlsBackend;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't expect we need to implement this. Is there a reason why we didn't just use things from reqwest?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is mainly a consideration of what features do we want to compile and ship. e.g.:

  1. we ship rustls, ring, all differents certs
  2. we ship one feature of rustls
  3. ...other combinations between these 2

Bindings will use this crate mostly so compilation size is also a concern.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I realized that I'm writing a library instead of an app. I'll fix this soon.

use opendal_http_transport_reqwest::ReqwestTransport;
use opendal::HttpTransporter;

// You can configure reqwest dynamically and select a compiled TLS backend.
let tls_backend = "rustls".parse::<ReqwestTlsBackend>().unwrap();
let transport = ReqwestTransport::builder()
.tls_backend(tls_backend)
.configure(|builder| builder.connect_timeout(Duration::from_secs(10)))
.build()
.unwrap();
```

### Bringing your own reqwest client

When you need full control over the TLS stack — custom `ClientConfig`,
client certificates, proxy settings, or connection pool tuning — build a
`reqwest::Client` yourself and wrap it:

```toml
[dependencies]
opendal = { version = "0.57", default-features = false, features = [
"services-s3",
"http-transport-reqwest-rustls-no-provider",
] }
opendal-http-transport-reqwest = { version = "0.57", default-features = false, features = ["rustls-no-provider"] }
rustls = { version = "0.23", features = ["ring"], default-features = false }
webpki-roots = "1"
```

```rust
use std::time::Duration;

use opendal::HttpTransporter;
use opendal::OperationContext;
use opendal_http_transport_reqwest::ReqwestTransport;

fn main() {
// 1. Configure your crypto provider and certificate roots.
let root_store =
rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());

let tls_config = rustls::ClientConfig::builder_with_provider(
rustls::crypto::ring::default_provider().into(),
)
.with_safe_default_protocol_versions()
.unwrap()
.with_root_certificates(root_store)
.with_no_client_auth();

// 2. Build a reqwest client with your TLS config.
let client = reqwest::Client::builder()
.tls_backend_preconfigured(tls_config)
.connect_timeout(Duration::from_secs(10))
.pool_max_idle_per_host(20)
.build()
.unwrap();

// 3. Wrap it as a ReqwestTransport and attach to an operator.
let transport = HttpTransporter::new(ReqwestTransport::new(client));

let op = opendal::Operator::via_iter("s3", [
("bucket".to_string(), "my-bucket".to_string()),
("region".to_string(), "us-east-1".to_string()),
])
.expect("failed to build operator")
.with_context(OperationContext::new().with_http_transport(transport));
}
```

This approach gives you complete ownership over TLS and client.

## License and Trademarks

Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.
Loading
Loading