From 6b067f3d14bd109af3705988e5cb28f620419d55 Mon Sep 17 00:00:00 2001 From: yoshidan Date: Thu, 30 Nov 2023 22:27:35 +0900 Subject: [PATCH 1/7] allow anonymouse access --- foundation/token/src/lib.rs | 2 +- storage/Cargo.toml | 3 ++- storage/README.md | 13 +++++++++++++ storage/src/client.rs | 27 +++++++++++++++++++++++++++ storage/src/http/storage_client.rs | 11 ++++++++--- storage/src/lib.rs | 14 ++++++++++++++ storage/src/token_source.rs | 22 ++++++++++++++++++++++ 7 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 storage/src/token_source.rs diff --git a/foundation/token/src/lib.rs b/foundation/token/src/lib.rs index 51b501af..7dceaa30 100644 --- a/foundation/token/src/lib.rs +++ b/foundation/token/src/lib.rs @@ -19,6 +19,6 @@ pub struct NopeTokenSourceProvider {} impl TokenSourceProvider for NopeTokenSourceProvider { fn token_source(&self) -> Arc { - panic!("This is dummy token source provider. you can use 'google_cloud_default' crate") + panic!("This is dummy token source provider. you can use 'google_cloud_auth' crate") } } diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 88e6f6c9..ace35dcb 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-cloud-storage" -version = "0.14.0" +version = "0.15.0" edition = "2021" authors = ["yoshidan "] repository = "https://github.com/yoshidan/google-cloud-rust/tree/main/storage" @@ -31,6 +31,7 @@ serde_json = "1.0" percent-encoding = "2.3" futures-util = "0.3" bytes = "1.5" +async-trait = "0.1" google-cloud-metadata = { optional = true, version = "0.4", path = "../foundation/metadata" } google-cloud-auth = { optional = true, version = "0.13", path="../foundation/auth", default-features=false } diff --git a/storage/README.md b/storage/README.md index 6fe82d88..7ed011d6 100644 --- a/storage/README.md +++ b/storage/README.md @@ -51,6 +51,19 @@ async fn run(cred: CredentialsFile) { } ``` +### Anonymous Access + +To provide [anonymous access without authentication](https://cloud.google.com/storage/docs/authentication), do the following. + +```rust +use google_cloud_storage::client::{ClientConfig, Client}; + +async fn run() { + let config = ClientConfig::default().anonymous(); + let client = Client::new(config); +} +``` + ### Usage ```rust diff --git a/storage/src/client.rs b/storage/src/client.rs index 4b8dac05..5ab6f201 100644 --- a/storage/src/client.rs +++ b/storage/src/client.rs @@ -35,6 +35,14 @@ impl Default for ClientConfig { } } +impl ClientConfig { + pub fn anonymous(mut self) -> Self { + self.token_source_provider = Box::new(AnonymousTokenSourceProvider {}); + self + } +} + +use crate::token_source::AnonymousTokenSourceProvider; #[cfg(feature = "auth")] pub use google_cloud_auth; @@ -249,6 +257,7 @@ mod test { use serial_test::serial; use crate::client::{Client, ClientConfig}; + use crate::http::buckets::get::GetBucketRequest; use crate::http::storage_client::test::bucket_name; use crate::sign::{SignedURLMethod, SignedURLOptions}; @@ -371,4 +380,22 @@ mod test { .unwrap(); assert_eq!(result, data); } + + #[tokio::test] + #[serial] + async fn test_anonymous() { + let project = ClientConfig::default().with_auth().await.unwrap().project_id.unwrap(); + let bucket = bucket_name(&project, "anonymous"); + + let config = ClientConfig::default().anonymous(); + let client = Client::new(config); + let result = client + .get_bucket(&GetBucketRequest { + bucket: bucket.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(result.name, bucket); + } } diff --git a/storage/src/http/storage_client.rs b/storage/src/http/storage_client.rs index 5b3bf963..13858e5c 100644 --- a/storage/src/http/storage_client.rs +++ b/storage/src/http/storage_client.rs @@ -1283,10 +1283,15 @@ impl StorageClient { async fn with_headers(&self, builder: RequestBuilder) -> Result { let token = self.ts.token().await.map_err(Error::TokenSource)?; - Ok(builder + let builder = builder .header("X-Goog-Api-Client", "rust") - .header(reqwest::header::USER_AGENT, "google-cloud-storage") - .header(reqwest::header::AUTHORIZATION, token)) + .header(reqwest::header::USER_AGENT, "google-cloud-storage"); + if !token.is_empty() { + Ok(builder.header(reqwest::header::AUTHORIZATION, token)) + } else { + tracing::trace!("Use anonymous access due to lack of token"); + Ok(builder) + } } async fn send_request(&self, request: Request) -> Result diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 468ecc3d..16451033 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -42,6 +42,19 @@ //! } //! ``` //! +//! ### Anonymous Access +//! +//! To provide [anonymous access without authentication](https://cloud.google.com/storage/docs/authentication), do the following. +//! +//! ```rust +//! use google_cloud_storage::client::{ClientConfig, Client}; +//! +//! async fn run() { +//! let config = ClientConfig::default().anonymous(); +//! let client = Client::new(config); +//! } +//! ``` +//! //! ### Usage //! //! ``` @@ -93,3 +106,4 @@ extern crate core; pub mod client; pub mod http; pub mod sign; +mod token_source; diff --git a/storage/src/token_source.rs b/storage/src/token_source.rs new file mode 100644 index 00000000..dd7f9386 --- /dev/null +++ b/storage/src/token_source.rs @@ -0,0 +1,22 @@ +use google_cloud_token::{TokenSource, TokenSourceProvider}; +use std::error::Error; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct AnonymousTokenSource {} + +#[async_trait::async_trait] +impl TokenSource for AnonymousTokenSource { + async fn token(&self) -> Result> { + Ok("".to_string()) + } +} + +#[derive(Debug)] +pub struct AnonymousTokenSourceProvider {} + +impl TokenSourceProvider for AnonymousTokenSourceProvider { + fn token_source(&self) -> Arc { + Arc::new(AnonymousTokenSource {}) + } +} From f6a4332c413ae7f9a4cd752277e8fb4a06db1887 Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 09:45:00 +0900 Subject: [PATCH 2/7] remove token source --- storage/src/client.rs | 17 +++++++++++------ storage/src/http/service_account_client.rs | 17 +++++++++++------ storage/src/http/storage_client.rs | 21 +++++++++++---------- storage/src/lib.rs | 1 - storage/src/token_source.rs | 22 ---------------------- 5 files changed, 33 insertions(+), 45 deletions(-) delete mode 100644 storage/src/token_source.rs diff --git a/storage/src/client.rs b/storage/src/client.rs index 5ab6f201..4a3e4fea 100644 --- a/storage/src/client.rs +++ b/storage/src/client.rs @@ -15,7 +15,7 @@ pub struct ClientConfig { pub http: Option, pub storage_endpoint: String, pub service_account_endpoint: String, - pub token_source_provider: Box, + pub token_source_provider: Option>, pub default_google_access_id: Option, pub default_sign_by: Option, pub project_id: Option, @@ -26,7 +26,7 @@ impl Default for ClientConfig { Self { http: None, storage_endpoint: "https://storage.googleapis.com".to_string(), - token_source_provider: Box::new(NopeTokenSourceProvider {}), + token_source_provider: Some(Box::new(NopeTokenSourceProvider {})), service_account_endpoint: "https://iamcredentials.googleapis.com".to_string(), default_google_access_id: None, default_sign_by: None, @@ -37,12 +37,11 @@ impl Default for ClientConfig { impl ClientConfig { pub fn anonymous(mut self) -> Self { - self.token_source_provider = Box::new(AnonymousTokenSourceProvider {}); + self.token_source_provider = None; self } } -use crate::token_source::AnonymousTokenSourceProvider; #[cfg(feature = "auth")] pub use google_cloud_auth; @@ -82,7 +81,7 @@ impl ClientConfig { self.default_google_access_id = google_cloud_metadata::email("default").await.ok(); } } - self.token_source_provider = Box::new(ts); + self.token_source_provider = Some(Box::new(ts)); self } @@ -120,7 +119,13 @@ impl Default for Client { impl Client { /// New client pub fn new(config: ClientConfig) -> Self { - let ts = config.token_source_provider.token_source(); + let ts = match config.token_source_provider { + Some(tsp) => Some(tsp.token_source()), + None => { + tracing::trace!("Use anonymous access due to lack of token"); + None + } + }; let http = config.http.unwrap_or_default(); let service_account_client = diff --git a/storage/src/http/service_account_client.rs b/storage/src/http/service_account_client.rs index 34305eb8..563f5ecb 100644 --- a/storage/src/http/service_account_client.rs +++ b/storage/src/http/service_account_client.rs @@ -6,13 +6,13 @@ use crate::http::{check_response_status, Error}; #[derive(Clone)] pub struct ServiceAccountClient { - ts: Arc, + ts: Option>, v1_endpoint: String, http: reqwest::Client, } impl ServiceAccountClient { - pub(crate) fn new(ts: Arc, endpoint: &str, http: reqwest::Client) -> Self { + pub(crate) fn new(ts: Option>, endpoint: &str, http: reqwest::Client) -> Self { Self { ts, v1_endpoint: format!("{endpoint}/v1"), @@ -24,14 +24,19 @@ impl ServiceAccountClient { pub async fn sign_blob(&self, name: &str, payload: &[u8]) -> Result, Error> { let url = format!("{}/{}:signBlob", self.v1_endpoint, name); let request = SignBlobRequest { payload }; - let token = self.ts.token().await.map_err(Error::TokenSource)?; let request = self .http .post(url) .json(&request) .header("X-Goog-Api-Client", "rust") - .header(reqwest::header::USER_AGENT, "google-cloud-storage") - .header(reqwest::header::AUTHORIZATION, token); + .header(reqwest::header::USER_AGENT, "google-cloud-storage"); + let request = match &self.ts { + Some(ts) => { + let token = ts.token().await.map_err(Error::TokenSource)?; + request .header(reqwest::header::AUTHORIZATION, token) + }, + None => request + }; let response = request.send().await?; let response = check_response_status(response).await?; Ok(response.json::().await?.signed_blob) @@ -73,7 +78,7 @@ mod test { let email = tsp.source_credentials.clone().unwrap().client_email.unwrap(); let ts = tsp.token_source(); ( - ServiceAccountClient::new(ts, "https://iamcredentials.googleapis.com", Client::default()), + ServiceAccountClient::new(Some(ts), "https://iamcredentials.googleapis.com", Client::default()), email, ) } diff --git a/storage/src/http/storage_client.rs b/storage/src/http/storage_client.rs index 13858e5c..0e4edf95 100644 --- a/storage/src/http/storage_client.rs +++ b/storage/src/http/storage_client.rs @@ -68,14 +68,14 @@ pub const SCOPES: [&str; 2] = [ #[derive(Clone)] pub struct StorageClient { - ts: Arc, + ts: Option>, v1_endpoint: String, v1_upload_endpoint: String, http: Client, } impl StorageClient { - pub(crate) fn new(ts: Arc, endpoint: &str, http: Client) -> Self { + pub(crate) fn new(ts: Option>, endpoint: &str, http: Client) -> Self { Self { ts, v1_endpoint: format!("{endpoint}/storage/v1"), @@ -1282,16 +1282,17 @@ impl StorageClient { } async fn with_headers(&self, builder: RequestBuilder) -> Result { - let token = self.ts.token().await.map_err(Error::TokenSource)?; let builder = builder .header("X-Goog-Api-Client", "rust") .header(reqwest::header::USER_AGENT, "google-cloud-storage"); - if !token.is_empty() { - Ok(builder.header(reqwest::header::AUTHORIZATION, token)) - } else { - tracing::trace!("Use anonymous access due to lack of token"); - Ok(builder) - } + let builder = match &self.ts { + Some(ts) => { + let token = ts.token().await.map_err(Error::TokenSource)?; + builder.header(reqwest::header::AUTHORIZATION, token) + }, + None => builder + }; + Ok(builder) } async fn send_request(&self, request: Request) -> Result @@ -1414,7 +1415,7 @@ pub(crate) mod test { .unwrap(); let cred = tsp.source_credentials.clone(); let ts = tsp.token_source(); - let client = StorageClient::new(ts, "https://storage.googleapis.com", reqwest::Client::new()); + let client = StorageClient::new(Some(ts), "https://storage.googleapis.com", reqwest::Client::new()); let cred = cred.unwrap(); (client, cred.project_id.unwrap(), cred.client_email.unwrap()) } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 16451033..4d2e2973 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -106,4 +106,3 @@ extern crate core; pub mod client; pub mod http; pub mod sign; -mod token_source; diff --git a/storage/src/token_source.rs b/storage/src/token_source.rs deleted file mode 100644 index dd7f9386..00000000 --- a/storage/src/token_source.rs +++ /dev/null @@ -1,22 +0,0 @@ -use google_cloud_token::{TokenSource, TokenSourceProvider}; -use std::error::Error; -use std::sync::Arc; - -#[derive(Debug, Clone)] -pub struct AnonymousTokenSource {} - -#[async_trait::async_trait] -impl TokenSource for AnonymousTokenSource { - async fn token(&self) -> Result> { - Ok("".to_string()) - } -} - -#[derive(Debug)] -pub struct AnonymousTokenSourceProvider {} - -impl TokenSourceProvider for AnonymousTokenSourceProvider { - fn token_source(&self) -> Arc { - Arc::new(AnonymousTokenSource {}) - } -} From fe28f0dd1c3a3d198ff5faf5531b8a0ee93f65e7 Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 09:45:43 +0900 Subject: [PATCH 3/7] fmt --- storage/src/http/service_account_client.rs | 6 +++--- storage/src/http/storage_client.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/storage/src/http/service_account_client.rs b/storage/src/http/service_account_client.rs index 563f5ecb..df4b660c 100644 --- a/storage/src/http/service_account_client.rs +++ b/storage/src/http/service_account_client.rs @@ -33,9 +33,9 @@ impl ServiceAccountClient { let request = match &self.ts { Some(ts) => { let token = ts.token().await.map_err(Error::TokenSource)?; - request .header(reqwest::header::AUTHORIZATION, token) - }, - None => request + request.header(reqwest::header::AUTHORIZATION, token) + } + None => request, }; let response = request.send().await?; let response = check_response_status(response).await?; diff --git a/storage/src/http/storage_client.rs b/storage/src/http/storage_client.rs index 0e4edf95..65466d92 100644 --- a/storage/src/http/storage_client.rs +++ b/storage/src/http/storage_client.rs @@ -1289,8 +1289,8 @@ impl StorageClient { Some(ts) => { let token = ts.token().await.map_err(Error::TokenSource)?; builder.header(reqwest::header::AUTHORIZATION, token) - }, - None => builder + } + None => builder, }; Ok(builder) } From 80eb89c864650e0a3ef740d154027eaff530e030 Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 14:51:55 +0900 Subject: [PATCH 4/7] update deps --- storage/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/Cargo.toml b/storage/Cargo.toml index ace35dcb..c4db503a 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -12,13 +12,13 @@ documentation = "https://docs.rs/google-cloud-storage/latest/google_cloud_storag [dependencies] google-cloud-token = { version = "0.1.1", path = "../foundation/token" } -rsa = "0.6" +rsa = "0.9" thiserror = "1.0" time = { version = "0.3", features = ["std", "macros", "formatting", "parsing", "serde"] } base64 = "0.21" regex = "1.9" sha2 = "0.10" -ring = "0.16" +ring = "0.17" tokio = { version="1.32", features=["macros"] } async-stream = "0.3" once_cell = "1.18" From 90713353c23c571f8ce1f983023f62438907705e Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 15:40:00 +0900 Subject: [PATCH 5/7] fix rsa --- storage/src/client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/src/client.rs b/storage/src/client.rs index 4a3e4fea..fd4f1eb2 100644 --- a/storage/src/client.rs +++ b/storage/src/client.rs @@ -228,9 +228,9 @@ impl Client { let der = pkcs .to_pkcs8_der() .map_err(|e| SignedURLError::CertError(e.to_string()))?; - let key_pair = ring::signature::RsaKeyPair::from_pkcs8(der.as_ref()) + let key_pair = ring::signature::RsaKeyPair::from_pkcs8(der.as_bytes()) .map_err(|e| SignedURLError::CertError(e.to_string()))?; - let mut signed = vec![0; key_pair.public_modulus_len()]; + let mut signed = vec![0; key_pair.public().modulus_len()]; key_pair .sign( &signature::RSA_PKCS1_SHA256, From 85a3ce7e1d9de7c8095a00874c634ce796cf84fd Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 16:14:22 +0900 Subject: [PATCH 6/7] allow rsa and add todo --- deny.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index 50c15096..0012e4c3 100644 --- a/deny.toml +++ b/deny.toml @@ -74,7 +74,8 @@ notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - #"RUSTSEC-0000-0000", + # TODO fix https://github.com/RustCrypto/RSA/issues/19#issuecomment-1822995643 + "RUSTSEC-2023-0071", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories From 21b1975bac99301261842d3276cffd78af28ad4b Mon Sep 17 00:00:00 2001 From: yoshidan Date: Fri, 1 Dec 2023 22:02:00 +0900 Subject: [PATCH 7/7] remove rsa crate --- deny.toml | 3 +-- storage/Cargo.toml | 2 +- storage/src/client.rs | 13 ++----------- storage/src/sign.rs | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/deny.toml b/deny.toml index 0012e4c3..50c15096 100644 --- a/deny.toml +++ b/deny.toml @@ -74,8 +74,7 @@ notice = "warn" # A list of advisory IDs to ignore. Note that ignored advisories will still # output a note when they are encountered. ignore = [ - # TODO fix https://github.com/RustCrypto/RSA/issues/19#issuecomment-1822995643 - "RUSTSEC-2023-0071", + #"RUSTSEC-0000-0000", ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories diff --git a/storage/Cargo.toml b/storage/Cargo.toml index c4db503a..7d3e7fea 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/google-cloud-storage/latest/google_cloud_storag [dependencies] google-cloud-token = { version = "0.1.1", path = "../foundation/token" } -rsa = "0.9" +pkcs8 = {version="0.10", features=["pem"]} thiserror = "1.0" time = { version = "0.3", features = ["std", "macros", "formatting", "parsing", "serde"] } base64 = "0.21" diff --git a/storage/src/client.rs b/storage/src/client.rs index fd4f1eb2..2ea19d79 100644 --- a/storage/src/client.rs +++ b/storage/src/client.rs @@ -1,14 +1,13 @@ use std::ops::Deref; use ring::{rand, signature}; -use rsa::pkcs8::{DecodePrivateKey, EncodePrivateKey}; use google_cloud_token::{NopeTokenSourceProvider, TokenSourceProvider}; use crate::http::service_account_client::ServiceAccountClient; use crate::http::storage_client::StorageClient; use crate::sign::SignBy::PrivateKey; -use crate::sign::{create_signed_buffer, SignBy, SignedURLError, SignedURLOptions}; +use crate::sign::{create_signed_buffer, RsaKeyPair, SignBy, SignedURLError, SignedURLOptions}; #[derive(Debug)] pub struct ClientConfig { @@ -221,15 +220,7 @@ impl Client { if private_key.is_empty() { return Err(SignedURLError::InvalidOption("No keys present")); } - - let str = String::from_utf8_lossy(private_key); - let pkcs = rsa::RsaPrivateKey::from_pkcs8_pem(str.as_ref()) - .map_err(|e| SignedURLError::CertError(e.to_string()))?; - let der = pkcs - .to_pkcs8_der() - .map_err(|e| SignedURLError::CertError(e.to_string()))?; - let key_pair = ring::signature::RsaKeyPair::from_pkcs8(der.as_bytes()) - .map_err(|e| SignedURLError::CertError(e.to_string()))?; + let key_pair = &RsaKeyPair::try_from(private_key)?; let mut signed = vec![0; key_pair.public().modulus_len()]; key_pair .sign( diff --git a/storage/src/sign.rs b/storage/src/sign.rs index 1cc4bf86..40a8a3c8 100644 --- a/storage/src/sign.rs +++ b/storage/src/sign.rs @@ -1,9 +1,12 @@ use std::collections::HashMap; use std::fmt::{Debug, Formatter}; +use std::ops::Deref; use std::time::Duration; use base64::prelude::*; use once_cell::sync::Lazy; +use pkcs8::der::pem::PemLabel; +use pkcs8::SecretDocument; use regex::Regex; use sha2::{Digest, Sha256}; use time::format_description::well_known::iso8601::{EncodedConfig, TimePrecision}; @@ -326,6 +329,35 @@ fn validate_options(opts: &SignedURLOptions) -> Result<(), SignedURLError> { Ok(()) } +pub struct RsaKeyPair { + inner: ring::signature::RsaKeyPair, +} + +impl PemLabel for RsaKeyPair { + const PEM_LABEL: &'static str = "PRIVATE KEY"; +} + +impl TryFrom<&Vec> for RsaKeyPair { + type Error = SignedURLError; + + fn try_from(pem: &Vec) -> Result { + let str = String::from_utf8_lossy(pem); + let (label, doc) = SecretDocument::from_pem(&str).map_err(|v| SignedURLError::CertError(v.to_string()))?; + Self::validate_pem_label(label).map_err(|_| SignedURLError::CertError(label.to_string()))?; + let key_pair = ring::signature::RsaKeyPair::from_pkcs8(doc.as_bytes()) + .map_err(|e| SignedURLError::CertError(e.to_string()))?; + Ok(Self { inner: key_pair }) + } +} + +impl Deref for RsaKeyPair { + type Target = ring::signature::RsaKeyPair; + + fn deref(&self) -> &ring::signature::RsaKeyPair { + &self.inner + } +} + #[cfg(test)] mod test { use std::collections::HashMap;