From bfe4a473d7c049a51150b466598198568705aa78 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 4 Apr 2025 11:48:03 +0300 Subject: [PATCH 1/3] increase mainline-content-discovery version and publish it --- content-discovery/iroh-mainline-content-discovery/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content-discovery/iroh-mainline-content-discovery/Cargo.toml b/content-discovery/iroh-mainline-content-discovery/Cargo.toml index 8388f2f..44bac6e 100644 --- a/content-discovery/iroh-mainline-content-discovery/Cargo.toml +++ b/content-discovery/iroh-mainline-content-discovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "iroh-mainline-content-discovery" -version = "0.5.0" +version = "0.6.0" edition = "2021" description = "Content discovery for iroh, using the bittorrent mainline DHT" license = "MIT OR Apache-2.0" From be7c5b0ef57d4cbd1f21bff09275f8f580969692 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 4 Apr 2025 12:24:54 +0300 Subject: [PATCH 2/3] Eliminate the tls crate. It is a dependency of iroh-mainline-content-discovery, yet I don't want to publish it. So I can't publish iroh-mainline-content-discovery itself. I moved all the stuff into iroh-mainline-content-discovery under the feature flag --- content-discovery/Cargo.toml | 1 - .../Cargo.toml | 31 +++++++++++++++++-- .../iroh-mainline-content-discovery/README.md | 9 ++++++ .../src/client.rs | 30 +++++++++--------- .../src/lib.rs | 2 ++ .../src/tls_utils}/certificate.rs | 0 .../src/tls_utils/mod.rs} | 0 .../src/tls_utils}/verifier.rs | 0 .../iroh-mainline-tracker/Cargo.toml | 3 +- .../iroh-mainline-tracker/src/main.rs | 14 ++++----- .../iroh-mainline-tracker/src/tracker.rs | 19 ++++++------ content-discovery/tls/Cargo.toml | 22 ------------- 12 files changed, 71 insertions(+), 60 deletions(-) create mode 100644 content-discovery/iroh-mainline-content-discovery/README.md rename content-discovery/{tls/src => iroh-mainline-content-discovery/src/tls_utils}/certificate.rs (100%) rename content-discovery/{tls/src/lib.rs => iroh-mainline-content-discovery/src/tls_utils/mod.rs} (100%) rename content-discovery/{tls/src => iroh-mainline-content-discovery/src/tls_utils}/verifier.rs (100%) delete mode 100644 content-discovery/tls/Cargo.toml diff --git a/content-discovery/Cargo.toml b/content-discovery/Cargo.toml index 6510ee8..6fc1258 100644 --- a/content-discovery/Cargo.toml +++ b/content-discovery/Cargo.toml @@ -3,7 +3,6 @@ members = [ "iroh-mainline-content-discovery", "iroh-mainline-content-discovery-cli", "iroh-mainline-tracker", - "tls", ] resolver = "2" diff --git a/content-discovery/iroh-mainline-content-discovery/Cargo.toml b/content-discovery/iroh-mainline-content-discovery/Cargo.toml index 44bac6e..bbebca6 100644 --- a/content-discovery/iroh-mainline-content-discovery/Cargo.toml +++ b/content-discovery/iroh-mainline-content-discovery/Cargo.toml @@ -22,7 +22,7 @@ hex = "0.4.3" # Optional features for the client functionality tracing = { version = "0.1", optional = true } -iroh-quinn = { version = "0.13", optional = true } +quinn = { package = "iroh-quinn", version = "0.13", optional = true } mainline = { version = "2.0.0", optional = true, features = ["async"] } anyhow = { version = "1", features = ["backtrace"], optional = true } postcard = { version = "1", default-features = false, features = ["alloc", "use-std"], optional = true } @@ -32,8 +32,33 @@ rustls = { version = "0.23", default-features = false, features = ["ring"], opti genawaiter = { version = "0.99.1", features = ["futures03"], optional = true } tokio = { workspace = true, optional = true } flume = "0.11.0" -tls = { path = "../tls", optional = true } + +# dependencies for the tls utils +der = { version = "0.7", features = ["alloc", "derive"], optional = true } +webpki = { package = "rustls-webpki", version = "0.102", optional = true } +x509-parser = { version = "0.16", optional = true } +thiserror = { version = "2", optional = true } +ring = { version = "0.17", optional = true } [features] -client = ["mainline", "iroh-quinn", "tracing", "anyhow", "rcgen", "genawaiter", "rustls", "futures", "postcard", "tokio", "tls"] +client = [ + "dep:mainline", + "dep:quinn", + "dep:tracing", + "dep:anyhow", + "dep:rcgen", + "dep:genawaiter", + "dep:rustls", + "dep:futures", + "dep:postcard", + "dep:tokio", + "tls-utils", +] +tls-utils = [ + "dep:der", + "dep:webpki", + "dep:x509-parser", + "dep:thiserror", + "dep:ring", +] default = ["client"] diff --git a/content-discovery/iroh-mainline-content-discovery/README.md b/content-discovery/iroh-mainline-content-discovery/README.md new file mode 100644 index 0000000..a458637 --- /dev/null +++ b/content-discovery/iroh-mainline-content-discovery/README.md @@ -0,0 +1,9 @@ +# Protocol and client for iroh mainline content discovery + +This provides a very minimal protocol for content discovery as well as a +client library for the protocol. + +## Features + +- client: the client that allows querying content discovery +- tls-utils: utilities to set of quinn connections, used by client diff --git a/content-discovery/iroh-mainline-content-discovery/src/client.rs b/content-discovery/iroh-mainline-content-discovery/src/client.rs index 28702dd..27ba35a 100644 --- a/content-discovery/iroh-mainline-content-discovery/src/client.rs +++ b/content-discovery/iroh-mainline-content-discovery/src/client.rs @@ -20,9 +20,9 @@ use iroh::{ }; use iroh_blobs::HashAndFormat; -use crate::protocol::{ +use crate::{protocol::{ AnnounceKind, Query, QueryResponse, Request, Response, SignedAnnounce, ALPN, REQUEST_SIZE_LIMIT, -}; +}, tls_utils}; /// Announce to a tracker. /// @@ -33,7 +33,7 @@ use crate::protocol::{ /// `content` is the content to announce. /// `kind` is the kind of the announcement. We can claim to have the complete data or only some of it. pub async fn announce_quinn( - connection: iroh_quinn::Connection, + connection: quinn::Connection, signed_announce: SignedAnnounce, ) -> anyhow::Result<()> { let (mut send, mut recv) = connection.open_bi().await?; @@ -119,14 +119,14 @@ async fn query_iroh_one( /// A connection provider that can be used to connect to a tracker. /// -/// This can either be a [`iroh_quinn::Endpoint`] where connections are created on demand, +/// This can either be a [`quinn::Endpoint`] where connections are created on demand, /// or some sort of connection pool. pub trait QuinnConnectionProvider: Clone { - fn connect(&self, addr: Addr) -> BoxFuture>; + fn connect(&self, addr: Addr) -> BoxFuture>; } -impl QuinnConnectionProvider for iroh_quinn::Endpoint { - fn connect(&self, addr: SocketAddr) -> BoxFuture> { +impl QuinnConnectionProvider for quinn::Endpoint { + fn connect(&self, addr: SocketAddr) -> BoxFuture> { async move { Ok(self.connect(addr, "localhost")?.await?) }.boxed() } } @@ -229,7 +229,7 @@ pub async fn query_iroh( /// Assume an existing connection to a tracker and query it for peers for some content. pub async fn query_quinn( - connection: iroh_quinn::Connection, + connection: quinn::Connection, args: Query, ) -> anyhow::Result { tracing::info!("connected to {:?}", connection.remote_address()); @@ -252,12 +252,12 @@ pub fn create_quinn_client( bind_addr: SocketAddr, alpn_protocols: Vec>, keylog: bool, -) -> anyhow::Result { +) -> anyhow::Result { let secret_key = iroh::SecretKey::generate(rand::thread_rng()); - let tls_client_config = tls::make_client_config(&secret_key, None, alpn_protocols, keylog)?; - let mut client_config = iroh_quinn::ClientConfig::new(Arc::new(tls_client_config)); - let mut endpoint = iroh_quinn::Endpoint::client(bind_addr)?; - let mut transport_config = iroh_quinn::TransportConfig::default(); + let tls_client_config = tls_utils::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); + let mut endpoint = quinn::Endpoint::client(bind_addr)?; + let mut transport_config = quinn::TransportConfig::default(); transport_config.keep_alive_interval(Some(Duration::from_secs(1))); client_config.transport_config(Arc::new(transport_config)); endpoint.set_default_client_config(client_config); @@ -340,7 +340,7 @@ pub async fn connect( pub enum Connection { Iroh(iroh::endpoint::Connection), - Quinn(iroh_quinn::Connection), + Quinn(quinn::Connection), } /// Create a iroh endpoint and connect to a tracker using the [crate::protocol::ALPN] protocol. @@ -363,7 +363,7 @@ async fn connect_iroh( async fn connect_socket( tracker: SocketAddr, local_addr: SocketAddr, -) -> anyhow::Result { +) -> anyhow::Result { let endpoint = create_quinn_client(local_addr, vec![ALPN.to_vec()], false)?; tracing::info!("trying t?o )connect to tracker at {:?}", tracker); let connection = endpoint.connect(tracker, "localhost")?.await?; diff --git a/content-discovery/iroh-mainline-content-discovery/src/lib.rs b/content-discovery/iroh-mainline-content-discovery/src/lib.rs index e0ccaab..fb6d859 100644 --- a/content-discovery/iroh-mainline-content-discovery/src/lib.rs +++ b/content-discovery/iroh-mainline-content-discovery/src/lib.rs @@ -8,3 +8,5 @@ mod client; pub mod protocol; #[cfg(feature = "client")] pub use client::*; +#[cfg(feature = "tls-utils")] +pub mod tls_utils; diff --git a/content-discovery/tls/src/certificate.rs b/content-discovery/iroh-mainline-content-discovery/src/tls_utils/certificate.rs similarity index 100% rename from content-discovery/tls/src/certificate.rs rename to content-discovery/iroh-mainline-content-discovery/src/tls_utils/certificate.rs diff --git a/content-discovery/tls/src/lib.rs b/content-discovery/iroh-mainline-content-discovery/src/tls_utils/mod.rs similarity index 100% rename from content-discovery/tls/src/lib.rs rename to content-discovery/iroh-mainline-content-discovery/src/tls_utils/mod.rs diff --git a/content-discovery/tls/src/verifier.rs b/content-discovery/iroh-mainline-content-discovery/src/tls_utils/verifier.rs similarity index 100% rename from content-discovery/tls/src/verifier.rs rename to content-discovery/iroh-mainline-content-discovery/src/tls_utils/verifier.rs diff --git a/content-discovery/iroh-mainline-tracker/Cargo.toml b/content-discovery/iroh-mainline-tracker/Cargo.toml index 8f4cbac..e20ad62 100644 --- a/content-discovery/iroh-mainline-tracker/Cargo.toml +++ b/content-discovery/iroh-mainline-tracker/Cargo.toml @@ -23,7 +23,6 @@ iroh-blobs = { workspace = true } mainline = { version = "2.0.0", features = ["async"] } pkarr = { version = "2.0.1", features = ["async"] } postcard = { version = "1", default-features = false, features = ["alloc", "use-std"] } -iroh-quinn = "0.13" rand = "0.8" rcgen = "0.12.0" redb = "1.5.0" @@ -42,7 +41,7 @@ url = "2.5.0" flume = "0.11.0" genawaiter = { version = "0.99.1", features = ["futures03"] } iroh-mainline-content-discovery = { path = "../iroh-mainline-content-discovery", features = ["client"] } -tls = { path = "../tls" } +quinn = { package = "iroh-quinn", version = "0.13" } clap = { version = "4", features = ["derive"], optional = true } serde-big-array = "0.5.1" diff --git a/content-discovery/iroh-mainline-tracker/src/main.rs b/content-discovery/iroh-mainline-tracker/src/main.rs index 8ace4f4..0828573 100644 --- a/content-discovery/iroh-mainline-tracker/src/main.rs +++ b/content-discovery/iroh-mainline-tracker/src/main.rs @@ -12,7 +12,7 @@ use std::{ use clap::Parser; use iroh::{discovery::pkarr::dht::DhtDiscovery, Endpoint, NodeId}; use iroh_blobs::util::fs::load_secret_key; -use iroh_mainline_content_discovery::protocol::ALPN; +use iroh_mainline_content_discovery::{protocol::ALPN, tls_utils}; use iroh_mainline_tracker::{ io::{ self, load_from_file, setup_logging, tracker_home, tracker_path, CONFIG_DEBUG_FILE, @@ -130,7 +130,7 @@ async fn server(args: Args) -> anyhow::Result<()> { let udp_socket = tokio::net::UdpSocket::bind(udp_bind_addr).await?; let quinn_bind_addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, options.quinn_port)); - let quinn_endpoint = iroh_quinn::Endpoint::server(server_config, quinn_bind_addr)?; + let quinn_endpoint = quinn::Endpoint::server(server_config, quinn_bind_addr)?; // set the quinn port to the actual port we bound to so the DHT will announce it correctly options.quinn_port = quinn_endpoint.local_addr()?.port(); let iroh_endpoint = create_endpoint(key.clone(), options.iroh_ipv4_addr, true).await?; @@ -185,7 +185,7 @@ async fn main() -> anyhow::Result<()> { /// Returns default server configuration along with its certificate. #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 -fn configure_server(secret_key: &iroh::SecretKey) -> anyhow::Result { +fn configure_server(secret_key: &iroh::SecretKey) -> anyhow::Result { make_server_config(secret_key, 8, 1024, vec![ALPN.to_vec()]) } @@ -195,10 +195,10 @@ pub fn make_server_config( max_streams: u64, max_connections: u32, alpn_protocols: Vec>, -) -> anyhow::Result { - let tls_server_config = tls::make_server_config(secret_key, alpn_protocols, false)?; - let mut server_config = iroh_quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); - let mut transport_config = iroh_quinn::TransportConfig::default(); +) -> anyhow::Result { + let tls_server_config = tls_utils::make_server_config(secret_key, alpn_protocols, false)?; + let mut server_config = quinn::ServerConfig::with_crypto(Arc::new(tls_server_config)); + let mut transport_config = quinn::TransportConfig::default(); transport_config .max_concurrent_bidi_streams(max_streams.try_into()?) .max_concurrent_uni_streams(0u32.into()); diff --git a/content-discovery/iroh-mainline-tracker/src/tracker.rs b/content-discovery/iroh-mainline-tracker/src/tracker.rs index 41f97b0..ae5f2f4 100644 --- a/content-discovery/iroh-mainline-tracker/src/tracker.rs +++ b/content-discovery/iroh-mainline-tracker/src/tracker.rs @@ -17,8 +17,7 @@ use iroh_mainline_content_discovery::{ protocol::{ AbsoluteTime, Announce, AnnounceKind, Query, QueryResponse, Request, Response, SignedAnnounce, REQUEST_SIZE_LIMIT, - }, - to_infohash, + }, tls_utils, to_infohash }; use rand::Rng; use redb::{ReadableTable, RedbValue}; @@ -883,7 +882,7 @@ impl Tracker { Ok(()) } - pub async fn quinn_accept_loop(self, endpoint: iroh_quinn::Endpoint) -> std::io::Result<()> { + pub async fn quinn_accept_loop(self, endpoint: quinn::Endpoint) -> std::io::Result<()> { let local_addr = endpoint.local_addr()?; println!("quinn listening on {local_addr:?}"); while let Some(incoming) = endpoint.accept().await { @@ -948,7 +947,7 @@ impl Tracker { /// Handle a single incoming connection on the tracker ALPN. pub async fn handle_quinn_connection( &self, - connection: iroh_quinn::Connection, + connection: quinn::Connection, ) -> anyhow::Result<()> { tracing::debug!("calling accept_bi"); let (mut send, mut recv) = connection.accept_bi().await?; @@ -1269,8 +1268,8 @@ impl Tracker { /// Accept an incoming connection and extract the client-provided [`NodeId`] and ALPN protocol. async fn accept_conn( - mut conn: iroh_quinn::Connecting, -) -> anyhow::Result<(NodeId, String, iroh_quinn::Connection)> { + mut conn: quinn::Connecting, +) -> anyhow::Result<(NodeId, String, quinn::Connection)> { let alpn = get_alpn(&mut conn).await?; let conn = conn.await?; let node_id = get_remote_node_id(&conn)?; @@ -1278,9 +1277,9 @@ async fn accept_conn( } /// Extract the ALPN protocol from the peer's TLS certificate. -pub async fn get_alpn(connecting: &mut iroh_quinn::Connecting) -> anyhow::Result { +pub async fn get_alpn(connecting: &mut quinn::Connecting) -> anyhow::Result { let data = connecting.handshake_data().await?; - match data.downcast::() { + match data.downcast::() { Ok(data) => match data.protocol { Some(protocol) => std::string::String::from_utf8(protocol).map_err(Into::into), None => anyhow::bail!("no ALPN protocol available"), @@ -1289,7 +1288,7 @@ pub async fn get_alpn(connecting: &mut iroh_quinn::Connecting) -> anyhow::Result } } -pub fn get_remote_node_id(connection: &iroh_quinn::Connection) -> anyhow::Result { +pub fn get_remote_node_id(connection: &quinn::Connection) -> anyhow::Result { let data = connection.peer_identity(); match data { None => anyhow::bail!("no peer certificate found"), @@ -1301,7 +1300,7 @@ pub fn get_remote_node_id(connection: &iroh_quinn::Connection) -> anyhow::Result certs.len() ); } - let cert = tls::certificate::parse(&certs[0])?; + let cert = tls_utils::certificate::parse(&certs[0])?; Ok(cert.peer_id()) } Err(_) => anyhow::bail!("invalid peer certificate"), diff --git a/content-discovery/tls/Cargo.toml b/content-discovery/tls/Cargo.toml deleted file mode 100644 index 6790d65..0000000 --- a/content-discovery/tls/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "tls" -version = "0.1.0" -edition = "2021" -description = "create tls configuration for quic connections" -license = "MIT OR Apache-2.0" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -iroh-base = { workspace = true } -der = { version = "0.7", features = ["alloc", "derive"] } -derive_more = { version = "1.0.0-beta.1", features = ["debug", "display", "from", "try_into"] } -quinn = { package = "iroh-quinn", version = "0.13.0" } -rand = "0.8.5" -rcgen = "0.13" -ring = "0.17" -rustls = { version = "0.23", default-features = false, features = ["ring"] } -thiserror = "2" -tracing = "0.1" -webpki = { package = "rustls-webpki", version = "0.102" } -x509-parser = "0.16" From c565dc03c3cec727f58552b78b1eb837af153335 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Fri, 4 Apr 2025 12:28:08 +0300 Subject: [PATCH 3/3] fmt --- .../iroh-mainline-content-discovery/src/client.rs | 13 +++++++++---- .../iroh-mainline-tracker/src/tracker.rs | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/content-discovery/iroh-mainline-content-discovery/src/client.rs b/content-discovery/iroh-mainline-content-discovery/src/client.rs index 27ba35a..57faed9 100644 --- a/content-discovery/iroh-mainline-content-discovery/src/client.rs +++ b/content-discovery/iroh-mainline-content-discovery/src/client.rs @@ -20,9 +20,13 @@ use iroh::{ }; use iroh_blobs::HashAndFormat; -use crate::{protocol::{ - AnnounceKind, Query, QueryResponse, Request, Response, SignedAnnounce, ALPN, REQUEST_SIZE_LIMIT, -}, tls_utils}; +use crate::{ + protocol::{ + AnnounceKind, Query, QueryResponse, Request, Response, SignedAnnounce, ALPN, + REQUEST_SIZE_LIMIT, + }, + tls_utils, +}; /// Announce to a tracker. /// @@ -254,7 +258,8 @@ pub fn create_quinn_client( keylog: bool, ) -> anyhow::Result { let secret_key = iroh::SecretKey::generate(rand::thread_rng()); - let tls_client_config = tls_utils::make_client_config(&secret_key, None, alpn_protocols, keylog)?; + let tls_client_config = + tls_utils::make_client_config(&secret_key, None, alpn_protocols, keylog)?; let mut client_config = quinn::ClientConfig::new(Arc::new(tls_client_config)); let mut endpoint = quinn::Endpoint::client(bind_addr)?; let mut transport_config = quinn::TransportConfig::default(); diff --git a/content-discovery/iroh-mainline-tracker/src/tracker.rs b/content-discovery/iroh-mainline-tracker/src/tracker.rs index ae5f2f4..6b559d2 100644 --- a/content-discovery/iroh-mainline-tracker/src/tracker.rs +++ b/content-discovery/iroh-mainline-tracker/src/tracker.rs @@ -17,7 +17,8 @@ use iroh_mainline_content_discovery::{ protocol::{ AbsoluteTime, Announce, AnnounceKind, Query, QueryResponse, Request, Response, SignedAnnounce, REQUEST_SIZE_LIMIT, - }, tls_utils, to_infohash + }, + tls_utils, to_infohash, }; use rand::Rng; use redb::{ReadableTable, RedbValue};