From 14bd62d870e61198048f57a0ef92f8db820aed06 Mon Sep 17 00:00:00 2001
From: carlhou <carlhou369@gmail.com>
Date: Wed, 13 Mar 2024 16:17:43 +0800
Subject: [PATCH 01/11] init keystore support vrf & p2p

---
 .gitignore                |   4 +-
 beta/Cargo.toml           |  32 ++-
 beta/build.rs             |  11 +
 beta/src/bin/demo.rs      | 191 ++++++++++++++--
 beta/src/keystore.rs      | 123 +++++++++++
 beta/src/lib.rs           |   7 +
 beta/src/p2p/behaviour.rs | 444 ++++++++++++++++++++++++++++++++++++++
 beta/src/p2p/mod.rs       |   2 +
 beta/src/p2p/utils.rs     |  56 +++++
 beta/src/reqres.proto     |  14 ++
 beta/src/vrf.rs           |  98 +++++++++
 11 files changed, 956 insertions(+), 26 deletions(-)
 create mode 100644 beta/build.rs
 create mode 100644 beta/src/keystore.rs
 create mode 100644 beta/src/p2p/behaviour.rs
 create mode 100644 beta/src/p2p/mod.rs
 create mode 100644 beta/src/p2p/utils.rs
 create mode 100644 beta/src/reqres.proto
 create mode 100644 beta/src/vrf.rs

diff --git a/.gitignore b/.gitignore
index 299cf6d..4ffdc62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,4 +38,6 @@ override.tf.json
 
 # Ignore CLI configuration files
 .terraformrc
-terraform.rc
\ No newline at end of file
+terraform.rc
+
+node*/
\ No newline at end of file
diff --git a/beta/Cargo.toml b/beta/Cargo.toml
index 2abc030..27cdd51 100644
--- a/beta/Cargo.toml
+++ b/beta/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
+edition = "2021"
 name = "beta"
 version = "0.1.0"
-edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -10,21 +10,33 @@ name = "demo"
 path = "src/bin/demo.rs"
 
 [dependencies]
-wirehair = { version = "0.1.0", path = "../wirehair" }
 actix = "0.13.2"
-actix-web = "4.4.1"
 actix-multipart = "0.6.1"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-serde_bytes = "0.11"
-tokio = { version = "1.36.0", features = ["macros", "rt-multi-thread", "tracing"] }
+actix-web = "4.4.1"
+anyhow = "1.0.80"
+bytes = "0.5.6"
+cid = "0.11.1"
+clap = {version = "4.5.1", features = ["derive"]}
+ed25519-dalek = {version = "2.0.0", features = ["serde", "rand_core"]}
 env_logger = "0.11.1"
 futures = "0.3.30"
-bytes = "0.5.6"
-reqwest = { version = "0.11.24", features = ["json"] }
+getrandom = "^0.2.12"
+libp2p = {version = "0.53.2", features = ["tokio", "cbor", "dns", "kad", "noise", "macros", "request-response", "tcp", "yamux", "gossipsub", "identify", "ping"]}
+log = "0.4.21"
+merlin = {version = "3.0.0", default-features = false}
+prost = "0.12"
 rand = "0.8.5"
 rayon = "1.9.0"
-cid = "0.11.1"
+reqwest = {version = "0.11.24", features = ["json"]}
+schnorrkel = "0.11.4"
+serde = {version = "1.0", features = ["derive"]}
+serde_bytes = "0.11"
+serde_json = "1"
+tokio = {version = "1.36.0", features = ["macros", "rt-multi-thread", "tracing"]}
+wirehair = {version = "0.1.0", path = "../wirehair"}
 
 [dev-dependencies]
 criterion = "0.3"
+
+[build-dependencies]
+prost-build = "0.12.3"
diff --git a/beta/build.rs b/beta/build.rs
new file mode 100644
index 0000000..f0a94ec
--- /dev/null
+++ b/beta/build.rs
@@ -0,0 +1,11 @@
+use std::io::Result;
+fn main() -> Result<()> {
+    let mut config = prost_build::Config::new();
+    config
+        .type_attribute(
+            ".",
+            "#[derive(serde::Serialize,serde::Deserialize,std::cmp::Eq)]",
+        )
+        .compile_protos(&["src/reqres.proto"], &["src/"])?;
+    Ok(())
+}
diff --git a/beta/src/bin/demo.rs b/beta/src/bin/demo.rs
index d576aef..0ab510e 100644
--- a/beta/src/bin/demo.rs
+++ b/beta/src/bin/demo.rs
@@ -1,21 +1,182 @@
-use serde_json::from_str;
-
+use anyhow::Result;
+use beta::keystore::{KeyStore};
+use beta::p2p::behaviour::{Action, Event, Network, PeerRequest, PeerResponse};
+use beta::p2p::utils::bootstrap_light;
+use beta::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
 use beta::server;
+use clap::Parser;
+use env_logger::{Builder, Env};
+use futures::future;
+use futures::StreamExt;
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt,
+};
+use libp2p::PeerId;
+use libp2p::{multiaddr::Protocol, Multiaddr};
+use log::{info};
+use rand::random;
+
+
+use std::{path::PathBuf};
+use tokio::time::{self, Duration};
+use tokio::{task::spawn};
+
+#[derive(clap::Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Sets a p2p network port
+    #[arg(long, value_name = "PORT")]
+    port: Option<u16>,
+
+    /// Sets a p2p network endpoint
+    #[arg(long, value_name = "P2P")]
+    p2p: bool,
+
+    /// Sets full node muldiaddress
+    #[arg(short, long, value_name = "FUllADDR")]
+    full: Option<String>,
+
+    #[arg(short, long, value_name = "KEY")]
+    key: Option<PathBuf>,
+
+    /// Turn debugging information on
+    #[arg(short, long, value_name = "LOGLEVL")]
+    log_level: Option<String>,
+}
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 20)]
-async fn main() {
-    // Test for 100 nodes.
-    let port = std::env::args().collect::<Vec<String>>()[1].clone();
-    let port = from_str(port.as_str()).unwrap();
-
-    println!("start server at {}", port);
-    let mut peers = Vec::new();
-    if port == 4000 {
-        for i in port + 1..port + 100 {
-            peers.push(("localhost".to_string(), i));
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let log_level = cli.log_level.unwrap_or("info".into());
+    let env = Env::default().default_filter_or(&log_level);
+    Builder::from_env(env).format_timestamp_millis().init();
+
+    if !cli.p2p {
+        let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
+        info!("start serve at {port}");
+        let mut peers = Vec::new();
+        if port == 4000 {
+            for i in port + 1..port + 100 {
+                peers.push(("localhost".to_string(), i));
+            }
+        }
+        let _ = server::Peer::default()
+            .start("0.0.0.0".to_string(), port, peers)
+            .await;
+        return Ok(());
+    }
+
+    // Init keystore
+    let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
+    let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
+    keystore.save_to(path)?;
+
+    // Demo p2p
+    let (mut action_sender, action_receiver) = mpsc::channel(0);
+    let (event_sender, mut event_receiver) = mpsc::channel(0);
+
+    // New p2p Network
+    let network = Network::new(Some(keystore.seed), action_receiver, event_sender)
+        .await
+        .unwrap();
+
+    let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
+    info!("start p2p at {port}");
+
+    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse()?;
+    let local_peer_id = network.local_peer_id();
+    address_local = address_local.with_p2p(local_peer_id).unwrap();
+
+    // Start P2P network listener
+    spawn(network.start(address_local.clone()));
+
+    // Demo P2P network event handler
+    let mut action_sender_dup = action_sender.clone();
+    spawn(async move {
+        while let Some(e) = event_receiver.next().await {
+            match e {
+                Event::InboundRequest { request, channel } => {
+                    info!("handle inboud request {request:?}");
+                    action_sender_dup
+                        .send(Action::SendResponse {
+                            response: PeerResponse(PeerResponseMessage {
+                                id: request.0.id,
+                                command: request.0.command,
+                                data: b"world".to_vec(),
+                            }),
+                            channel,
+                        })
+                        .await
+                        .unwrap()
+                }
+                Event::IncomeConnection {
+                    peer_id,
+                    connection_id,
+                } => {
+                    info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
+                }
+                Event::ConnectionClosed {
+                    peer_id,
+                    connection_id,
+                } => {
+                    info!("connection closed peerID {peer_id} connectionID {connection_id}");
+                }
+                _ => {}
+            }
+        }
+    });
+
+    // Start light node demo
+    if let Some(full_node_addr_str) = cli.full {
+        let full_node: Multiaddr = full_node_addr_str.parse()?;
+        if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
+            // Bootstrap light node
+            bootstrap_light(action_sender.clone(), full_node.clone(), address_local)
+                .await
+                .unwrap();
+
+            // Light node send demo requests
+            let mut inter = time::interval(Duration::from_secs(10));
+            loop {
+                inter.tick().await;
+                let (sender, receiver) = oneshot::channel();
+
+                info!("send request greeting");
+                action_sender
+                    .send(Action::SendRequest {
+                        peer_id,
+                        msg: PeerRequest(PeerRequestMessage {
+                            id: format!("{}", random::<u16>()),
+                            command: "greeting".to_string(),
+                            data: b"hello".to_vec(),
+                        }),
+                        sender,
+                    })
+                    .await
+                    .unwrap();
+
+                let res = receiver.await.unwrap();
+                info!("get response {:?}", res);
+
+                let (sender, receiver) = oneshot::channel();
+                // Demo discovery query random peers
+                let key = PeerId::random();
+                action_sender
+                    .send(Action::GetPeers {
+                        key: key.into(),
+                        sender,
+                    })
+                    .await
+                    .unwrap();
+                let res = receiver.await.unwrap();
+                info!("get closest peers {:?}", res);
+            }
+        } else {
+            panic!("peer addr format error");
         }
     }
-    let _ = server::Peer::default()
-        .start("0.0.0.0".to_string(), port, peers)
-        .await;
+    let p = future::pending();
+    let () = p.await;
+    Ok(())
 }
diff --git a/beta/src/keystore.rs b/beta/src/keystore.rs
new file mode 100644
index 0000000..7c725a6
--- /dev/null
+++ b/beta/src/keystore.rs
@@ -0,0 +1,123 @@
+#![allow(dead_code)]
+use std::{
+    io::{Read, Write},
+    path::PathBuf,
+};
+
+use anyhow::{anyhow, Ok, Result};
+use ed25519_dalek::ed25519::signature::SignerMut;
+use merlin::Transcript;
+use rand::{rngs::OsRng, RngCore};
+use std::fs::{File, OpenOptions};
+
+use crate::vrf::{VrfHash, VrfPair, VrfProof, VrfPublickey};
+
+pub struct KeyStore {
+    pub seed: [u8; 32],
+    vrf_pair: Option<VrfPair>,
+    ed25519_secret: Option<ed25519_dalek::SigningKey>,
+    path: Option<PathBuf>,
+}
+
+impl KeyStore {
+    // todo: save encrypted seed
+    pub fn save_to(&self, path: PathBuf) -> Result<()> {
+        let mut file = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .truncate(true)
+            .open(path)?;
+        Ok(file.write_all(&self.seed)?)
+    }
+
+    pub fn generate_from_file(path: PathBuf) -> Result<Self> {
+        let mut file = File::open(path)?;
+        let mut seed = [0u8; 32];
+        file.read_exact(&mut seed)?;
+        Self::generate_with_seed(&seed)
+    }
+
+    pub fn generate() -> Self {
+        let mut seed: [u8; 32] = [0u8; 32];
+        let mut rng = OsRng;
+        rng.fill_bytes(&mut seed);
+
+        Self::generate_with_seed(&seed).unwrap()
+    }
+
+    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self> {
+        let mut ctx = Transcript::new(b"generate");
+        ctx.append_message(b"raw", seed);
+        let mut ed25519_seed = [0u8; 32];
+        ctx.challenge_bytes(b"ed25519", &mut ed25519_seed);
+        let ed25519_secret = ed25519_dalek::SigningKey::from_bytes(&ed25519_seed);
+
+        let mut vrf_seed = [0u8; 32];
+        ctx.challenge_bytes(b"vrf", &mut vrf_seed);
+        let vrf_pair = VrfPair::generate_with_seed(&vrf_seed)?;
+
+        Ok(KeyStore {
+            seed: seed.to_owned(),
+            vrf_pair: Some(vrf_pair),
+            ed25519_secret: Some(ed25519_secret),
+            path: None,
+        })
+    }
+
+    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof)> {
+        self.vrf_pair
+            .clone()
+            .ok_or(anyhow!("vrf keypair not exist"))
+            .and_then(|pair| pair.vrf_sign(input))
+    }
+
+    pub fn vrf_public(&self) -> Option<VrfPublickey> {
+        Some(self.vrf_pair.clone()?.get_public())
+    }
+
+    pub fn ed25519_public(&self) -> Option<ed25519_dalek::VerifyingKey> {
+        Some(self.ed25519_secret.clone()?.verifying_key())
+    }
+
+    pub fn ed25519_sign(&self, msg: &[u8]) -> Result<ed25519_dalek::Signature> {
+        self.ed25519_secret
+            .clone()
+            .ok_or(anyhow!("ed25519 keypair not exist"))
+            .and_then(|key| key.clone().try_sign(msg).map_err(|e| anyhow!("{}", e)))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    
+
+    use crate::keystore::*;
+    #[test]
+    fn keystore_vrf_sign_verify() {
+        let keystore = KeyStore::generate();
+        let msg = b"hello mars";
+        let msg_fraud = b"hello moon";
+
+        let (vrf_hash, vrf_proof) = keystore.vrf_sign(msg).unwrap();
+        let pubkey = keystore.vrf_public().unwrap();
+        pubkey
+            .verify_vrf(msg, vrf_hash, vrf_proof)
+            .expect("should pass verify");
+
+        pubkey
+            .verify_vrf(msg_fraud, vrf_hash, vrf_proof)
+            .expect_err("should not pass verfiy");
+    }
+    #[test]
+    fn keystore_ed25519_sign_verigy() {
+        let keystore = KeyStore::generate();
+        let msg = b"hello mars";
+        let msg_fraud = b"hello moon";
+        let sig = keystore.ed25519_sign(msg).unwrap();
+        let pubkey = keystore.ed25519_public().unwrap();
+        pubkey.verify_strict(msg, &sig).expect("should pass verify");
+        pubkey
+            .verify_strict(msg_fraud, &sig)
+            .expect_err("should not pass verify");
+    }
+}
diff --git a/beta/src/lib.rs b/beta/src/lib.rs
index ad9d7e7..3065055 100644
--- a/beta/src/lib.rs
+++ b/beta/src/lib.rs
@@ -1,5 +1,12 @@
 pub mod codec;
 pub mod server;
+pub mod p2p;
+pub mod keystore;
+mod vrf;
+
+pub mod reqres_proto {
+    include!(concat!(env!("OUT_DIR"), "/reqres_proto.rs"));
+}
 
 #[cfg(test)]
 pub mod test {
diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
new file mode 100644
index 0000000..571071a
--- /dev/null
+++ b/beta/src/p2p/behaviour.rs
@@ -0,0 +1,444 @@
+use anyhow::{anyhow, Result};
+use futures::channel::{mpsc, oneshot};
+use futures::prelude::*;
+use futures::StreamExt;
+
+use libp2p::core::ConnectedPoint;
+use libp2p::kad::{Caching, Config, QueryId};
+use libp2p::swarm::ConnectionId;
+use libp2p::{
+    core::Multiaddr,
+    gossipsub, identify, identity, kad,
+    multiaddr::Protocol,
+    noise, ping,
+    request_response::{self, OutboundRequestId, ProtocolSupport, ResponseChannel},
+    swarm::{NetworkBehaviour, Swarm, SwarmEvent},
+    tcp, yamux, PeerId,
+};
+
+
+use crate::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
+use libp2p::StreamProtocol;
+use serde::{Deserialize, Serialize};
+use std::collections::{hash_map, HashMap};
+
+use std::str::FromStr;
+use std::time::Duration;
+
+use log::{debug, info, warn};
+
+// Simple file exchange protocol
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PeerRequest(pub PeerRequestMessage);
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PeerResponse(pub PeerResponseMessage);
+
+#[derive(NetworkBehaviour)]
+pub struct Behaviour {
+    request_response: request_response::cbor::Behaviour<PeerRequest, PeerResponse>,
+    kademlia: kad::Behaviour<kad::store::MemoryStore>,
+    gossipsub: gossipsub::Behaviour,
+    identify: identify::Behaviour,
+    ping: ping::Behaviour,
+}
+
+pub enum Action {
+    Dial {
+        peer_id: PeerId,
+        peer_addr: Multiaddr,
+        sender: oneshot::Sender<Result<()>>,
+    },
+    SendRequest {
+        peer_id: PeerId,
+        msg: PeerRequest,
+        sender: oneshot::Sender<Result<PeerResponse>>,
+    },
+    GetPeers {
+        key: Vec<u8>,
+        sender: oneshot::Sender<Result<Vec<PeerId>>>,
+    },
+    SendResponse {
+        response: PeerResponse,
+        channel: ResponseChannel<PeerResponse>,
+    },
+    Bootstrap {},
+}
+
+pub enum Event {
+    InboundRequest {
+        request: PeerRequest,
+        channel: ResponseChannel<PeerResponse>,
+    },
+    IncomeConnection {
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+    },
+    ConnectionClosed {
+        peer_id: PeerId,
+        connection_id: ConnectionId,
+    },
+    Bootstrap,
+}
+
+pub struct Network {
+    swarm: Swarm<Behaviour>,
+    action_receiver: mpsc::Receiver<Action>,
+    event_sender: mpsc::Sender<Event>,
+    pending_request: HashMap<OutboundRequestId, oneshot::Sender<Result<PeerResponse>>>,
+    pending_get_peers: HashMap<QueryId, oneshot::Sender<Result<Vec<PeerId>>>>,
+    pending_dial: HashMap<PeerId, oneshot::Sender<Result<()>>>,
+}
+
+impl Network {
+    pub async fn new(
+        secret_key_seed: Option<[u8; 32]>,
+        action_receiver: mpsc::Receiver<Action>,
+        event_sender: mpsc::Sender<Event>,
+    ) -> Result<Self> {
+        let id_keys = match secret_key_seed {
+            Some(seed) => identity::Keypair::ed25519_from_bytes(seed).unwrap(),
+            None => identity::Keypair::generate_ed25519(),
+        };
+        let peer_id = id_keys.public().to_peer_id();
+
+        let mut swarm = libp2p::SwarmBuilder::with_existing_identity(id_keys)
+            .with_tokio()
+            .with_tcp(
+                tcp::Config::default(),
+                noise::Config::new,
+                yamux::Config::default,
+            )?
+            .with_behaviour(|key| {
+                let gossipsub_config = gossipsub::ConfigBuilder::default()
+                    .max_transmit_size(262144)
+                    .build()
+                    .unwrap();
+                let mut kad_config = Config::default();
+                kad_config
+                    .set_protocol_names(vec![StreamProtocol::new("/entropy_kad")])
+                    .set_caching(Caching::Enabled { max_peers: 10 });
+                Ok(Behaviour {
+                    kademlia: kad::Behaviour::with_config(
+                        peer_id,
+                        kad::store::MemoryStore::new(key.public().to_peer_id()),
+                        kad_config,
+                    ),
+                    request_response: request_response::cbor::Behaviour::new(
+                        [(StreamProtocol::new("/reqres"), ProtocolSupport::Full)],
+                        request_response::Config::default(),
+                    ),
+                    gossipsub: gossipsub::Behaviour::new(
+                        gossipsub::MessageAuthenticity::Signed(key.clone()),
+                        gossipsub_config,
+                    )
+                    .expect("Valid configuration"),
+                    identify: identify::Behaviour::new(identify::Config::new(
+                        "/entropy/0.1.0".into(),
+                        key.public(),
+                    )),
+                    ping: ping::Behaviour::new(ping::Config::new()),
+                })
+            })?
+            .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60)))
+            .build();
+
+        swarm
+            .behaviour_mut()
+            .kademlia
+            .set_mode(Some(kad::Mode::Server));
+
+        swarm
+            .behaviour_mut()
+            .gossipsub
+            .subscribe(&Self::gossip_topic())
+            .unwrap();
+
+        Ok(Self {
+            swarm,
+            action_receiver,
+            event_sender,
+            pending_request: HashMap::new(),
+            pending_dial: HashMap::new(),
+            pending_get_peers: HashMap::new(),
+        })
+    }
+
+    pub async fn start(mut self, multiaddr: Multiaddr) {
+        let peer_id = self.swarm.local_peer_id().to_owned();
+        self.swarm
+            .behaviour_mut()
+            .kademlia
+            .add_address(&peer_id, multiaddr.clone());
+
+        self.swarm.listen_on(multiaddr.clone()).unwrap();
+
+        self.swarm.add_external_address(multiaddr.clone());
+
+        let mut inter = tokio::time::interval(Duration::from_secs(10));
+        loop {
+            tokio::select! {
+                event = self.swarm.select_next_some() => self.handle_event(event).await,
+                action = self.action_receiver.next() => match action {
+                    Some(c) => self.handle_action(c).await,
+                    None=>  return,
+                },
+                _ = inter.tick() => {
+                    self.list_peers();
+                }
+            }
+        }
+    }
+
+    async fn handle_event(&mut self, event: SwarmEvent<BehaviourEvent>) {
+        match event {
+            SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
+                kad::Event::OutboundQueryProgressed {
+                    id: _,
+                    result: kad::QueryResult::Bootstrap(res),
+                    ..
+                },
+            )) => {
+                debug!("bootstrap result {res:?}");
+            }
+
+            SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
+                kad::Event::OutboundQueryProgressed {
+                    id,
+                    result:
+                        kad::QueryResult::GetClosestPeers(Ok(kad::GetClosestPeersOk { peers, key: _ })),
+                    // stats,
+                    ..
+                },
+            )) => {
+                if let Some(sender) = self.pending_get_peers.remove(&id) {
+                    let _ = sender.send(Ok(peers.clone()));
+                    debug!("get peers progress {peers:?}");
+                    if let Some(mut q) = self.swarm
+                        .behaviour_mut()
+                        .kademlia
+                        .query_mut(&id) { q.finish(); }
+                }
+            }
+            SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
+                kad::Event::OutboundQueryProgressed {
+                    id,
+                    result: kad::QueryResult::GetClosestPeers(Err(e)),
+                    step,
+                    ..
+                },
+            )) => {
+                if let Some(sender) = self.pending_get_peers.remove(&id) {
+                    debug!("get closest peers error {e}");
+                    if step.last {
+                        let _ = sender.send(Err(anyhow!("{e}")));
+                        if let Some(mut q) = self.swarm
+                            .behaviour_mut()
+                            .kademlia
+                            .query_mut(&id) { q.finish(); }
+                    }
+                }
+            }
+            SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
+                request_response::Event::Message { message, .. },
+            )) => match message {
+                request_response::Message::Request {
+                    request, channel, ..
+                } => {
+                    let data = request.0.data.clone();
+                    if request.0.command == *"multiaddress" {
+                        if let Ok(remote_address) =
+                            Multiaddr::from_str(String::from_utf8(data).unwrap().as_str())
+                        {
+                            if let Some(Protocol::P2p(remote_peer_id)) =
+                                remote_address.iter().last()
+                            {
+                                debug!(
+                                    "add address from ack {} {}",
+                                    remote_peer_id.clone(),
+                                    remote_address.clone()
+                                );
+                                self.swarm
+                                    .behaviour_mut()
+                                    .kademlia
+                                    .add_address(&remote_peer_id, remote_address);
+                            }
+                        }
+                    }
+                    self.event_sender
+                        .send(Event::InboundRequest {
+                            request: PeerRequest(request.0),
+                            channel,
+                        })
+                        .await
+                        .expect("Event receiver not to be dropped.");
+                }
+                request_response::Message::Response {
+                    request_id,
+                    response,
+                } => {
+                    let _ = self
+                        .pending_request
+                        .remove(&request_id)
+                        .expect("Request to still be pending.")
+                        .send(Ok(response));
+                }
+            },
+            SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
+                request_response::Event::OutboundFailure {
+                    request_id, error, ..
+                },
+            )) => {
+                let _ = self
+                    .pending_request
+                    .remove(&request_id)
+                    .expect("Request to still be pending.")
+                    .send(Err(anyhow!("{}", error)));
+            }
+            SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
+                request_response::Event::ResponseSent { .. },
+            )) => {}
+            SwarmEvent::NewListenAddr { address, .. } => {
+                let local_peer_id = *self.swarm.local_peer_id();
+                info!(
+                    "Local node is listening on {:?}",
+                    address.with(Protocol::P2p(local_peer_id))
+                );
+            }
+            SwarmEvent::IncomingConnection { .. } => {}
+            SwarmEvent::ConnectionEstablished {
+                peer_id,
+                connection_id,
+                endpoint,
+                ..
+            } => {
+                match endpoint {
+                    ConnectedPoint::Dialer { address, .. } => {
+                        if let Some(sender) = self.pending_dial.remove(&peer_id) {
+                            let _ = sender.send(Ok(()));
+                            self.swarm
+                                .behaviour_mut()
+                                .kademlia
+                                .add_address(&peer_id, address.clone());
+                        }
+                        debug!("connection dialer address {address}");
+                    }
+                    ConnectedPoint::Listener { send_back_addr, .. } => {
+                        self
+                            .event_sender
+                            .send(Event::IncomeConnection {
+                                peer_id,
+                                connection_id,
+                            })
+                            .await
+                            .expect("Event receiver not to be dropped.");
+                        debug!("connection listener send back address {send_back_addr}");
+                    }
+                }
+                debug!("connected to {peer_id}");
+            }
+            SwarmEvent::ConnectionClosed {
+                peer_id,
+                connection_id,
+                num_established: _,
+                endpoint: _,
+                cause,
+            } => {
+                self
+                    .event_sender
+                    .send(Event::ConnectionClosed {
+                        peer_id,
+                        connection_id,
+                    })
+                    .await
+                    .expect("Event receiver not to be dropped.");
+                debug!("connection closed {peer_id}, cause {cause:?}");
+            }
+            SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => {
+                if let Some(peer_id) = peer_id {
+                    if let Some(sender) = self.pending_dial.remove(&peer_id) {
+                        let _ = sender.send(Err(anyhow!("{}", error)));
+                    }
+                }
+            }
+            SwarmEvent::IncomingConnectionError { .. } => {}
+            SwarmEvent::Dialing {
+                peer_id: Some(peer_id),
+                ..
+            } => debug!("Dialing {peer_id}"),
+            e => debug!("unhandle event: {e:?}"),
+        }
+    }
+
+    async fn handle_action(&mut self, command: Action) {
+        match command {
+            Action::Bootstrap {} => {
+                let _ = self.swarm.behaviour_mut().kademlia.bootstrap().unwrap();
+            }
+            Action::GetPeers { key, sender } => {
+                let query_id = self.swarm.behaviour_mut().kademlia.get_closest_peers(key);
+                self.pending_get_peers.insert(query_id, sender);
+            }
+            Action::Dial {
+                peer_id,
+                peer_addr,
+                sender,
+            } => {
+                if let hash_map::Entry::Vacant(e) = self.pending_dial.entry(peer_id) {
+                    self.swarm
+                        .behaviour_mut()
+                        .kademlia
+                        .add_address(&peer_id, peer_addr.clone());
+                    match self.swarm.dial(peer_addr.with(Protocol::P2p(peer_id))) {
+                        Ok(()) => {
+                            e.insert(sender);
+                        }
+                        Err(e) => {
+                            let _ = sender.send(Err(anyhow!(e)));
+                        }
+                    }
+                } else {
+                    // already dialing
+                    warn!("already dialing {peer_id}")
+                }
+            }
+            Action::SendRequest {
+                peer_id,
+                msg,
+                sender,
+            } => {
+                let req_id = self
+                    .swarm
+                    .behaviour_mut()
+                    .request_response
+                    .send_request(&peer_id, msg);
+                self.pending_request.insert(req_id, sender);
+            }
+            Action::SendResponse { response, channel } => {
+                self.swarm
+                    .behaviour_mut()
+                    .request_response
+                    .send_response(channel, response)
+                    .unwrap();
+            }
+        }
+    }
+
+    pub fn local_peer_id(&self) -> PeerId {
+        self.swarm.local_peer_id().to_owned()
+    }
+
+    fn gossip_topic() -> gossipsub::IdentTopic {
+        gossipsub::IdentTopic::new("entropy_gossip")
+    }
+
+    fn list_peers(&mut self) {
+        for bucket in self.swarm.behaviour_mut().kademlia.kbuckets() {
+            if bucket.num_entries() > 0 {
+                for item in bucket.iter() {
+                    debug!("Peer ID: {:?}", item.node.key);
+                }
+            }
+        }
+    }
+}
diff --git a/beta/src/p2p/mod.rs b/beta/src/p2p/mod.rs
new file mode 100644
index 0000000..5465fa9
--- /dev/null
+++ b/beta/src/p2p/mod.rs
@@ -0,0 +1,2 @@
+pub mod behaviour;
+pub mod utils;
\ No newline at end of file
diff --git a/beta/src/p2p/utils.rs b/beta/src/p2p/utils.rs
new file mode 100644
index 0000000..2ef8f44
--- /dev/null
+++ b/beta/src/p2p/utils.rs
@@ -0,0 +1,56 @@
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt,
+};
+use libp2p::{multiaddr::Protocol, Multiaddr};
+use log::debug;
+use rand::random;
+
+use crate::{p2p::behaviour::PeerRequest, reqres_proto::PeerRequestMessage};
+
+use super::behaviour::Action;
+
+pub async fn bootstrap_light(
+    mut action_sender: mpsc::Sender<Action>,
+    full_node: Multiaddr,
+    address_local: Multiaddr,
+) -> Result<()> {
+    if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
+        let (sender, receiver) = oneshot::channel();
+        // dial
+        let action = Action::Dial {
+            peer_id,
+            peer_addr: full_node,
+            sender,
+        };
+        action_sender.send(action).await.unwrap();
+        receiver.await.unwrap().unwrap();
+        debug!("dial {peer_id} done");
+
+        // bootstrap
+        action_sender.send(Action::Bootstrap {}).await.unwrap();
+        tokio::time::sleep(Duration::from_secs(3)).await;
+
+        // acknowledge multiaddress
+        let (sender, receiver) = oneshot::channel();
+        action_sender
+            .send(Action::SendRequest {
+                peer_id,
+                msg: PeerRequest(PeerRequestMessage {
+                    id: format!("{}", random::<u16>()),
+                    command: "multiaddress".to_string(),
+                    data: address_local.clone().to_string().as_bytes().to_vec(),
+                }),
+                sender,
+            })
+            .await
+            .unwrap();
+        receiver.await.unwrap().unwrap();
+        Ok(())
+    } else {
+        Err(anyhow!("full node address format error {}", full_node))
+    }
+}
diff --git a/beta/src/reqres.proto b/beta/src/reqres.proto
new file mode 100644
index 0000000..06490d5
--- /dev/null
+++ b/beta/src/reqres.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+package reqres_proto;
+
+message PeerRequestMessage {
+    string id = 1;
+    string command = 2;
+    bytes data = 3;
+}
+
+message PeerResponseMessage {
+    string id = 1;
+    string command = 2;
+    bytes data = 3;
+}
\ No newline at end of file
diff --git a/beta/src/vrf.rs b/beta/src/vrf.rs
new file mode 100644
index 0000000..032245d
--- /dev/null
+++ b/beta/src/vrf.rs
@@ -0,0 +1,98 @@
+#![allow(dead_code)]
+use anyhow::{anyhow, Ok, Result};
+use rand::{rngs::OsRng, RngCore};
+use schnorrkel::{
+    context::SigningContext,
+    points::RistrettoBoth,
+    signing_context,
+    vrf::{VRFInOut, VRFProof},
+    ExpansionMode, MiniSecretKey, PublicKey,
+};
+
+pub type VrfHash = [u8; 32];
+
+pub type VrfProof = [u8; 64];
+
+#[derive(Clone, Debug)]
+pub struct VrfPair(schnorrkel::Keypair);
+
+pub struct VrfPublickey(schnorrkel::PublicKey);
+
+fn context() -> SigningContext {
+    signing_context(b"vrf")
+}
+
+impl VrfPair {
+    pub fn new() -> Self {
+        let mut rng = OsRng;
+        let mut seed = [0u8; 32];
+        rng.fill_bytes(&mut seed);
+        Self::generate_with_seed(&seed).unwrap()
+    }
+
+    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self> {
+        let mini_secret = MiniSecretKey::from_bytes(seed).map_err(|e| anyhow!("{}", e))?;
+        Ok(VrfPair(
+            mini_secret.expand_to_keypair(ExpansionMode::Ed25519),
+        ))
+    }
+
+    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof)> {
+        let (in_out, proof, _) = self
+            .0
+            .secret
+            .clone()
+            .to_keypair()
+            .vrf_sign(context().bytes(input));
+        Ok((in_out.output.to_bytes(), proof.to_bytes()))
+    }
+
+    pub fn vrf_verify(&self, input: &[u8], vrf_hash: VrfHash, proof: VrfProof) -> Result<()> {
+        self.get_public().verify_vrf(input, vrf_hash, proof)
+    }
+
+    pub fn get_public(&self) -> VrfPublickey {
+        VrfPublickey(self.0.public)
+    }
+}
+
+impl VrfPublickey {
+    pub fn verify_vrf(&self, input: &[u8], vrf_hash: VrfHash, proof: VrfProof) -> Result<()> {
+        let output = RistrettoBoth::from_bytes(vrf_hash.as_slice())
+            .map_err(|e| anyhow!("ristretto point from bytes error {}", e))?;
+        let proof =
+            VRFProof::from_bytes(&proof).map_err(|e| anyhow!("parse vrf proof error {}", e))?;
+        let in_out = VRFInOut {
+            input: self.0.vrf_hash(context().bytes(input)),
+            output,
+        };
+        self.0
+            .vrf_verify(context().bytes(input), &in_out.to_preout(), &proof)
+            .map_err(|e| anyhow!("{}", e))?;
+        Ok(())
+    }
+
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
+        Ok(VrfPublickey(
+            PublicKey::from_bytes(bytes).map_err(|e| anyhow!("{}", e))?,
+        ))
+    }
+
+    pub fn to_bytes(&self) -> [u8; 32] {
+        self.0.to_bytes()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::vrf::VrfPair;
+    #[test]
+    fn vrf_sign_verify() {
+        let secret = VrfPair::new();
+        let msg = b"hello world";
+        let (hash, proof) = secret.vrf_sign(msg).expect("should vrf hash msg");
+        secret
+            .vrf_verify(msg, hash, proof)
+            .expect("should pass verify");
+    }
+}

From 872a7a891e2115b28c1a8f0f4d9aa9643e955eb9 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Wed, 13 Mar 2024 16:38:53 +0800
Subject: [PATCH 02/11] fix

---
 Cargo.lock | 2142 ++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 1981 insertions(+), 161 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 02f0b31..7114134 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12,7 +12,7 @@ dependencies = [
  "actix-rt",
  "actix_derive",
  "bitflags 2.4.0",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "crossbeam-channel",
  "futures-core",
  "futures-sink",
@@ -34,7 +34,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
 dependencies = [
  "bitflags 1.3.2",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-core",
  "futures-sink",
  "memchr",
@@ -54,11 +54,11 @@ dependencies = [
  "actix-rt",
  "actix-service",
  "actix-utils",
- "ahash 0.8.3",
+ "ahash 0.8.11",
  "base64",
  "bitflags 2.4.0",
  "brotli",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "bytestring",
  "derive_more",
  "encoding_rs",
@@ -90,7 +90,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
 dependencies = [
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -102,7 +102,7 @@ dependencies = [
  "actix-multipart-derive",
  "actix-utils",
  "actix-web",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "derive_more",
  "futures-core",
  "futures-util",
@@ -128,7 +128,7 @@ dependencies = [
  "parse-size",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -225,8 +225,8 @@ dependencies = [
  "actix-service",
  "actix-utils",
  "actix-web-codegen",
- "ahash 0.8.3",
- "bytes 1.4.0",
+ "ahash 0.8.11",
+ "bytes 1.5.0",
  "bytestring",
  "cfg-if",
  "cookie",
@@ -285,7 +285,7 @@ checksum = "7c7db3d5a9718568e4cf4a537cfd7070e6e6ff7481510d0237fb529ac850f6d3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -303,6 +303,41 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "aead"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
+dependencies = [
+ "crypto-common",
+ "generic-array",
+]
+
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "aes-gcm"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"
+dependencies = [
+ "aead",
+ "aes",
+ "cipher",
+ "ctr",
+ "ghash",
+ "subtle",
+]
+
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -316,14 +351,15 @@ dependencies = [
 
 [[package]]
 name = "ahash"
-version = "0.8.3"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"
 dependencies = [
  "cfg-if",
  "getrandom",
  "once_cell",
  "version_check",
+ "zerocopy",
 ]
 
 [[package]]
@@ -351,19 +387,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "anstream"
-version = "0.3.2"
+name = "allocator-api2"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon 1.0.2",
- "colorchoice",
- "is-terminal",
- "utf8parse",
-]
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
 
 [[package]]
 name = "anstream"
@@ -374,7 +401,7 @@ dependencies = [
  "anstyle",
  "anstyle-parse",
  "anstyle-query",
- "anstyle-wincon 3.0.2",
+ "anstyle-wincon",
  "colorchoice",
  "utf8parse",
 ]
@@ -405,39 +432,134 @@ dependencies = [
 
 [[package]]
 name = "anstyle-wincon"
-version = "1.0.2"
+version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c"
+checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
 dependencies = [
  "anstyle",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
-name = "anstyle-wincon"
-version = "3.0.2"
+name = "anyhow"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7"
+checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+
+[[package]]
+name = "arrayref"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+
+[[package]]
+name = "asn1-rs"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0"
 dependencies = [
- "anstyle",
+ "asn1-rs-derive",
+ "asn1-rs-impl",
+ "displaydoc",
+ "nom",
+ "num-traits",
+ "rusticata-macros",
+ "thiserror",
+ "time",
+]
+
+[[package]]
+name = "asn1-rs-derive"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "synstructure",
+]
+
+[[package]]
+name = "asn1-rs-impl"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "async-io"
+version = "2.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884"
+dependencies = [
+ "async-lock",
+ "cfg-if",
+ "concurrent-queue",
+ "futures-io",
+ "futures-lite",
+ "parking",
+ "polling",
+ "rustix",
+ "slab",
+ "tracing",
  "windows-sys 0.52.0",
 ]
 
 [[package]]
-name = "anyhow"
-version = "1.0.75"
+name = "async-lock"
+version = "3.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b"
+dependencies = [
+ "event-listener",
+ "event-listener-strategy",
+ "pin-project-lite",
+]
 
 [[package]]
 name = "async-trait"
-version = "0.1.73"
+version = "0.1.77"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
+checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
+]
+
+[[package]]
+name = "asynchronous-codec"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a860072022177f903e59730004fb5dc13db9275b79bb2aef7ba8ce831956c233"
+dependencies = [
+ "bytes 1.5.0",
+ "futures-sink",
+ "futures-util",
+ "memchr",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "attohttpc"
+version = "0.24.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2"
+dependencies = [
+ "http",
+ "log",
+ "url",
 ]
 
 [[package]]
@@ -471,7 +593,7 @@ dependencies = [
  "actix-utils",
  "ahash 0.7.6",
  "base64",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "cfg-if",
  "cookie",
  "derive_more",
@@ -500,7 +622,7 @@ dependencies = [
  "async-trait",
  "axum-core",
  "bitflags 1.3.2",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-util",
  "http",
  "http-body",
@@ -526,7 +648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
 dependencies = [
  "async-trait",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-util",
  "http",
  "http-body",
@@ -559,9 +681,9 @@ checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270"
 
 [[package]]
 name = "base64"
-version = "0.21.2"
+version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 
 [[package]]
 name = "base64ct"
@@ -576,14 +698,24 @@ dependencies = [
  "actix",
  "actix-multipart",
  "actix-web",
+ "anyhow",
  "bytes 0.5.6",
  "cid",
+ "clap 4.5.2",
  "criterion",
+ "ed25519-dalek",
  "env_logger",
  "futures",
+ "getrandom",
+ "libp2p",
+ "log",
+ "merlin",
+ "prost 0.12.3",
+ "prost-build",
  "rand",
  "rayon",
  "reqwest",
+ "schnorrkel",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -612,6 +744,15 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
 
+[[package]]
+name = "blake2"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
+dependencies = [
+ "digest",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -642,12 +783,27 @@ dependencies = [
  "alloc-stdlib",
 ]
 
+[[package]]
+name = "bs58"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896"
+dependencies = [
+ "tinyvec",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
 
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
 [[package]]
 name = "bytes"
 version = "0.5.6"
@@ -656,9 +812,9 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
 
 [[package]]
 name = "bytes"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
 
 [[package]]
 name = "bytestring"
@@ -666,7 +822,7 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
 ]
 
 [[package]]
@@ -675,6 +831,15 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
+[[package]]
+name = "cbor4ii"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59b4c883b9cc4757b061600d39001d4d0232bece4a3174696cf8f58a14db107d"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.83"
@@ -691,6 +856,30 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chacha20"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures",
+]
+
+[[package]]
+name = "chacha20poly1305"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
+dependencies = [
+ "aead",
+ "chacha20",
+ "cipher",
+ "poly1305",
+ "zeroize",
+]
+
 [[package]]
 name = "cid"
 version = "0.11.1"
@@ -703,6 +892,17 @@ dependencies = [
  "unsigned-varint 0.8.0",
 ]
 
+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+ "zeroize",
+]
+
 [[package]]
 name = "clap"
 version = "2.34.0"
@@ -716,44 +916,43 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.3.23"
+version = "4.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3"
+checksum = "b230ab84b0ffdf890d5a10abdbc8b83ae1c4918275daea1ab8801f71536b2651"
 dependencies = [
  "clap_builder",
  "clap_derive",
- "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.3.23"
+version = "4.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98"
+checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4"
 dependencies = [
- "anstream 0.3.2",
+ "anstream",
  "anstyle",
  "clap_lex",
- "strsim",
+ "strsim 0.11.0",
 ]
 
 [[package]]
 name = "clap_derive"
-version = "4.3.12"
+version = "4.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050"
+checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47"
 dependencies = [
  "heck",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
+checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"
 
 [[package]]
 name = "cmake"
@@ -770,6 +969,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
+[[package]]
+name = "concurrent-queue"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.5"
@@ -884,36 +1092,34 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.3"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
 dependencies = [
- "cfg-if",
  "crossbeam-epoch",
  "crossbeam-utils",
 ]
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.15"
+version = "0.9.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
 dependencies = [
- "autocfg",
- "cfg-if",
  "crossbeam-utils",
- "memoffset",
- "scopeguard",
 ]
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
-dependencies = [
- "cfg-if",
-]
+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"
+
+[[package]]
+name = "crunchy"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 
 [[package]]
 name = "crypto-common"
@@ -922,6 +1128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
+ "rand_core",
  "typenum",
 ]
 
@@ -946,11 +1153,20 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "ctr"
+version = "0.9.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"
+dependencies = [
+ "cipher",
+]
+
 [[package]]
 name = "curve25519-dalek"
-version = "4.0.0"
+version = "4.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f711ade317dd348950a9910f81c5947e3d8907ebd2b83f76203ff1807e6a2bc2"
+checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -971,7 +1187,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -994,8 +1210,8 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
- "syn 2.0.29",
+ "strsim 0.10.0",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1006,7 +1222,7 @@ checksum = "1d1545d67a2149e1d93b7e5c7752dce5a7426eb5d1357ddcfd89336b94444f77"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1045,6 +1261,20 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "der-parser"
+version = "8.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e"
+dependencies = [
+ "asn1-rs",
+ "displaydoc",
+ "nom",
+ "num-bigint",
+ "num-traits",
+ "rusticata-macros",
+]
+
 [[package]]
 name = "deranged"
 version = "0.3.8"
@@ -1072,8 +1302,26 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
  "crypto-common",
+ "subtle",
+]
+
+[[package]]
+name = "displaydoc"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
 ]
 
+[[package]]
+name = "dtoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
+
 [[package]]
 name = "ed25519"
 version = "2.2.2"
@@ -1122,7 +1370,7 @@ dependencies = [
  "actix-web-opentelemetry",
  "awc",
  "bincode",
- "clap 4.3.23",
+ "clap 4.5.2",
  "ed25519-dalek",
  "opentelemetry",
  "opentelemetry-otlp",
@@ -1137,6 +1385,18 @@ dependencies = [
  "wirehair",
 ]
 
+[[package]]
+name = "enum-as-inner"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "env_filter"
 version = "0.1.0"
@@ -1153,13 +1413,19 @@ version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "05e7cf40684ae96ade6232ed84582f40ce0a66efcd43a5117aef610534f8e0b8"
 dependencies = [
- "anstream 0.6.11",
+ "anstream",
  "anstyle",
  "env_filter",
  "humantime",
  "log",
 ]
 
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "errno"
 version = "0.3.8"
@@ -1170,6 +1436,27 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "event-listener"
+version = "4.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "event-listener-strategy"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3"
+dependencies = [
+ "event-listener",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "fastrand"
 version = "2.0.1"
@@ -1178,9 +1465,15 @@ checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
 
 [[package]]
 name = "fiat-crypto"
-version = "0.1.20"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382"
+
+[[package]]
+name = "fixedbitset"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77"
+checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flate2"
@@ -1237,6 +1530,16 @@ dependencies = [
  "futures-util",
 ]
 
+[[package]]
+name = "futures-bounded"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1e2774cc104e198ef3d3e1ff4ab40f86fa3245d6cb6a3a46174f21463cee173"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.30"
@@ -1262,6 +1565,7 @@ dependencies = [
  "futures-core",
  "futures-task",
  "futures-util",
+ "num_cpus",
 ]
 
 [[package]]
@@ -1270,6 +1574,16 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
 
+[[package]]
+name = "futures-lite"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "futures-macro"
 version = "0.3.30"
@@ -1278,7 +1592,17 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
+]
+
+[[package]]
+name = "futures-rustls"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28"
+dependencies = [
+ "futures-io",
+ "rustls",
 ]
 
 [[package]]
@@ -1294,15 +1618,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
 
 [[package]]
-name = "futures-util"
-version = "0.3.30"
+name = "futures-ticker"
+version = "0.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+checksum = "9763058047f713632a52e916cc7f6a4b3fc6e9fc1ff8c5b1dc49e5a89041682e"
 dependencies = [
- "futures-channel",
- "futures-core",
- "futures-io",
- "futures-macro",
+ "futures",
+ "futures-timer",
+ "instant",
+]
+
+[[package]]
+name = "futures-timer"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+
+[[package]]
+name = "futures-util"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
  "futures-sink",
  "futures-task",
  "memchr",
@@ -1323,15 +1664,35 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.10"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"
 dependencies = [
  "cfg-if",
  "libc",
  "wasi",
 ]
 
+[[package]]
+name = "getrandom_or_panic"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ea1015b5a70616b688dc230cfe50c8af89d972cb132d5a622814d29773b10b9"
+dependencies = [
+ "rand",
+ "rand_core",
+]
+
+[[package]]
+name = "ghash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"
+dependencies = [
+ "opaque-debug",
+ "polyval",
+]
+
 [[package]]
 name = "gimli"
 version = "0.27.3"
@@ -1344,13 +1705,13 @@ version = "0.3.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "fnv",
  "futures-core",
  "futures-sink",
  "futures-util",
  "http",
- "indexmap",
+ "indexmap 1.9.3",
  "slab",
  "tokio",
  "tokio-util",
@@ -1372,6 +1733,16 @@ dependencies = [
  "ahash 0.7.6",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+dependencies = [
+ "ahash 0.8.11",
+ "allocator-api2",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -1393,13 +1764,109 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hex_fmt"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f"
+
+[[package]]
+name = "hickory-proto"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "091a6fbccf4860009355e3efc52ff4acf37a63489aad7435372d44ceeb6fbbcf"
+dependencies = [
+ "async-trait",
+ "cfg-if",
+ "data-encoding",
+ "enum-as-inner",
+ "futures-channel",
+ "futures-io",
+ "futures-util",
+ "idna",
+ "ipnet",
+ "once_cell",
+ "rand",
+ "socket2 0.5.5",
+ "thiserror",
+ "tinyvec",
+ "tokio",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "hickory-resolver"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35b8f021164e6a984c9030023544c57789c51760065cd510572fedcfb04164e8"
+dependencies = [
+ "cfg-if",
+ "futures-util",
+ "hickory-proto",
+ "ipconfig",
+ "lru-cache",
+ "once_cell",
+ "parking_lot",
+ "rand",
+ "resolv-conf",
+ "smallvec",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "hkdf"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
+dependencies = [
+ "hmac",
+]
+
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
+[[package]]
+name = "home"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "hostname"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867"
+dependencies = [
+ "libc",
+ "match_cfg",
+ "winapi",
+]
+
 [[package]]
 name = "http"
 version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "fnv",
  "itoa",
 ]
@@ -1410,7 +1877,7 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "http",
  "pin-project-lite",
 ]
@@ -1439,7 +1906,7 @@ version = "0.14.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1475,7 +1942,7 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "hyper",
  "native-tls",
  "tokio",
@@ -1498,6 +1965,54 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "if-addrs"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cabb0019d51a643781ff15c9c8a3e5dedc365c47211270f4e8f82812fedd8f0a"
+dependencies = [
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "if-watch"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6b0422c86d7ce0e97169cc42e04ae643caf278874a7a3c87b8150a220dc7e1e"
+dependencies = [
+ "async-io",
+ "core-foundation",
+ "fnv",
+ "futures",
+ "if-addrs",
+ "ipnet",
+ "log",
+ "rtnetlink",
+ "system-configuration",
+ "tokio",
+ "windows",
+]
+
+[[package]]
+name = "igd-next"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4"
+dependencies = [
+ "async-trait",
+ "attohttpc",
+ "bytes 1.5.0",
+ "futures",
+ "http",
+ "hyper",
+ "log",
+ "rand",
+ "tokio",
+ "url",
+ "xmltree",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.3"
@@ -1505,26 +2020,55 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
 dependencies = [
  "autocfg",
- "hashbrown",
+ "hashbrown 0.12.3",
 ]
 
 [[package]]
-name = "ipnet"
-version = "2.9.0"
+name = "indexmap"
+version = "2.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.3",
+]
 
 [[package]]
-name = "is-terminal"
-version = "0.4.9"
+name = "inout"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
 dependencies = [
- "hermit-abi 0.3.2",
- "rustix",
+ "generic-array",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "ipconfig"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f"
+dependencies = [
+ "socket2 0.5.5",
+ "widestring",
  "windows-sys 0.48.0",
+ "winreg",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -1558,6 +2102,15 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "keccak"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654"
+dependencies = [
+ "cpufeatures",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"
@@ -1576,6 +2129,449 @@ version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
+[[package]]
+name = "libp2p"
+version = "0.53.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "681fb3f183edfbedd7a57d32ebe5dcdc0b9f94061185acf3c30249349cc6fc99"
+dependencies = [
+ "bytes 1.5.0",
+ "either",
+ "futures",
+ "futures-timer",
+ "getrandom",
+ "instant",
+ "libp2p-allow-block-list",
+ "libp2p-connection-limits",
+ "libp2p-core",
+ "libp2p-dns",
+ "libp2p-gossipsub",
+ "libp2p-identify",
+ "libp2p-identity",
+ "libp2p-kad",
+ "libp2p-mdns",
+ "libp2p-metrics",
+ "libp2p-noise",
+ "libp2p-ping",
+ "libp2p-quic",
+ "libp2p-request-response",
+ "libp2p-swarm",
+ "libp2p-tcp",
+ "libp2p-upnp",
+ "libp2p-yamux",
+ "multiaddr",
+ "pin-project",
+ "rw-stream-sink",
+ "thiserror",
+]
+
+[[package]]
+name = "libp2p-allow-block-list"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "107b238b794cb83ab53b74ad5dcf7cca3200899b72fe662840cfb52f5b0a32e6"
+dependencies = [
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "void",
+]
+
+[[package]]
+name = "libp2p-connection-limits"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7cd50a78ccfada14de94cbacd3ce4b0138157f376870f13d3a8422cd075b4fd"
+dependencies = [
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "void",
+]
+
+[[package]]
+name = "libp2p-core"
+version = "0.41.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8130a8269e65a2554d55131c770bdf4bcd94d2b8d4efb24ca23699be65066c05"
+dependencies = [
+ "either",
+ "fnv",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-identity",
+ "multiaddr",
+ "multihash",
+ "multistream-select",
+ "once_cell",
+ "parking_lot",
+ "pin-project",
+ "quick-protobuf",
+ "rand",
+ "rw-stream-sink",
+ "smallvec",
+ "thiserror",
+ "tracing",
+ "unsigned-varint 0.8.0",
+ "void",
+]
+
+[[package]]
+name = "libp2p-dns"
+version = "0.41.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d17cbcf7160ff35c3e8e560de4a068fe9d6cb777ea72840e48eb76ff9576c4b6"
+dependencies = [
+ "async-trait",
+ "futures",
+ "hickory-resolver",
+ "libp2p-core",
+ "libp2p-identity",
+ "parking_lot",
+ "smallvec",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-gossipsub"
+version = "0.46.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d665144a616dadebdc5fff186b1233488cdcd8bfb1223218ff084b6d052c94f7"
+dependencies = [
+ "asynchronous-codec",
+ "base64",
+ "byteorder",
+ "bytes 1.5.0",
+ "either",
+ "fnv",
+ "futures",
+ "futures-ticker",
+ "getrandom",
+ "hex_fmt",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "prometheus-client",
+ "quick-protobuf",
+ "quick-protobuf-codec",
+ "rand",
+ "regex",
+ "sha2",
+ "smallvec",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-identify"
+version = "0.44.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "20499a945d2f0221fdc6269b3848892c0f370d2ee3e19c7f65a29d8f860f6126"
+dependencies = [
+ "asynchronous-codec",
+ "either",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "lru",
+ "quick-protobuf",
+ "quick-protobuf-codec",
+ "smallvec",
+ "thiserror",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-identity"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "999ec70441b2fb35355076726a6bc466c932e9bdc66f6a11c6c0aa17c7ab9be0"
+dependencies = [
+ "bs58",
+ "ed25519-dalek",
+ "hkdf",
+ "multihash",
+ "quick-protobuf",
+ "rand",
+ "sha2",
+ "thiserror",
+ "tracing",
+ "zeroize",
+]
+
+[[package]]
+name = "libp2p-kad"
+version = "0.45.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc5767727d062c4eac74dd812c998f0e488008e82cce9c33b463d38423f9ad2"
+dependencies = [
+ "arrayvec",
+ "asynchronous-codec",
+ "bytes 1.5.0",
+ "either",
+ "fnv",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "quick-protobuf",
+ "quick-protobuf-codec",
+ "rand",
+ "sha2",
+ "smallvec",
+ "thiserror",
+ "tracing",
+ "uint",
+ "void",
+]
+
+[[package]]
+name = "libp2p-mdns"
+version = "0.45.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49007d9a339b3e1d7eeebc4d67c05dbf23d300b7d091193ec2d3f26802d7faf2"
+dependencies = [
+ "data-encoding",
+ "futures",
+ "hickory-proto",
+ "if-watch",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand",
+ "smallvec",
+ "socket2 0.5.5",
+ "tokio",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-metrics"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fdac91ae4f291046a3b2660c039a2830c931f84df2ee227989af92f7692d3357"
+dependencies = [
+ "futures",
+ "instant",
+ "libp2p-core",
+ "libp2p-gossipsub",
+ "libp2p-identify",
+ "libp2p-identity",
+ "libp2p-kad",
+ "libp2p-ping",
+ "libp2p-swarm",
+ "pin-project",
+ "prometheus-client",
+]
+
+[[package]]
+name = "libp2p-noise"
+version = "0.44.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecd0545ce077f6ea5434bcb76e8d0fe942693b4380aaad0d34a358c2bd05793"
+dependencies = [
+ "asynchronous-codec",
+ "bytes 1.5.0",
+ "curve25519-dalek",
+ "futures",
+ "libp2p-core",
+ "libp2p-identity",
+ "multiaddr",
+ "multihash",
+ "once_cell",
+ "quick-protobuf",
+ "rand",
+ "sha2",
+ "snow",
+ "static_assertions",
+ "thiserror",
+ "tracing",
+ "x25519-dalek",
+ "zeroize",
+]
+
+[[package]]
+name = "libp2p-ping"
+version = "0.44.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76b94ee41bd8c294194fe608851e45eb98de26fe79bc7913838cbffbfe8c7ce2"
+dependencies = [
+ "either",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-quic"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0375cdfee57b47b313ef1f0fdb625b78aed770d33a40cf1c294a371ff5e6666"
+dependencies = [
+ "bytes 1.5.0",
+ "futures",
+ "futures-timer",
+ "if-watch",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-tls",
+ "parking_lot",
+ "quinn",
+ "rand",
+ "ring 0.16.20",
+ "rustls",
+ "socket2 0.5.5",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-request-response"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e12823250fe0c45bdddea6eefa2be9a609aff1283ff4e1d8a294fdbb89572f6f"
+dependencies = [
+ "async-trait",
+ "cbor4ii",
+ "futures",
+ "futures-bounded",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand",
+ "serde",
+ "smallvec",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-swarm"
+version = "0.44.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e92532fc3c4fb292ae30c371815c9b10103718777726ea5497abc268a4761866"
+dependencies = [
+ "either",
+ "fnv",
+ "futures",
+ "futures-timer",
+ "instant",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm-derive",
+ "multistream-select",
+ "once_cell",
+ "rand",
+ "smallvec",
+ "tokio",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-swarm-derive"
+version = "0.34.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b644268b4acfdaa6a6100b31226ee7a36d96ab4c43287d113bfd2308607d8b6f"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
+[[package]]
+name = "libp2p-tcp"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2460fc2748919adff99ecbc1aab296e4579e41f374fb164149bd2c9e529d4c"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "if-watch",
+ "libc",
+ "libp2p-core",
+ "libp2p-identity",
+ "socket2 0.5.5",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "libp2p-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93ce7e3c2e7569d685d08ec795157981722ff96e9e9f9eae75df3c29d02b07a5"
+dependencies = [
+ "futures",
+ "futures-rustls",
+ "libp2p-core",
+ "libp2p-identity",
+ "rcgen",
+ "ring 0.16.20",
+ "rustls",
+ "rustls-webpki",
+ "thiserror",
+ "x509-parser",
+ "yasna",
+]
+
+[[package]]
+name = "libp2p-upnp"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b49cc89949bf0e06869297cd4fe2c132358c23fe93e76ad43950453df4da3d35"
+dependencies = [
+ "futures",
+ "futures-timer",
+ "igd-next",
+ "libp2p-core",
+ "libp2p-swarm",
+ "tokio",
+ "tracing",
+ "void",
+]
+
+[[package]]
+name = "libp2p-yamux"
+version = "0.45.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "200cbe50349a44760927d50b431d77bed79b9c0a3959de1af8d24a63434b71e5"
+dependencies = [
+ "either",
+ "futures",
+ "libp2p-core",
+ "thiserror",
+ "tracing",
+ "yamux 0.12.1",
+ "yamux 0.13.1",
+]
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.13"
@@ -1612,9 +2608,33 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+
+[[package]]
+name = "lru"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc"
+dependencies = [
+ "hashbrown 0.14.3",
+]
+
+[[package]]
+name = "lru-cache"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
+dependencies = [
+ "linked-hash-map",
+]
+
+[[package]]
+name = "match_cfg"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
 
 [[package]]
 name = "matchers"
@@ -1633,24 +2653,33 @@ checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef"
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
 
 [[package]]
-name = "memoffset"
-version = "0.9.0"
+name = "merlin"
+version = "3.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
+checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d"
 dependencies = [
- "autocfg",
+ "byteorder",
+ "keccak",
+ "rand_core",
+ "zeroize",
 ]
 
 [[package]]
 name = "mime"
 version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
@@ -1673,6 +2702,25 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "multiaddr"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b852bc02a2da5feed68cd14fa50d0774b92790a5bdbfa932a813926c8472070"
+dependencies = [
+ "arrayref",
+ "byteorder",
+ "data-encoding",
+ "libp2p-identity",
+ "multibase",
+ "multihash",
+ "percent-encoding",
+ "serde",
+ "static_assertions",
+ "unsigned-varint 0.7.2",
+ "url",
+]
+
 [[package]]
 name = "multibase"
 version = "0.9.1"
@@ -1694,6 +2742,26 @@ dependencies = [
  "unsigned-varint 0.7.2",
 ]
 
+[[package]]
+name = "multimap"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
+
+[[package]]
+name = "multistream-select"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19"
+dependencies = [
+ "bytes 1.5.0",
+ "futures",
+ "log",
+ "pin-project",
+ "smallvec",
+ "unsigned-varint 0.7.2",
+]
+
 [[package]]
 name = "native-tls"
 version = "0.2.11"
@@ -1712,6 +2780,99 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "netlink-packet-core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "libc",
+ "netlink-packet-utils",
+]
+
+[[package]]
+name = "netlink-packet-route"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9ea4302b9759a7a88242299225ea3688e63c85ea136371bb6cf94fd674efaab"
+dependencies = [
+ "anyhow",
+ "bitflags 1.3.2",
+ "byteorder",
+ "libc",
+ "netlink-packet-core",
+ "netlink-packet-utils",
+]
+
+[[package]]
+name = "netlink-packet-utils"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ede8a08c71ad5a95cdd0e4e52facd37190977039a4704eb82a283f713747d34"
+dependencies = [
+ "anyhow",
+ "byteorder",
+ "paste",
+ "thiserror",
+]
+
+[[package]]
+name = "netlink-proto"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6"
+dependencies = [
+ "bytes 1.5.0",
+ "futures",
+ "log",
+ "netlink-packet-core",
+ "netlink-sys",
+ "thiserror",
+ "tokio",
+]
+
+[[package]]
+name = "netlink-sys"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411"
+dependencies = [
+ "bytes 1.5.0",
+ "futures",
+ "libc",
+ "log",
+ "tokio",
+]
+
+[[package]]
+name = "nix"
+version = "0.24.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+]
+
+[[package]]
+name = "nohash-hasher"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -1722,6 +2883,26 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.16"
@@ -1750,6 +2931,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "oid-registry"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff"
+dependencies = [
+ "asn1-rs",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.18.0"
@@ -1762,6 +2952,12 @@ version = "11.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
+[[package]]
+name = "opaque-debug"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
+
 [[package]]
 name = "openssl"
 version = "0.10.63"
@@ -1785,7 +2981,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -1829,7 +3025,7 @@ dependencies = [
  "opentelemetry-semantic-conventions",
  "opentelemetry_api",
  "opentelemetry_sdk",
- "prost",
+ "prost 0.11.9",
  "thiserror",
  "tokio",
  "tonic",
@@ -1843,7 +3039,7 @@ checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb"
 dependencies = [
  "opentelemetry_api",
  "opentelemetry_sdk",
- "prost",
+ "prost 0.11.9",
  "tonic",
 ]
 
@@ -1864,7 +3060,7 @@ checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b"
 dependencies = [
  "futures-channel",
  "futures-util",
- "indexmap",
+ "indexmap 1.9.3",
  "js-sys",
  "once_cell",
  "pin-project-lite",
@@ -1910,6 +3106,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 
+[[package]]
+name = "parking"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -1945,12 +3147,32 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
+[[package]]
+name = "pem"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310"
+dependencies = [
+ "base64",
+ "serde",
+]
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
+[[package]]
+name = "petgraph"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9"
+dependencies = [
+ "fixedbitset",
+ "indexmap 2.2.5",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.1.3"
@@ -1968,7 +3190,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2033,29 +3255,131 @@ dependencies = [
  "plotters-backend",
 ]
 
+[[package]]
+name = "polling"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9"
+dependencies = [
+ "cfg-if",
+ "concurrent-queue",
+ "pin-project-lite",
+ "rustix",
+ "tracing",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "poly1305"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
+dependencies = [
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
+[[package]]
+name = "polyval"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "opaque-debug",
+ "universal-hash",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
+[[package]]
+name = "prettyplease"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5"
+dependencies = [
+ "proc-macro2",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "proc-macro2"
-version = "1.0.66"
+version = "1.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
+checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
 dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prometheus-client"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1ca959da22a332509f2a73ae9e5f23f9dcfc31fd3a54d71f159495bd5909baa"
+dependencies = [
+ "dtoa",
+ "itoa",
+ "parking_lot",
+ "prometheus-client-derive-encode",
+]
+
+[[package]]
+name = "prometheus-client-derive-encode"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "prost"
 version = "0.11.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd"
 dependencies = [
- "bytes 1.4.0",
- "prost-derive",
+ "bytes 1.5.0",
+ "prost-derive 0.11.9",
+]
+
+[[package]]
+name = "prost"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a"
+dependencies = [
+ "bytes 1.5.0",
+ "prost-derive 0.12.3",
+]
+
+[[package]]
+name = "prost-build"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2"
+dependencies = [
+ "bytes 1.5.0",
+ "heck",
+ "itertools",
+ "log",
+ "multimap",
+ "once_cell",
+ "petgraph",
+ "prettyplease",
+ "prost 0.12.3",
+ "prost-types",
+ "regex",
+ "syn 2.0.52",
+ "tempfile",
+ "which",
 ]
 
 [[package]]
@@ -2071,11 +3395,109 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "prost-derive"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e"
+dependencies = [
+ "prost 0.12.3",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
+
+[[package]]
+name = "quick-protobuf"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d6da84cc204722a989e01ba2f6e1e276e190f22263d0cb6ce8526fcdb0d2e1f"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "quick-protobuf-codec"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474"
+dependencies = [
+ "asynchronous-codec",
+ "bytes 1.5.0",
+ "quick-protobuf",
+ "thiserror",
+ "unsigned-varint 0.8.0",
+]
+
+[[package]]
+name = "quinn"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75"
+dependencies = [
+ "bytes 1.5.0",
+ "futures-io",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "thiserror",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a"
+dependencies = [
+ "bytes 1.5.0",
+ "rand",
+ "ring 0.16.20",
+ "rustc-hash",
+ "rustls",
+ "slab",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7"
+dependencies = [
+ "bytes 1.5.0",
+ "libc",
+ "socket2 0.5.5",
+ "tracing",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
 dependencies = [
  "proc-macro2",
 ]
@@ -2112,9 +3534,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.9.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
 dependencies = [
  "either",
  "rayon-core",
@@ -2130,6 +3552,18 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "rcgen"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6"
+dependencies = [
+ "pem",
+ "ring 0.16.20",
+ "time",
+ "yasna",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.3.5"
@@ -2150,14 +3584,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.3"
+version = "1.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
+checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.3.6",
- "regex-syntax 0.7.4",
+ "regex-automata 0.4.6",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -2171,13 +3605,13 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.3.6"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.4",
+ "regex-syntax 0.8.2",
 ]
 
 [[package]]
@@ -2188,9 +3622,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.4"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
 
 [[package]]
 name = "reqwest"
@@ -2199,7 +3633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251"
 dependencies = [
  "base64",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "encoding_rs",
  "futures-core",
  "futures-util",
@@ -2232,12 +3666,73 @@ dependencies = [
  "winreg",
 ]
 
+[[package]]
+name = "resolv-conf"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00"
+dependencies = [
+ "hostname",
+ "quick-error",
+]
+
+[[package]]
+name = "ring"
+version = "0.16.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
+dependencies = [
+ "cc",
+ "libc",
+ "once_cell",
+ "spin 0.5.2",
+ "untrusted 0.7.1",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom",
+ "libc",
+ "spin 0.9.8",
+ "untrusted 0.9.0",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rtnetlink"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "322c53fd76a18698f1c27381d58091de3a043d356aa5bd0d510608b565f469a0"
+dependencies = [
+ "futures",
+ "log",
+ "netlink-packet-route",
+ "netlink-proto",
+ "nix",
+ "thiserror",
+ "tokio",
+]
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
 
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.0"
@@ -2247,6 +3742,15 @@ dependencies = [
  "semver",
 ]
 
+[[package]]
+name = "rusticata-macros"
+version = "4.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632"
+dependencies = [
+ "nom",
+]
+
 [[package]]
 name = "rustix"
 version = "0.38.31"
@@ -2260,6 +3764,18 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.21.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
+dependencies = [
+ "log",
+ "ring 0.17.8",
+ "rustls-webpki",
+ "sct",
+]
+
 [[package]]
 name = "rustls-pemfile"
 version = "1.0.4"
@@ -2269,12 +3785,33 @@ dependencies = [
  "base64",
 ]
 
+[[package]]
+name = "rustls-webpki"
+version = "0.101.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765"
+dependencies = [
+ "ring 0.17.8",
+ "untrusted 0.9.0",
+]
+
 [[package]]
 name = "rustversion"
 version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
+[[package]]
+name = "rw-stream-sink"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1"
+dependencies = [
+ "futures",
+ "pin-project",
+ "static_assertions",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.15"
@@ -2299,12 +3836,41 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "schnorrkel"
+version = "0.11.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de18f6d8ba0aad7045f5feae07ec29899c1112584a38509a84ad7b04451eaa0"
+dependencies = [
+ "aead",
+ "arrayref",
+ "arrayvec",
+ "curve25519-dalek",
+ "getrandom_or_panic",
+ "merlin",
+ "rand_core",
+ "serde_bytes",
+ "sha2",
+ "subtle",
+ "zeroize",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "sct"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414"
+dependencies = [
+ "ring 0.17.8",
+ "untrusted 0.9.0",
+]
+
 [[package]]
 name = "security-framework"
 version = "2.9.2"
@@ -2370,7 +3936,7 @@ checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2418,9 +3984,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.7"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -2462,9 +4028,26 @@ dependencies = [
 
 [[package]]
 name = "smallvec"
-version = "1.11.0"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
+
+[[package]]
+name = "snow"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
+checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85"
+dependencies = [
+ "aes-gcm",
+ "blake2",
+ "chacha20poly1305",
+ "curve25519-dalek",
+ "rand_core",
+ "ring 0.17.8",
+ "rustc_version",
+ "sha2",
+ "subtle",
+]
 
 [[package]]
 name = "socket2"
@@ -2486,6 +4069,18 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
 [[package]]
 name = "spki"
 version = "0.7.2"
@@ -2496,12 +4091,24 @@ dependencies = [
  "der",
 ]
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "strsim"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
+[[package]]
+name = "strsim"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01"
+
 [[package]]
 name = "subtle"
 version = "2.5.0"
@@ -2521,9 +4128,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.29"
+version = "2.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
+checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2536,6 +4143,18 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
+[[package]]
+name = "synstructure"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "unicode-xid",
+]
+
 [[package]]
 name = "system-configuration"
 version = "0.5.1"
@@ -2581,22 +4200,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.47"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
+checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.47"
+version = "1.0.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
+checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2669,7 +4288,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931"
 dependencies = [
  "backtrace",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "libc",
  "mio",
  "num_cpus",
@@ -2700,7 +4319,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2730,11 +4349,11 @@ version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
 dependencies = [
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-core",
  "futures-sink",
  "futures-util",
- "hashbrown",
+ "hashbrown 0.12.3",
  "pin-project-lite",
  "tokio",
  "tracing",
@@ -2749,7 +4368,7 @@ dependencies = [
  "async-trait",
  "axum",
  "base64",
- "bytes 1.4.0",
+ "bytes 1.5.0",
  "futures-core",
  "futures-util",
  "h2",
@@ -2759,7 +4378,7 @@ dependencies = [
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
- "prost",
+ "prost 0.11.9",
  "tokio",
  "tokio-stream",
  "tower",
@@ -2776,7 +4395,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
 dependencies = [
  "futures-core",
  "futures-util",
- "indexmap",
+ "indexmap 1.9.3",
  "pin-project",
  "pin-project-lite",
  "rand",
@@ -2821,7 +4440,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
 ]
 
 [[package]]
@@ -2902,6 +4521,18 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
+[[package]]
+name = "uint"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76f64bba2c53b04fcab63c01a7d7427eadc821e3bc48c34dc9ba29c501164b52"
+dependencies = [
+ "byteorder",
+ "crunchy",
+ "hex",
+ "static_assertions",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.13"
@@ -2929,6 +4560,22 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
 
+[[package]]
+name = "unicode-xid"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
+
+[[package]]
+name = "universal-hash"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
+dependencies = [
+ "crypto-common",
+ "subtle",
+]
+
 [[package]]
 name = "unsigned-varint"
 version = "0.7.2"
@@ -2941,6 +4588,18 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
 [[package]]
 name = "url"
 version = "2.4.0"
@@ -2982,6 +4641,12 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "void"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"
@@ -3028,7 +4693,7 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
  "wasm-bindgen-shared",
 ]
 
@@ -3062,7 +4727,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.29",
+ "syn 2.0.52",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3083,6 +4748,24 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix",
+]
+
+[[package]]
+name = "widestring"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -3114,6 +4797,25 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.48.0"
@@ -3264,11 +4966,129 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "x25519-dalek"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277"
+dependencies = [
+ "curve25519-dalek",
+ "rand_core",
+ "serde",
+ "zeroize",
+]
+
+[[package]]
+name = "x509-parser"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da"
+dependencies = [
+ "asn1-rs",
+ "data-encoding",
+ "der-parser",
+ "lazy_static",
+ "nom",
+ "oid-registry",
+ "rusticata-macros",
+ "thiserror",
+ "time",
+]
+
+[[package]]
+name = "xml-rs"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a"
+
+[[package]]
+name = "xmltree"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb"
+dependencies = [
+ "xml-rs",
+]
+
+[[package]]
+name = "yamux"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed0164ae619f2dc144909a9f082187ebb5893693d8c0196e8085283ccd4b776"
+dependencies = [
+ "futures",
+ "log",
+ "nohash-hasher",
+ "parking_lot",
+ "pin-project",
+ "rand",
+ "static_assertions",
+]
+
+[[package]]
+name = "yamux"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad1d0148b89300047e72994bee99ecdabd15a9166a7b70c8b8c37c314dcc9002"
+dependencies = [
+ "futures",
+ "instant",
+ "log",
+ "nohash-hasher",
+ "parking_lot",
+ "pin-project",
+ "rand",
+ "static_assertions",
+]
+
+[[package]]
+name = "yasna"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd"
+dependencies = [
+ "time",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.7.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
+dependencies = [
+ "zeroize_derive",
+]
+
+[[package]]
+name = "zeroize_derive"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.52",
+]
 
 [[package]]
 name = "zstd"

From b61ef12ea670d8b8504187eea2b228b5daaa0e20 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Wed, 13 Mar 2024 17:38:21 +0800
Subject: [PATCH 03/11] bin new net demo

---
 beta/src/bin/demo.rs | 193 ++++---------------------------------------
 beta/src/bin/net.rs  | 182 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 198 insertions(+), 177 deletions(-)
 create mode 100644 beta/src/bin/net.rs

diff --git a/beta/src/bin/demo.rs b/beta/src/bin/demo.rs
index 0ab510e..8c351ca 100644
--- a/beta/src/bin/demo.rs
+++ b/beta/src/bin/demo.rs
@@ -1,182 +1,21 @@
-use anyhow::Result;
-use beta::keystore::{KeyStore};
-use beta::p2p::behaviour::{Action, Event, Network, PeerRequest, PeerResponse};
-use beta::p2p::utils::bootstrap_light;
-use beta::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
-use beta::server;
-use clap::Parser;
-use env_logger::{Builder, Env};
-use futures::future;
-use futures::StreamExt;
-use futures::{
-    channel::{mpsc, oneshot},
-    SinkExt,
-};
-use libp2p::PeerId;
-use libp2p::{multiaddr::Protocol, Multiaddr};
-use log::{info};
-use rand::random;
-
-
-use std::{path::PathBuf};
-use tokio::time::{self, Duration};
-use tokio::{task::spawn};
-
-#[derive(clap::Parser)]
-#[command(version, about, long_about = None)]
-struct Cli {
-    /// Sets a p2p network port
-    #[arg(long, value_name = "PORT")]
-    port: Option<u16>,
-
-    /// Sets a p2p network endpoint
-    #[arg(long, value_name = "P2P")]
-    p2p: bool,
-
-    /// Sets full node muldiaddress
-    #[arg(short, long, value_name = "FUllADDR")]
-    full: Option<String>,
+use serde_json::from_str;
 
-    #[arg(short, long, value_name = "KEY")]
-    key: Option<PathBuf>,
-
-    /// Turn debugging information on
-    #[arg(short, long, value_name = "LOGLEVL")]
-    log_level: Option<String>,
-}
+use beta::server;
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 20)]
-async fn main() -> Result<()> {
-    let cli = Cli::parse();
-    let log_level = cli.log_level.unwrap_or("info".into());
-    let env = Env::default().default_filter_or(&log_level);
-    Builder::from_env(env).format_timestamp_millis().init();
-
-    if !cli.p2p {
-        let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
-        info!("start serve at {port}");
-        let mut peers = Vec::new();
-        if port == 4000 {
-            for i in port + 1..port + 100 {
-                peers.push(("localhost".to_string(), i));
-            }
-        }
-        let _ = server::Peer::default()
-            .start("0.0.0.0".to_string(), port, peers)
-            .await;
-        return Ok(());
-    }
-
-    // Init keystore
-    let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
-    let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
-    keystore.save_to(path)?;
-
-    // Demo p2p
-    let (mut action_sender, action_receiver) = mpsc::channel(0);
-    let (event_sender, mut event_receiver) = mpsc::channel(0);
-
-    // New p2p Network
-    let network = Network::new(Some(keystore.seed), action_receiver, event_sender)
-        .await
-        .unwrap();
-
-    let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
-    info!("start p2p at {port}");
-
-    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse()?;
-    let local_peer_id = network.local_peer_id();
-    address_local = address_local.with_p2p(local_peer_id).unwrap();
-
-    // Start P2P network listener
-    spawn(network.start(address_local.clone()));
-
-    // Demo P2P network event handler
-    let mut action_sender_dup = action_sender.clone();
-    spawn(async move {
-        while let Some(e) = event_receiver.next().await {
-            match e {
-                Event::InboundRequest { request, channel } => {
-                    info!("handle inboud request {request:?}");
-                    action_sender_dup
-                        .send(Action::SendResponse {
-                            response: PeerResponse(PeerResponseMessage {
-                                id: request.0.id,
-                                command: request.0.command,
-                                data: b"world".to_vec(),
-                            }),
-                            channel,
-                        })
-                        .await
-                        .unwrap()
-                }
-                Event::IncomeConnection {
-                    peer_id,
-                    connection_id,
-                } => {
-                    info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
-                }
-                Event::ConnectionClosed {
-                    peer_id,
-                    connection_id,
-                } => {
-                    info!("connection closed peerID {peer_id} connectionID {connection_id}");
-                }
-                _ => {}
-            }
-        }
-    });
-
-    // Start light node demo
-    if let Some(full_node_addr_str) = cli.full {
-        let full_node: Multiaddr = full_node_addr_str.parse()?;
-        if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
-            // Bootstrap light node
-            bootstrap_light(action_sender.clone(), full_node.clone(), address_local)
-                .await
-                .unwrap();
-
-            // Light node send demo requests
-            let mut inter = time::interval(Duration::from_secs(10));
-            loop {
-                inter.tick().await;
-                let (sender, receiver) = oneshot::channel();
-
-                info!("send request greeting");
-                action_sender
-                    .send(Action::SendRequest {
-                        peer_id,
-                        msg: PeerRequest(PeerRequestMessage {
-                            id: format!("{}", random::<u16>()),
-                            command: "greeting".to_string(),
-                            data: b"hello".to_vec(),
-                        }),
-                        sender,
-                    })
-                    .await
-                    .unwrap();
-
-                let res = receiver.await.unwrap();
-                info!("get response {:?}", res);
-
-                let (sender, receiver) = oneshot::channel();
-                // Demo discovery query random peers
-                let key = PeerId::random();
-                action_sender
-                    .send(Action::GetPeers {
-                        key: key.into(),
-                        sender,
-                    })
-                    .await
-                    .unwrap();
-                let res = receiver.await.unwrap();
-                info!("get closest peers {:?}", res);
-            }
-        } else {
-            panic!("peer addr format error");
+async fn main() {
+    // Test for 100 nodes.
+    let port = std::env::args().collect::<Vec<String>>()[1].clone();
+    let port = from_str(port.as_str()).unwrap();
+
+    println!("start server at {}", port);
+    let mut peers = Vec::new();
+    if port == 4000 {
+        for i in port + 1..port + 100 {
+            peers.push(("localhost".to_string(), i));
         }
     }
-    let p = future::pending();
-    let () = p.await;
-    Ok(())
-}
+    let _ = server::Peer::default()
+        .start("0.0.0.0".to_string(), port, peers)
+        .await;
+}
\ No newline at end of file
diff --git a/beta/src/bin/net.rs b/beta/src/bin/net.rs
new file mode 100644
index 0000000..0ab510e
--- /dev/null
+++ b/beta/src/bin/net.rs
@@ -0,0 +1,182 @@
+use anyhow::Result;
+use beta::keystore::{KeyStore};
+use beta::p2p::behaviour::{Action, Event, Network, PeerRequest, PeerResponse};
+use beta::p2p::utils::bootstrap_light;
+use beta::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
+use beta::server;
+use clap::Parser;
+use env_logger::{Builder, Env};
+use futures::future;
+use futures::StreamExt;
+use futures::{
+    channel::{mpsc, oneshot},
+    SinkExt,
+};
+use libp2p::PeerId;
+use libp2p::{multiaddr::Protocol, Multiaddr};
+use log::{info};
+use rand::random;
+
+
+use std::{path::PathBuf};
+use tokio::time::{self, Duration};
+use tokio::{task::spawn};
+
+#[derive(clap::Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Sets a p2p network port
+    #[arg(long, value_name = "PORT")]
+    port: Option<u16>,
+
+    /// Sets a p2p network endpoint
+    #[arg(long, value_name = "P2P")]
+    p2p: bool,
+
+    /// Sets full node muldiaddress
+    #[arg(short, long, value_name = "FUllADDR")]
+    full: Option<String>,
+
+    #[arg(short, long, value_name = "KEY")]
+    key: Option<PathBuf>,
+
+    /// Turn debugging information on
+    #[arg(short, long, value_name = "LOGLEVL")]
+    log_level: Option<String>,
+}
+
+#[tokio::main(flavor = "multi_thread", worker_threads = 20)]
+async fn main() -> Result<()> {
+    let cli = Cli::parse();
+    let log_level = cli.log_level.unwrap_or("info".into());
+    let env = Env::default().default_filter_or(&log_level);
+    Builder::from_env(env).format_timestamp_millis().init();
+
+    if !cli.p2p {
+        let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
+        info!("start serve at {port}");
+        let mut peers = Vec::new();
+        if port == 4000 {
+            for i in port + 1..port + 100 {
+                peers.push(("localhost".to_string(), i));
+            }
+        }
+        let _ = server::Peer::default()
+            .start("0.0.0.0".to_string(), port, peers)
+            .await;
+        return Ok(());
+    }
+
+    // Init keystore
+    let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
+    let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
+    keystore.save_to(path)?;
+
+    // Demo p2p
+    let (mut action_sender, action_receiver) = mpsc::channel(0);
+    let (event_sender, mut event_receiver) = mpsc::channel(0);
+
+    // New p2p Network
+    let network = Network::new(Some(keystore.seed), action_receiver, event_sender)
+        .await
+        .unwrap();
+
+    let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
+    info!("start p2p at {port}");
+
+    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse()?;
+    let local_peer_id = network.local_peer_id();
+    address_local = address_local.with_p2p(local_peer_id).unwrap();
+
+    // Start P2P network listener
+    spawn(network.start(address_local.clone()));
+
+    // Demo P2P network event handler
+    let mut action_sender_dup = action_sender.clone();
+    spawn(async move {
+        while let Some(e) = event_receiver.next().await {
+            match e {
+                Event::InboundRequest { request, channel } => {
+                    info!("handle inboud request {request:?}");
+                    action_sender_dup
+                        .send(Action::SendResponse {
+                            response: PeerResponse(PeerResponseMessage {
+                                id: request.0.id,
+                                command: request.0.command,
+                                data: b"world".to_vec(),
+                            }),
+                            channel,
+                        })
+                        .await
+                        .unwrap()
+                }
+                Event::IncomeConnection {
+                    peer_id,
+                    connection_id,
+                } => {
+                    info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
+                }
+                Event::ConnectionClosed {
+                    peer_id,
+                    connection_id,
+                } => {
+                    info!("connection closed peerID {peer_id} connectionID {connection_id}");
+                }
+                _ => {}
+            }
+        }
+    });
+
+    // Start light node demo
+    if let Some(full_node_addr_str) = cli.full {
+        let full_node: Multiaddr = full_node_addr_str.parse()?;
+        if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
+            // Bootstrap light node
+            bootstrap_light(action_sender.clone(), full_node.clone(), address_local)
+                .await
+                .unwrap();
+
+            // Light node send demo requests
+            let mut inter = time::interval(Duration::from_secs(10));
+            loop {
+                inter.tick().await;
+                let (sender, receiver) = oneshot::channel();
+
+                info!("send request greeting");
+                action_sender
+                    .send(Action::SendRequest {
+                        peer_id,
+                        msg: PeerRequest(PeerRequestMessage {
+                            id: format!("{}", random::<u16>()),
+                            command: "greeting".to_string(),
+                            data: b"hello".to_vec(),
+                        }),
+                        sender,
+                    })
+                    .await
+                    .unwrap();
+
+                let res = receiver.await.unwrap();
+                info!("get response {:?}", res);
+
+                let (sender, receiver) = oneshot::channel();
+                // Demo discovery query random peers
+                let key = PeerId::random();
+                action_sender
+                    .send(Action::GetPeers {
+                        key: key.into(),
+                        sender,
+                    })
+                    .await
+                    .unwrap();
+                let res = receiver.await.unwrap();
+                info!("get closest peers {:?}", res);
+            }
+        } else {
+            panic!("peer addr format error");
+        }
+    }
+    let p = future::pending();
+    let () = p.await;
+    Ok(())
+}

From afee75e84672ac31558071af542e30a73f669e03 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Tue, 19 Mar 2024 14:44:58 +0800
Subject: [PATCH 04/11] refactor:replace anyhow error with thiserror

---
 Cargo.lock                |  2 +-
 beta/Cargo.toml           |  1 +
 beta/src/bin/net.rs       | 23 +++++++++---------
 beta/src/keystore.rs      | 42 ++++++++++++++++++++++-----------
 beta/src/p2p/behaviour.rs | 49 ++++++++++++++++++---------------------
 beta/src/p2p/error.rs     | 20 ++++++++++++++++
 beta/src/p2p/mod.rs       |  3 ++-
 beta/src/p2p/utils.rs     |  8 +++----
 beta/src/vrf.rs           | 46 +++++++++++++++++++++++++-----------
 9 files changed, 123 insertions(+), 71 deletions(-)
 create mode 100644 beta/src/p2p/error.rs

diff --git a/Cargo.lock b/Cargo.lock
index 7114134..cf2a8ab 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -698,7 +698,6 @@ dependencies = [
  "actix",
  "actix-multipart",
  "actix-web",
- "anyhow",
  "bytes 0.5.6",
  "cid",
  "clap 4.5.2",
@@ -719,6 +718,7 @@ dependencies = [
  "serde",
  "serde_bytes",
  "serde_json",
+ "thiserror",
  "tokio",
  "wirehair",
 ]
diff --git a/beta/Cargo.toml b/beta/Cargo.toml
index 27cdd51..4d716a4 100644
--- a/beta/Cargo.toml
+++ b/beta/Cargo.toml
@@ -34,6 +34,7 @@ serde_bytes = "0.11"
 serde_json = "1"
 tokio = {version = "1.36.0", features = ["macros", "rt-multi-thread", "tracing"]}
 wirehair = {version = "0.1.0", path = "../wirehair"}
+thiserror = "1.0.58"
 
 [dev-dependencies]
 criterion = "0.3"
diff --git a/beta/src/bin/net.rs b/beta/src/bin/net.rs
index 0ab510e..c256e4c 100644
--- a/beta/src/bin/net.rs
+++ b/beta/src/bin/net.rs
@@ -1,7 +1,6 @@
-use anyhow::Result;
-use beta::keystore::{KeyStore};
+use beta::keystore::KeyStore;
 use beta::p2p::behaviour::{Action, Event, Network, PeerRequest, PeerResponse};
-use beta::p2p::utils::bootstrap_light;
+use beta::p2p::utils::bootstrap_peer;
 use beta::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
 use beta::server;
 use clap::Parser;
@@ -14,13 +13,13 @@ use futures::{
 };
 use libp2p::PeerId;
 use libp2p::{multiaddr::Protocol, Multiaddr};
-use log::{info};
+use log::info;
 use rand::random;
 
-
-use std::{path::PathBuf};
+use std::error::Error;
+use std::path::PathBuf;
+use tokio::task::spawn;
 use tokio::time::{self, Duration};
-use tokio::{task::spawn};
 
 #[derive(clap::Parser)]
 #[command(version, about, long_about = None)]
@@ -46,7 +45,7 @@ struct Cli {
 }
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 20)]
-async fn main() -> Result<()> {
+async fn main() -> Result<(), Box<dyn Error>> {
     let cli = Cli::parse();
     let log_level = cli.log_level.unwrap_or("info".into());
     let env = Env::default().default_filter_or(&log_level);
@@ -70,7 +69,7 @@ async fn main() -> Result<()> {
     // Init keystore
     let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
     let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
-    keystore.save_to(path)?;
+    keystore.save_to(path).unwrap();
 
     // Demo p2p
     let (mut action_sender, action_receiver) = mpsc::channel(0);
@@ -84,7 +83,7 @@ async fn main() -> Result<()> {
     let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
     info!("start p2p at {port}");
 
-    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse()?;
+    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap();
     let local_peer_id = network.local_peer_id();
     address_local = address_local.with_p2p(local_peer_id).unwrap();
 
@@ -129,10 +128,10 @@ async fn main() -> Result<()> {
 
     // Start light node demo
     if let Some(full_node_addr_str) = cli.full {
-        let full_node: Multiaddr = full_node_addr_str.parse()?;
+        let full_node: Multiaddr = full_node_addr_str.parse().unwrap();
         if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
             // Bootstrap light node
-            bootstrap_light(action_sender.clone(), full_node.clone(), address_local)
+            bootstrap_peer(action_sender.clone(), full_node.clone(), address_local)
                 .await
                 .unwrap();
 
diff --git a/beta/src/keystore.rs b/beta/src/keystore.rs
index 7c725a6..66a7381 100644
--- a/beta/src/keystore.rs
+++ b/beta/src/keystore.rs
@@ -1,16 +1,27 @@
 #![allow(dead_code)]
 use std::{
-    io::{Read, Write},
+    io::{self, Read, Write},
     path::PathBuf,
 };
-
-use anyhow::{anyhow, Ok, Result};
+use thiserror::Error;
 use ed25519_dalek::ed25519::signature::SignerMut;
 use merlin::Transcript;
 use rand::{rngs::OsRng, RngCore};
 use std::fs::{File, OpenOptions};
 
-use crate::vrf::{VrfHash, VrfPair, VrfProof, VrfPublickey};
+use crate::vrf::{VrfError, VrfHash, VrfPair, VrfProof, VrfPublickey};
+
+#[derive(Error, Debug)]
+pub enum KeyStoreError {
+    #[error("io error")]
+    IOError(#[from] io::Error),
+    #[error("vrf error")]
+    VrfError(#[from] VrfError),
+    #[error("keypair not exist error")]
+    KeyNotExist,
+    #[error("ed25519 sign error")]
+    Ed25519SignError(String),
+}
 
 pub struct KeyStore {
     pub seed: [u8; 32],
@@ -21,7 +32,7 @@ pub struct KeyStore {
 
 impl KeyStore {
     // todo: save encrypted seed
-    pub fn save_to(&self, path: PathBuf) -> Result<()> {
+    pub fn save_to(&self, path: PathBuf) -> Result<(), KeyStoreError> {
         let mut file = OpenOptions::new()
             .write(true)
             .create(true)
@@ -30,7 +41,7 @@ impl KeyStore {
         Ok(file.write_all(&self.seed)?)
     }
 
-    pub fn generate_from_file(path: PathBuf) -> Result<Self> {
+    pub fn generate_from_file(path: PathBuf) -> Result<Self, KeyStoreError> {
         let mut file = File::open(path)?;
         let mut seed = [0u8; 32];
         file.read_exact(&mut seed)?;
@@ -45,7 +56,7 @@ impl KeyStore {
         Self::generate_with_seed(&seed).unwrap()
     }
 
-    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self> {
+    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self, KeyStoreError> {
         let mut ctx = Transcript::new(b"generate");
         ctx.append_message(b"raw", seed);
         let mut ed25519_seed = [0u8; 32];
@@ -64,11 +75,11 @@ impl KeyStore {
         })
     }
 
-    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof)> {
+    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof), KeyStoreError> {
         self.vrf_pair
             .clone()
-            .ok_or(anyhow!("vrf keypair not exist"))
-            .and_then(|pair| pair.vrf_sign(input))
+            .ok_or(KeyStoreError::KeyNotExist)
+            .and_then(|pair| pair.vrf_sign(input).map_err(KeyStoreError::VrfError))
     }
 
     pub fn vrf_public(&self) -> Option<VrfPublickey> {
@@ -79,17 +90,20 @@ impl KeyStore {
         Some(self.ed25519_secret.clone()?.verifying_key())
     }
 
-    pub fn ed25519_sign(&self, msg: &[u8]) -> Result<ed25519_dalek::Signature> {
+    pub fn ed25519_sign(&self, msg: &[u8]) -> Result<ed25519_dalek::Signature, KeyStoreError> {
         self.ed25519_secret
             .clone()
-            .ok_or(anyhow!("ed25519 keypair not exist"))
-            .and_then(|key| key.clone().try_sign(msg).map_err(|e| anyhow!("{}", e)))
+            .ok_or(KeyStoreError::KeyNotExist)
+            .and_then(|key| {
+                key.clone()
+                    .try_sign(msg)
+                    .map_err(|e| KeyStoreError::Ed25519SignError(format!("{e}")))
+            })
     }
 }
 
 #[cfg(test)]
 mod tests {
-    
 
     use crate::keystore::*;
     #[test]
diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index 571071a..c66e4f5 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -1,4 +1,4 @@
-use anyhow::{anyhow, Result};
+use crate::p2p::error::P2PNetworkError;
 use futures::channel::{mpsc, oneshot};
 use futures::prelude::*;
 use futures::StreamExt;
@@ -16,7 +16,6 @@ use libp2p::{
     tcp, yamux, PeerId,
 };
 
-
 use crate::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
 use libp2p::StreamProtocol;
 use serde::{Deserialize, Serialize};
@@ -47,16 +46,16 @@ pub enum Action {
     Dial {
         peer_id: PeerId,
         peer_addr: Multiaddr,
-        sender: oneshot::Sender<Result<()>>,
+        sender: oneshot::Sender<Result<(), P2PNetworkError>>,
     },
     SendRequest {
         peer_id: PeerId,
         msg: PeerRequest,
-        sender: oneshot::Sender<Result<PeerResponse>>,
+        sender: oneshot::Sender<Result<PeerResponse, P2PNetworkError>>,
     },
     GetPeers {
         key: Vec<u8>,
-        sender: oneshot::Sender<Result<Vec<PeerId>>>,
+        sender: oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>,
     },
     SendResponse {
         response: PeerResponse,
@@ -85,9 +84,10 @@ pub struct Network {
     swarm: Swarm<Behaviour>,
     action_receiver: mpsc::Receiver<Action>,
     event_sender: mpsc::Sender<Event>,
-    pending_request: HashMap<OutboundRequestId, oneshot::Sender<Result<PeerResponse>>>,
-    pending_get_peers: HashMap<QueryId, oneshot::Sender<Result<Vec<PeerId>>>>,
-    pending_dial: HashMap<PeerId, oneshot::Sender<Result<()>>>,
+    pending_request:
+        HashMap<OutboundRequestId, oneshot::Sender<Result<PeerResponse, P2PNetworkError>>>,
+    pending_get_peers: HashMap<QueryId, oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>>,
+    pending_dial: HashMap<PeerId, oneshot::Sender<Result<(), P2PNetworkError>>>,
 }
 
 impl Network {
@@ -95,7 +95,7 @@ impl Network {
         secret_key_seed: Option<[u8; 32]>,
         action_receiver: mpsc::Receiver<Action>,
         event_sender: mpsc::Sender<Event>,
-    ) -> Result<Self> {
+    ) -> Result<Self, P2PNetworkError> {
         let id_keys = match secret_key_seed {
             Some(seed) => identity::Keypair::ed25519_from_bytes(seed).unwrap(),
             None => identity::Keypair::generate_ed25519(),
@@ -139,7 +139,8 @@ impl Network {
                     )),
                     ping: ping::Behaviour::new(ping::Config::new()),
                 })
-            })?
+            })
+            .map_err(|e| P2PNetworkError::NewBehaviourError(format!("{e}")))?
             .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60)))
             .build();
 
@@ -214,10 +215,9 @@ impl Network {
                 if let Some(sender) = self.pending_get_peers.remove(&id) {
                     let _ = sender.send(Ok(peers.clone()));
                     debug!("get peers progress {peers:?}");
-                    if let Some(mut q) = self.swarm
-                        .behaviour_mut()
-                        .kademlia
-                        .query_mut(&id) { q.finish(); }
+                    if let Some(mut q) = self.swarm.behaviour_mut().kademlia.query_mut(&id) {
+                        q.finish();
+                    }
                 }
             }
             SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
@@ -231,11 +231,10 @@ impl Network {
                 if let Some(sender) = self.pending_get_peers.remove(&id) {
                     debug!("get closest peers error {e}");
                     if step.last {
-                        let _ = sender.send(Err(anyhow!("{e}")));
-                        if let Some(mut q) = self.swarm
-                            .behaviour_mut()
-                            .kademlia
-                            .query_mut(&id) { q.finish(); }
+                        let _ = sender.send(Err(P2PNetworkError::KadQueryError(e)));
+                        if let Some(mut q) = self.swarm.behaviour_mut().kademlia.query_mut(&id) {
+                            q.finish();
+                        }
                     }
                 }
             }
@@ -293,7 +292,7 @@ impl Network {
                     .pending_request
                     .remove(&request_id)
                     .expect("Request to still be pending.")
-                    .send(Err(anyhow!("{}", error)));
+                    .send(Err(P2PNetworkError::SendRequestFailure(error)));
             }
             SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
                 request_response::Event::ResponseSent { .. },
@@ -324,8 +323,7 @@ impl Network {
                         debug!("connection dialer address {address}");
                     }
                     ConnectedPoint::Listener { send_back_addr, .. } => {
-                        self
-                            .event_sender
+                        self.event_sender
                             .send(Event::IncomeConnection {
                                 peer_id,
                                 connection_id,
@@ -344,8 +342,7 @@ impl Network {
                 endpoint: _,
                 cause,
             } => {
-                self
-                    .event_sender
+                self.event_sender
                     .send(Event::ConnectionClosed {
                         peer_id,
                         connection_id,
@@ -357,7 +354,7 @@ impl Network {
             SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => {
                 if let Some(peer_id) = peer_id {
                     if let Some(sender) = self.pending_dial.remove(&peer_id) {
-                        let _ = sender.send(Err(anyhow!("{}", error)));
+                        let _ = sender.send(Err(P2PNetworkError::DialError(error)));
                     }
                 }
             }
@@ -394,7 +391,7 @@ impl Network {
                             e.insert(sender);
                         }
                         Err(e) => {
-                            let _ = sender.send(Err(anyhow!(e)));
+                            let _ = sender.send(Err(P2PNetworkError::DialError(e)));
                         }
                     }
                 } else {
diff --git a/beta/src/p2p/error.rs b/beta/src/p2p/error.rs
new file mode 100644
index 0000000..e7be54e
--- /dev/null
+++ b/beta/src/p2p/error.rs
@@ -0,0 +1,20 @@
+use libp2p::{
+    kad::GetClosestPeersError, request_response::OutboundFailure, swarm::DialError, Multiaddr,
+};
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum P2PNetworkError {
+    #[error("kad query error")]
+    KadQueryError(#[from] GetClosestPeersError),
+    #[error("send request outbound failure")]
+    SendRequestFailure(#[from] OutboundFailure),
+    #[error("dial error")]
+    DialError(#[from] DialError),
+    #[error("kk")]
+    NewNetworkError(#[from] libp2p::noise::Error),
+    #[error("new behaviour error")]
+    NewBehaviourError(String),
+    #[error("multi addr format error")]
+    MultiAddrFormatError(Multiaddr),
+}
diff --git a/beta/src/p2p/mod.rs b/beta/src/p2p/mod.rs
index 5465fa9..62a2080 100644
--- a/beta/src/p2p/mod.rs
+++ b/beta/src/p2p/mod.rs
@@ -1,2 +1,3 @@
 pub mod behaviour;
-pub mod utils;
\ No newline at end of file
+pub mod utils;
+pub mod error;
\ No newline at end of file
diff --git a/beta/src/p2p/utils.rs b/beta/src/p2p/utils.rs
index 2ef8f44..3722b86 100644
--- a/beta/src/p2p/utils.rs
+++ b/beta/src/p2p/utils.rs
@@ -1,6 +1,6 @@
 use std::time::Duration;
 
-use anyhow::{anyhow, Result};
+use crate::p2p::error::P2PNetworkError;
 use futures::{
     channel::{mpsc, oneshot},
     SinkExt,
@@ -13,11 +13,11 @@ use crate::{p2p::behaviour::PeerRequest, reqres_proto::PeerRequestMessage};
 
 use super::behaviour::Action;
 
-pub async fn bootstrap_light(
+pub async fn bootstrap_peer(
     mut action_sender: mpsc::Sender<Action>,
     full_node: Multiaddr,
     address_local: Multiaddr,
-) -> Result<()> {
+) -> Result<(), P2PNetworkError> {
     if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
         let (sender, receiver) = oneshot::channel();
         // dial
@@ -51,6 +51,6 @@ pub async fn bootstrap_light(
         receiver.await.unwrap().unwrap();
         Ok(())
     } else {
-        Err(anyhow!("full node address format error {}", full_node))
+        Err(P2PNetworkError::MultiAddrFormatError(full_node))
     }
 }
diff --git a/beta/src/vrf.rs b/beta/src/vrf.rs
index 032245d..f6f6f4d 100644
--- a/beta/src/vrf.rs
+++ b/beta/src/vrf.rs
@@ -1,18 +1,28 @@
 #![allow(dead_code)]
-use anyhow::{anyhow, Ok, Result};
+use std::io;
+
 use rand::{rngs::OsRng, RngCore};
 use schnorrkel::{
     context::SigningContext,
     points::RistrettoBoth,
     signing_context,
     vrf::{VRFInOut, VRFProof},
-    ExpansionMode, MiniSecretKey, PublicKey,
+    ExpansionMode, MiniSecretKey, PublicKey, SignatureError,
 };
+use thiserror::Error;
 
 pub type VrfHash = [u8; 32];
 
 pub type VrfProof = [u8; 64];
 
+#[derive(Error, Debug)]
+pub enum VrfError {
+    #[error("sigature error")]
+    SignatureError(SignatureError),
+    #[error("io error")]
+    IOError(#[from] io::Error),
+}
+
 #[derive(Clone, Debug)]
 pub struct VrfPair(schnorrkel::Keypair);
 
@@ -30,14 +40,15 @@ impl VrfPair {
         Self::generate_with_seed(&seed).unwrap()
     }
 
-    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self> {
-        let mini_secret = MiniSecretKey::from_bytes(seed).map_err(|e| anyhow!("{}", e))?;
+    pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self, VrfError> {
+        let mini_secret =
+            MiniSecretKey::from_bytes(seed).map_err(VrfError::SignatureError)?;
         Ok(VrfPair(
             mini_secret.expand_to_keypair(ExpansionMode::Ed25519),
         ))
     }
 
-    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof)> {
+    pub fn vrf_sign(&self, input: &[u8]) -> Result<(VrfHash, VrfProof), VrfError> {
         let (in_out, proof, _) = self
             .0
             .secret
@@ -47,7 +58,12 @@ impl VrfPair {
         Ok((in_out.output.to_bytes(), proof.to_bytes()))
     }
 
-    pub fn vrf_verify(&self, input: &[u8], vrf_hash: VrfHash, proof: VrfProof) -> Result<()> {
+    pub fn vrf_verify(
+        &self,
+        input: &[u8],
+        vrf_hash: VrfHash,
+        proof: VrfProof,
+    ) -> Result<(), VrfError> {
         self.get_public().verify_vrf(input, vrf_hash, proof)
     }
 
@@ -57,24 +73,28 @@ impl VrfPair {
 }
 
 impl VrfPublickey {
-    pub fn verify_vrf(&self, input: &[u8], vrf_hash: VrfHash, proof: VrfProof) -> Result<()> {
+    pub fn verify_vrf(
+        &self,
+        input: &[u8],
+        vrf_hash: VrfHash,
+        proof: VrfProof,
+    ) -> Result<(), VrfError> {
         let output = RistrettoBoth::from_bytes(vrf_hash.as_slice())
-            .map_err(|e| anyhow!("ristretto point from bytes error {}", e))?;
-        let proof =
-            VRFProof::from_bytes(&proof).map_err(|e| anyhow!("parse vrf proof error {}", e))?;
+            .map_err(VrfError::SignatureError)?;
+        let proof = VRFProof::from_bytes(&proof).map_err(VrfError::SignatureError)?;
         let in_out = VRFInOut {
             input: self.0.vrf_hash(context().bytes(input)),
             output,
         };
         self.0
             .vrf_verify(context().bytes(input), &in_out.to_preout(), &proof)
-            .map_err(|e| anyhow!("{}", e))?;
+            .map_err(VrfError::SignatureError)?;
         Ok(())
     }
 
-    pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, VrfError> {
         Ok(VrfPublickey(
-            PublicKey::from_bytes(bytes).map_err(|e| anyhow!("{}", e))?,
+            PublicKey::from_bytes(bytes).map_err(VrfError::SignatureError)?,
         ))
     }
 

From 03e831922b8eac0c18d3e02843902460c92b450d Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Tue, 19 Mar 2024 15:01:29 +0800
Subject: [PATCH 05/11] move gossipsub to non-default feature

---
 beta/Cargo.toml           | 3 +++
 beta/src/p2p/behaviour.rs | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/beta/Cargo.toml b/beta/Cargo.toml
index 4d716a4..77e4aeb 100644
--- a/beta/Cargo.toml
+++ b/beta/Cargo.toml
@@ -41,3 +41,6 @@ criterion = "0.3"
 
 [build-dependencies]
 prost-build = "0.12.3"
+
+[features]
+gossipsub = []
\ No newline at end of file
diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index c66e4f5..b7be53e 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -4,11 +4,13 @@ use futures::prelude::*;
 use futures::StreamExt;
 
 use libp2p::core::ConnectedPoint;
+#[cfg(feature = "gossipsub")]
+use libp2p::gossipsub;
 use libp2p::kad::{Caching, Config, QueryId};
 use libp2p::swarm::ConnectionId;
 use libp2p::{
     core::Multiaddr,
-    gossipsub, identify, identity, kad,
+    identify, identity, kad,
     multiaddr::Protocol,
     noise, ping,
     request_response::{self, OutboundRequestId, ProtocolSupport, ResponseChannel},
@@ -37,6 +39,7 @@ pub struct PeerResponse(pub PeerResponseMessage);
 pub struct Behaviour {
     request_response: request_response::cbor::Behaviour<PeerRequest, PeerResponse>,
     kademlia: kad::Behaviour<kad::store::MemoryStore>,
+    #[cfg(feature = "gossipsub")]
     gossipsub: gossipsub::Behaviour,
     identify: identify::Behaviour,
     ping: ping::Behaviour,
@@ -110,6 +113,7 @@ impl Network {
                 yamux::Config::default,
             )?
             .with_behaviour(|key| {
+                #[cfg(feature = "gossipsub")]
                 let gossipsub_config = gossipsub::ConfigBuilder::default()
                     .max_transmit_size(262144)
                     .build()
@@ -128,6 +132,7 @@ impl Network {
                         [(StreamProtocol::new("/reqres"), ProtocolSupport::Full)],
                         request_response::Config::default(),
                     ),
+                    #[cfg(feature = "gossipsub")]
                     gossipsub: gossipsub::Behaviour::new(
                         gossipsub::MessageAuthenticity::Signed(key.clone()),
                         gossipsub_config,
@@ -149,6 +154,7 @@ impl Network {
             .kademlia
             .set_mode(Some(kad::Mode::Server));
 
+        #[cfg(feature = "gossipsub")]
         swarm
             .behaviour_mut()
             .gossipsub
@@ -425,6 +431,7 @@ impl Network {
         self.swarm.local_peer_id().to_owned()
     }
 
+    #[cfg(feature = "gossipsub")]
     fn gossip_topic() -> gossipsub::IdentTopic {
         gossipsub::IdentTopic::new("entropy_gossip")
     }

From 0c8e11024debbb92835e922408e6a7ac27e4473d Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Wed, 20 Mar 2024 16:37:27 +0800
Subject: [PATCH 06/11] format and add comments

---
 beta/src/p2p/behaviour.rs | 15 ++++++++++++++-
 beta/src/p2p/utils.rs     |  7 ++++---
 beta/src/vrf.rs           |  8 ++++----
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index b7be53e..ba370c8 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -28,7 +28,6 @@ use std::time::Duration;
 
 use log::{debug, info, warn};
 
-// Simple file exchange protocol
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct PeerRequest(pub PeerRequestMessage);
 
@@ -45,21 +44,27 @@ pub struct Behaviour {
     ping: ping::Behaviour,
 }
 
+// Action contains commands Network handles from external.
 pub enum Action {
+    // Command to dial another peer.
+    // peer_id is an unique idendity of a peer, peer_addr is a multiaddr including ip, port, transport protocol info.
     Dial {
         peer_id: PeerId,
         peer_addr: Multiaddr,
         sender: oneshot::Sender<Result<(), P2PNetworkError>>,
     },
+    // Command to send request to a peer
     SendRequest {
         peer_id: PeerId,
         msg: PeerRequest,
         sender: oneshot::Sender<Result<PeerResponse, P2PNetworkError>>,
     },
+    // Command to get peers closest to key in the DHT network. Can be used for peer discovery.
     GetPeers {
         key: Vec<u8>,
         sender: oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>,
     },
+    // Command to send response, corresbonding to a request.
     SendResponse {
         response: PeerResponse,
         channel: ResponseChannel<PeerResponse>,
@@ -67,19 +72,24 @@ pub enum Action {
     Bootstrap {},
 }
 
+// Event contains events send from Network to external.
 pub enum Event {
+    // Event when received a request.
     InboundRequest {
         request: PeerRequest,
         channel: ResponseChannel<PeerResponse>,
     },
+    // Event when being dialed and successfully connected to another peer.
     IncomeConnection {
         peer_id: PeerId,
         connection_id: ConnectionId,
     },
+    // Event when connection closed.
     ConnectionClosed {
         peer_id: PeerId,
         connection_id: ConnectionId,
     },
+    // Event when bootstrap done.
     Bootstrap,
 }
 
@@ -94,6 +104,8 @@ pub struct Network {
 }
 
 impl Network {
+    // New P2P network. action_receiver is for receiving external action commands, i.e. dial another peer, send resquest.
+    // event_sender is for sending network events to external, i.e. connection established/closed, request received.
     pub async fn new(
         secret_key_seed: Option<[u8; 32]>,
         action_receiver: mpsc::Receiver<Action>,
@@ -171,6 +183,7 @@ impl Network {
         })
     }
 
+    // Start network listening to multiaddr.
     pub async fn start(mut self, multiaddr: Multiaddr) {
         let peer_id = self.swarm.local_peer_id().to_owned();
         self.swarm
diff --git a/beta/src/p2p/utils.rs b/beta/src/p2p/utils.rs
index 3722b86..0cd2d30 100644
--- a/beta/src/p2p/utils.rs
+++ b/beta/src/p2p/utils.rs
@@ -13,6 +13,7 @@ use crate::{p2p::behaviour::PeerRequest, reqres_proto::PeerRequestMessage};
 
 use super::behaviour::Action;
 
+// Bootstrap_peer dials an existing peer in the p2p network and get discoverable by other peers. 
 pub async fn bootstrap_peer(
     mut action_sender: mpsc::Sender<Action>,
     full_node: Multiaddr,
@@ -20,7 +21,7 @@ pub async fn bootstrap_peer(
 ) -> Result<(), P2PNetworkError> {
     if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
         let (sender, receiver) = oneshot::channel();
-        // dial
+        // Dial peer
         let action = Action::Dial {
             peer_id,
             peer_addr: full_node,
@@ -30,11 +31,11 @@ pub async fn bootstrap_peer(
         receiver.await.unwrap().unwrap();
         debug!("dial {peer_id} done");
 
-        // bootstrap
+        // Bootstrap
         action_sender.send(Action::Bootstrap {}).await.unwrap();
         tokio::time::sleep(Duration::from_secs(3)).await;
 
-        // acknowledge multiaddress
+        // Acknowledge multiaddress
         let (sender, receiver) = oneshot::channel();
         action_sender
             .send(Action::SendRequest {
diff --git a/beta/src/vrf.rs b/beta/src/vrf.rs
index f6f6f4d..7b90a56 100644
--- a/beta/src/vrf.rs
+++ b/beta/src/vrf.rs
@@ -41,8 +41,7 @@ impl VrfPair {
     }
 
     pub fn generate_with_seed(seed: &[u8; 32]) -> Result<Self, VrfError> {
-        let mini_secret =
-            MiniSecretKey::from_bytes(seed).map_err(VrfError::SignatureError)?;
+        let mini_secret = MiniSecretKey::from_bytes(seed).map_err(VrfError::SignatureError)?;
         Ok(VrfPair(
             mini_secret.expand_to_keypair(ExpansionMode::Ed25519),
         ))
@@ -79,8 +78,8 @@ impl VrfPublickey {
         vrf_hash: VrfHash,
         proof: VrfProof,
     ) -> Result<(), VrfError> {
-        let output = RistrettoBoth::from_bytes(vrf_hash.as_slice())
-            .map_err(VrfError::SignatureError)?;
+        let output =
+            RistrettoBoth::from_bytes(vrf_hash.as_slice()).map_err(VrfError::SignatureError)?;
         let proof = VRFProof::from_bytes(&proof).map_err(VrfError::SignatureError)?;
         let in_out = VRFInOut {
             input: self.0.vrf_hash(context().bytes(input)),
@@ -106,6 +105,7 @@ impl VrfPublickey {
 #[cfg(test)]
 mod tests {
     use crate::vrf::VrfPair;
+
     #[test]
     fn vrf_sign_verify() {
         let secret = VrfPair::new();

From 4585e6edc75328e8bede0cba03570b8a34db0ed0 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Wed, 20 Mar 2024 19:55:50 +0800
Subject: [PATCH 07/11] add multi reqres bench

---
 beta/src/bin/net.rs | 72 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 68 insertions(+), 4 deletions(-)

diff --git a/beta/src/bin/net.rs b/beta/src/bin/net.rs
index c256e4c..ff9de88 100644
--- a/beta/src/bin/net.rs
+++ b/beta/src/bin/net.rs
@@ -9,6 +9,7 @@ use futures::future;
 use futures::StreamExt;
 use futures::{
     channel::{mpsc, oneshot},
+    future::join_all,
     SinkExt,
 };
 use libp2p::PeerId;
@@ -16,10 +17,17 @@ use libp2p::{multiaddr::Protocol, Multiaddr};
 use log::info;
 use rand::random;
 
+use std::collections::HashSet;
 use std::error::Error;
 use std::path::PathBuf;
+
 use tokio::task::spawn;
-use tokio::time::{self, Duration};
+use tokio::time::{self, Duration, Instant};
+
+use std::sync::Arc;
+use tokio::sync::Mutex;
+
+const DEFAULT_BENCH_CNT: usize = 100;
 
 #[derive(clap::Parser)]
 #[command(version, about, long_about = None)]
@@ -39,13 +47,17 @@ struct Cli {
     #[arg(short, long, value_name = "KEY")]
     key: Option<PathBuf>,
 
+    #[arg(short, long, value_name = "KEY")]
+    bench_peers: Option<usize>,
+
     /// Turn debugging information on
     #[arg(short, long, value_name = "LOGLEVL")]
     log_level: Option<String>,
 }
 
-#[tokio::main(flavor = "multi_thread", worker_threads = 20)]
+#[tokio::main(flavor = "multi_thread", worker_threads = 8)]
 async fn main() -> Result<(), Box<dyn Error>> {
+    // console_subscriber::init();
     let cli = Cli::parse();
     let log_level = cli.log_level.unwrap_or("info".into());
     let env = Env::default().default_filter_or(&log_level);
@@ -66,6 +78,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
         return Ok(());
     }
 
+    let bench_light_node_cnt: usize = cli.bench_peers.unwrap_or(DEFAULT_BENCH_CNT);
+
     // Init keystore
     let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
     let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
@@ -90,6 +104,9 @@ async fn main() -> Result<(), Box<dyn Error>> {
     // Start P2P network listener
     spawn(network.start(address_local.clone()));
 
+    let connected_peers = Arc::new(Mutex::new(HashSet::new()));
+    let peers_clone = Arc::clone(&connected_peers);
+
     // Demo P2P network event handler
     let mut action_sender_dup = action_sender.clone();
     spawn(async move {
@@ -102,7 +119,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
                             response: PeerResponse(PeerResponseMessage {
                                 id: request.0.id,
                                 command: request.0.command,
-                                data: b"world".to_vec(),
+                                data: b"ack".to_vec(),
                             }),
                             channel,
                         })
@@ -113,12 +130,16 @@ async fn main() -> Result<(), Box<dyn Error>> {
                     peer_id,
                     connection_id,
                 } => {
+                    let mut peers = peers_clone.lock().await;
+                    peers.insert(peer_id);
                     info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
                 }
                 Event::ConnectionClosed {
                     peer_id,
                     connection_id,
                 } => {
+                    let mut peers = peers_clone.lock().await;
+                    peers.remove(&peer_id);
                     info!("connection closed peerID {peer_id} connectionID {connection_id}");
                 }
                 _ => {}
@@ -126,8 +147,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
         }
     });
 
-    // Start light node demo
     if let Some(full_node_addr_str) = cli.full {
+        // Start light node demo
         let full_node: Multiaddr = full_node_addr_str.parse().unwrap();
         if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
             // Bootstrap light node
@@ -174,7 +195,50 @@ async fn main() -> Result<(), Box<dyn Error>> {
         } else {
             panic!("peer addr format error");
         }
+    } else {
+        // Start as full node demo
+        let peers_clone = Arc::clone(&connected_peers);
+        let mut action_sender_dup = action_sender.clone();
+        let mut inter = time::interval(Duration::from_secs(1));
+        loop {
+            inter.tick().await;
+            let peers = peers_clone.lock().await;
+            let mut receivers = vec![];
+            if peers.len() >= bench_light_node_cnt {
+                let start_time = Instant::now();
+                let msg = PeerRequest(PeerRequestMessage {
+                    id: format!("{}", random::<u16>()),
+                    command: "broadcast".to_string(),
+                    data: b"enough peers".to_vec(),
+                });
+                for peer_id in peers.iter() {
+                    let (sender, receiver) = oneshot::channel();
+                    receivers.push(receiver);
+                    action_sender_dup
+                        .send(Action::SendRequest {
+                            peer_id: peer_id.to_owned(),
+                            msg: msg.clone(),
+                            sender,
+                        })
+                        .await
+                        .unwrap();
+                }
+                join_all(receivers.into_iter().map(|r| async {
+                    match r.await {
+                        Ok(Ok(res)) => {
+                            info!("broadcast ack {:?}", res);
+                        }
+                        _ => panic!("response error"),
+                    }
+                }))
+                .await;
+                let dur = Instant::now() - start_time;
+                info!("broadcast and wait ack takes {:?} ms", dur.as_millis());
+                break;
+            }
+        }
     }
+
     let p = future::pending();
     let () = p.await;
     Ok(())

From e3a2c1d502c34fc801e901cb07efa7fe198fae91 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Mon, 25 Mar 2024 21:36:58 +0800
Subject: [PATCH 08/11] add p2p network client & p2p data chunk protocol

---
 Cargo.lock                | 105 +++++++++++++----
 beta/Cargo.toml           |  12 +-
 beta/src/bin/net.rs       | 161 ++++++++------------------
 beta/src/p2p/behaviour.rs |  80 +++++++++++--
 beta/src/p2p/client.rs    | 235 ++++++++++++++++++++++++++++++++++++++
 beta/src/p2p/error.rs     |  10 ++
 beta/src/p2p/mod.rs       |   3 +-
 beta/src/p2p/utils.rs     |  11 +-
 8 files changed, 469 insertions(+), 148 deletions(-)
 create mode 100644 beta/src/p2p/client.rs

diff --git a/Cargo.lock b/Cargo.lock
index cf2a8ab..1ac12dc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -538,6 +538,19 @@ dependencies = [
  "syn 2.0.52",
 ]
 
+[[package]]
+name = "asynchronous-codec"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4057f2c32adbb2fc158e22fb38433c8e9bbf76b75a4732c7c0cbaf695fb65568"
+dependencies = [
+ "bytes 1.5.0",
+ "futures-sink",
+ "futures-util",
+ "memchr",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "asynchronous-codec"
 version = "0.7.0"
@@ -698,6 +711,7 @@ dependencies = [
  "actix",
  "actix-multipart",
  "actix-web",
+ "blake3",
  "bytes 0.5.6",
  "cid",
  "clap 4.5.2",
@@ -706,7 +720,10 @@ dependencies = [
  "env_logger",
  "futures",
  "getrandom",
+ "hex",
  "libp2p",
+ "libp2p-mplex",
+ "libp2p-stream",
  "log",
  "merlin",
  "prost 0.12.3",
@@ -718,6 +735,7 @@ dependencies = [
  "serde",
  "serde_bytes",
  "serde_json",
+ "tempfile",
  "thiserror",
  "tokio",
  "wirehair",
@@ -753,6 +771,19 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "blake3"
+version = "1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -785,9 +816,9 @@ dependencies = [
 
 [[package]]
 name = "bs58"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896"
+checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4"
 dependencies = [
  "tinyvec",
 ]
@@ -984,6 +1015,12 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f"
 
+[[package]]
+name = "constant_time_eq"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2"
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -2239,7 +2276,7 @@ version = "0.46.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d665144a616dadebdc5fff186b1233488cdcd8bfb1223218ff084b6d052c94f7"
 dependencies = [
- "asynchronous-codec",
+ "asynchronous-codec 0.7.0",
  "base64",
  "byteorder",
  "bytes 1.5.0",
@@ -2270,7 +2307,7 @@ version = "0.44.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "20499a945d2f0221fdc6269b3848892c0f370d2ee3e19c7f65a29d8f860f6126"
 dependencies = [
- "asynchronous-codec",
+ "asynchronous-codec 0.7.0",
  "either",
  "futures",
  "futures-bounded",
@@ -2312,7 +2349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5cc5767727d062c4eac74dd812c998f0e488008e82cce9c33b463d38423f9ad2"
 dependencies = [
  "arrayvec",
- "asynchronous-codec",
+ "asynchronous-codec 0.7.0",
  "bytes 1.5.0",
  "either",
  "fnv",
@@ -2374,13 +2411,32 @@ dependencies = [
  "prometheus-client",
 ]
 
+[[package]]
+name = "libp2p-mplex"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e895765e27e30217b25f7cb7ac4686dad1ff80bf2fdeffd1d898566900a924"
+dependencies = [
+ "asynchronous-codec 0.6.2",
+ "bytes 1.5.0",
+ "futures",
+ "libp2p-core",
+ "libp2p-identity",
+ "nohash-hasher",
+ "parking_lot",
+ "rand",
+ "smallvec",
+ "tracing",
+ "unsigned-varint 0.7.2",
+]
+
 [[package]]
 name = "libp2p-noise"
 version = "0.44.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecd0545ce077f6ea5434bcb76e8d0fe942693b4380aaad0d34a358c2bd05793"
 dependencies = [
- "asynchronous-codec",
+ "asynchronous-codec 0.7.0",
  "bytes 1.5.0",
  "curve25519-dalek",
  "futures",
@@ -2464,6 +2520,21 @@ dependencies = [
  "void",
 ]
 
+[[package]]
+name = "libp2p-stream"
+version = "0.1.0-alpha"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e1f34086b787422d4cf148bc0f0c72557a1cb97811dd001f363af7b1f82057e"
+dependencies = [
+ "futures",
+ "libp2p-core",
+ "libp2p-identity",
+ "libp2p-swarm",
+ "rand",
+ "tracing",
+ "void",
+]
+
 [[package]]
 name = "libp2p-swarm"
 version = "0.44.1"
@@ -3130,7 +3201,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447"
 dependencies = [
  "cfg-if",
  "libc",
- "redox_syscall 0.3.5",
+ "redox_syscall",
  "smallvec",
  "windows-targets 0.48.5",
 ]
@@ -3438,7 +3509,7 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "15a0580ab32b169745d7a39db2ba969226ca16738931be152a3209b409de2474"
 dependencies = [
- "asynchronous-codec",
+ "asynchronous-codec 0.7.0",
  "bytes 1.5.0",
  "quick-protobuf",
  "thiserror",
@@ -3573,15 +3644,6 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
-[[package]]
-name = "redox_syscall"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
-dependencies = [
- "bitflags 1.3.2",
-]
-
 [[package]]
 name = "regex"
 version = "1.10.3"
@@ -4178,13 +4240,12 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.9.0"
+version = "3.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
 dependencies = [
  "cfg-if",
  "fastrand",
- "redox_syscall 0.4.1",
  "rustix",
  "windows-sys 0.52.0",
 ]
@@ -4581,6 +4642,10 @@ name = "unsigned-varint"
 version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105"
+dependencies = [
+ "asynchronous-codec 0.6.2",
+ "bytes 1.5.0",
+]
 
 [[package]]
 name = "unsigned-varint"
diff --git a/beta/Cargo.toml b/beta/Cargo.toml
index 77e4aeb..581ea6d 100644
--- a/beta/Cargo.toml
+++ b/beta/Cargo.toml
@@ -13,7 +13,7 @@ path = "src/bin/demo.rs"
 actix = "0.13.2"
 actix-multipart = "0.6.1"
 actix-web = "4.4.1"
-anyhow = "1.0.80"
+blake3 = "1.5.1"
 bytes = "0.5.6"
 cid = "0.11.1"
 clap = {version = "4.5.1", features = ["derive"]}
@@ -21,7 +21,10 @@ ed25519-dalek = {version = "2.0.0", features = ["serde", "rand_core"]}
 env_logger = "0.11.1"
 futures = "0.3.30"
 getrandom = "^0.2.12"
-libp2p = {version = "0.53.2", features = ["tokio", "cbor", "dns", "kad", "noise", "macros", "request-response", "tcp", "yamux", "gossipsub", "identify", "ping"]}
+hex = "0.4.3"
+libp2p = {version = "0.53.2", features = ["tokio", "cbor", "dns", "kad", "noise", "macros", "request-response", "tcp", "yamux", "gossipsub", "identify", "ping", "quic"]}
+libp2p-mplex = "0.41.0"
+libp2p-stream = "0.1.0-alpha"
 log = "0.4.21"
 merlin = {version = "3.0.0", default-features = false}
 prost = "0.12"
@@ -32,9 +35,10 @@ schnorrkel = "0.11.4"
 serde = {version = "1.0", features = ["derive"]}
 serde_bytes = "0.11"
 serde_json = "1"
+tempfile = "3.10.1"
+thiserror = "1.0.58"
 tokio = {version = "1.36.0", features = ["macros", "rt-multi-thread", "tracing"]}
 wirehair = {version = "0.1.0", path = "../wirehair"}
-thiserror = "1.0.58"
 
 [dev-dependencies]
 criterion = "0.3"
@@ -43,4 +47,4 @@ criterion = "0.3"
 prost-build = "0.12.3"
 
 [features]
-gossipsub = []
\ No newline at end of file
+gossipsub = []
diff --git a/beta/src/bin/net.rs b/beta/src/bin/net.rs
index ff9de88..29f5666 100644
--- a/beta/src/bin/net.rs
+++ b/beta/src/bin/net.rs
@@ -1,23 +1,20 @@
 use beta::keystore::KeyStore;
-use beta::p2p::behaviour::{Action, Event, Network, PeerRequest, PeerResponse};
-use beta::p2p::utils::bootstrap_peer;
-use beta::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
+use beta::p2p::behaviour::Network;
+use beta::p2p::client::Client;
+use beta::p2p::utils::random_req_id;
+use beta::reqres_proto::PeerRequestMessage;
 use beta::server;
 use clap::Parser;
 use env_logger::{Builder, Env};
 use futures::future;
-use futures::StreamExt;
-use futures::{
-    channel::{mpsc, oneshot},
-    future::join_all,
-    SinkExt,
-};
+use futures::{channel::mpsc, future::join_all};
 use libp2p::PeerId;
 use libp2p::{multiaddr::Protocol, Multiaddr};
 use log::info;
-use rand::random;
+use rand::distributions::Uniform;
+use rand::{random, Rng};
+use tokio::sync::Mutex;
 
-use std::collections::HashSet;
 use std::error::Error;
 use std::path::PathBuf;
 
@@ -25,7 +22,6 @@ use tokio::task::spawn;
 use tokio::time::{self, Duration, Instant};
 
 use std::sync::Arc;
-use tokio::sync::Mutex;
 
 const DEFAULT_BENCH_CNT: usize = 100;
 
@@ -50,6 +46,9 @@ struct Cli {
     #[arg(short, long, value_name = "KEY")]
     bench_peers: Option<usize>,
 
+    #[arg(short, long, value_name = "Size")]
+    payload_size: Option<usize>,
+
     /// Turn debugging information on
     #[arg(short, long, value_name = "LOGLEVL")]
     log_level: Option<String>,
@@ -57,7 +56,6 @@ struct Cli {
 
 #[tokio::main(flavor = "multi_thread", worker_threads = 8)]
 async fn main() -> Result<(), Box<dyn Error>> {
-    // console_subscriber::init();
     let cli = Cli::parse();
     let log_level = cli.log_level.unwrap_or("info".into());
     let env = Env::default().default_filter_or(&log_level);
@@ -79,6 +77,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
     }
 
     let bench_light_node_cnt: usize = cli.bench_peers.unwrap_or(DEFAULT_BENCH_CNT);
+    let payload_size: usize = cli.payload_size.unwrap_or(1024 * 1024);
 
     // Init keystore
     let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
@@ -86,110 +85,61 @@ async fn main() -> Result<(), Box<dyn Error>> {
     keystore.save_to(path).unwrap();
 
     // Demo p2p
-    let (mut action_sender, action_receiver) = mpsc::channel(0);
-    let (event_sender, mut event_receiver) = mpsc::channel(0);
+    let (action_sender, action_receiver) = mpsc::channel(0);
+    let (event_sender, event_receiver) = mpsc::channel(0);
 
     // New p2p Network
     let network = Network::new(Some(keystore.seed), action_receiver, event_sender)
         .await
         .unwrap();
 
+    let control = network.control();
+
     let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
     info!("start p2p at {port}");
-
     let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap();
+
     let local_peer_id = network.local_peer_id();
     address_local = address_local.with_p2p(local_peer_id).unwrap();
 
+    // New client
+    let client = Client::new(
+        address_local.clone(),
+        action_sender.clone(),
+        Arc::new(Mutex::new(event_receiver)),
+        control,
+    );
+
     // Start P2P network listener
     spawn(network.start(address_local.clone()));
 
-    let connected_peers = Arc::new(Mutex::new(HashSet::new()));
-    let peers_clone = Arc::clone(&connected_peers);
-
-    // Demo P2P network event handler
-    let mut action_sender_dup = action_sender.clone();
-    spawn(async move {
-        while let Some(e) = event_receiver.next().await {
-            match e {
-                Event::InboundRequest { request, channel } => {
-                    info!("handle inboud request {request:?}");
-                    action_sender_dup
-                        .send(Action::SendResponse {
-                            response: PeerResponse(PeerResponseMessage {
-                                id: request.0.id,
-                                command: request.0.command,
-                                data: b"ack".to_vec(),
-                            }),
-                            channel,
-                        })
-                        .await
-                        .unwrap()
-                }
-                Event::IncomeConnection {
-                    peer_id,
-                    connection_id,
-                } => {
-                    let mut peers = peers_clone.lock().await;
-                    peers.insert(peer_id);
-                    info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
-                }
-                Event::ConnectionClosed {
-                    peer_id,
-                    connection_id,
-                } => {
-                    let mut peers = peers_clone.lock().await;
-                    peers.remove(&peer_id);
-                    info!("connection closed peerID {peer_id} connectionID {connection_id}");
-                }
-                _ => {}
-            }
-        }
-    });
+    let client_clone = client.clone();
+    spawn(async move { client_clone.handle_event().await });
 
     if let Some(full_node_addr_str) = cli.full {
         // Start light node demo
         let full_node: Multiaddr = full_node_addr_str.parse().unwrap();
         if let Some(Protocol::P2p(peer_id)) = full_node.iter().last() {
             // Bootstrap light node
-            bootstrap_peer(action_sender.clone(), full_node.clone(), address_local)
-                .await
-                .unwrap();
+            client.bootstrap_light(full_node).await.unwrap();
 
             // Light node send demo requests
             let mut inter = time::interval(Duration::from_secs(10));
             loop {
                 inter.tick().await;
-                let (sender, receiver) = oneshot::channel();
 
                 info!("send request greeting");
-                action_sender
-                    .send(Action::SendRequest {
-                        peer_id,
-                        msg: PeerRequest(PeerRequestMessage {
-                            id: format!("{}", random::<u16>()),
-                            command: "greeting".to_string(),
-                            data: b"hello".to_vec(),
-                        }),
-                        sender,
-                    })
-                    .await
-                    .unwrap();
-
-                let res = receiver.await.unwrap();
+                let msg = PeerRequestMessage {
+                    id: random_req_id(),
+                    command: "greeting".to_string(),
+                    data: b"hello".to_vec(),
+                };
+                let res = client.request(peer_id, msg).await.unwrap();
                 info!("get response {:?}", res);
 
-                let (sender, receiver) = oneshot::channel();
                 // Demo discovery query random peers
                 let key = PeerId::random();
-                action_sender
-                    .send(Action::GetPeers {
-                        key: key.into(),
-                        sender,
-                    })
-                    .await
-                    .unwrap();
-                let res = receiver.await.unwrap();
+                let res = client.get_closest_peers(key).await.unwrap();
                 info!("get closest peers {:?}", res);
             }
         } else {
@@ -197,43 +147,26 @@ async fn main() -> Result<(), Box<dyn Error>> {
         }
     } else {
         // Start as full node demo
-        let peers_clone = Arc::clone(&connected_peers);
-        let mut action_sender_dup = action_sender.clone();
+        let mut rng = rand::thread_rng();
+        let between = Uniform::from(0..=255);
+        let payload: Vec<u8> = (0..payload_size).map(|_| rng.sample(between)).collect();
+
+        // Bench send data large chunk
         let mut inter = time::interval(Duration::from_secs(1));
         loop {
             inter.tick().await;
-            let peers = peers_clone.lock().await;
-            let mut receivers = vec![];
+            let peers = client.get_connected_peers().await;
             if peers.len() >= bench_light_node_cnt {
+                info!("start sending chunks");
                 let start_time = Instant::now();
-                let msg = PeerRequest(PeerRequestMessage {
-                    id: format!("{}", random::<u16>()),
-                    command: "broadcast".to_string(),
-                    data: b"enough peers".to_vec(),
-                });
+                let mut tasks = Vec::new();
                 for peer_id in peers.iter() {
-                    let (sender, receiver) = oneshot::channel();
-                    receivers.push(receiver);
-                    action_sender_dup
-                        .send(Action::SendRequest {
-                            peer_id: peer_id.to_owned(),
-                            msg: msg.clone(),
-                            sender,
-                        })
-                        .await
-                        .unwrap();
+                    let task = client.send_chunk(peer_id.to_owned(), payload.clone());
+                    tasks.push(task);
                 }
-                join_all(receivers.into_iter().map(|r| async {
-                    match r.await {
-                        Ok(Ok(res)) => {
-                            info!("broadcast ack {:?}", res);
-                        }
-                        _ => panic!("response error"),
-                    }
-                }))
-                .await;
+                join_all(tasks).await;
                 let dur = Instant::now() - start_time;
-                info!("broadcast and wait ack takes {:?} ms", dur.as_millis());
+                info!("send chunks and wait ack takes {:?} ms", dur.as_millis());
                 break;
             }
         }
diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index ba370c8..a0d54f5 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -2,6 +2,7 @@ use crate::p2p::error::P2PNetworkError;
 use futures::channel::{mpsc, oneshot};
 use futures::prelude::*;
 use futures::StreamExt;
+use tempfile::NamedTempFile;
 
 use libp2p::core::ConnectedPoint;
 #[cfg(feature = "gossipsub")]
@@ -19,14 +20,18 @@ use libp2p::{
 };
 
 use crate::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
-use libp2p::StreamProtocol;
+use libp2p::{Stream, StreamProtocol};
 use serde::{Deserialize, Serialize};
 use std::collections::{hash_map, HashMap};
 
+use std::io::Write;
 use std::str::FromStr;
 use std::time::Duration;
 
-use log::{debug, info, warn};
+use log::{debug, error, info, warn};
+
+#[derive(Eq, Hash, PartialEq, Clone)]
+pub struct ChunkID(pub Vec<u8>);
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct PeerRequest(pub PeerRequestMessage);
@@ -42,6 +47,7 @@ pub struct Behaviour {
     gossipsub: gossipsub::Behaviour,
     identify: identify::Behaviour,
     ping: ping::Behaviour,
+    stream: libp2p_stream::Behaviour,
 }
 
 // Action contains commands Network handles from external.
@@ -89,6 +95,11 @@ pub enum Event {
         peer_id: PeerId,
         connection_id: ConnectionId,
     },
+    // Chunk Received
+    ChunkReceived {
+        peer_id: PeerId,
+        chunk_id: ChunkID,
+    },
     // Event when bootstrap done.
     Bootstrap,
 }
@@ -116,14 +127,13 @@ impl Network {
             None => identity::Keypair::generate_ed25519(),
         };
         let peer_id = id_keys.public().to_peer_id();
+        let _yamux_config = yamux::Config::default();
+        let mut plex_config = libp2p_mplex::MplexConfig::default();
+        plex_config.set_split_send_size(1024 * 1024 * 10);
 
         let mut swarm = libp2p::SwarmBuilder::with_existing_identity(id_keys)
             .with_tokio()
-            .with_tcp(
-                tcp::Config::default(),
-                noise::Config::new,
-                yamux::Config::default,
-            )?
+            .with_tcp(tcp::Config::default(), noise::Config::new, || plex_config)?
             .with_behaviour(|key| {
                 #[cfg(feature = "gossipsub")]
                 let gossipsub_config = gossipsub::ConfigBuilder::default()
@@ -142,7 +152,8 @@ impl Network {
                     ),
                     request_response: request_response::cbor::Behaviour::new(
                         [(StreamProtocol::new("/reqres"), ProtocolSupport::Full)],
-                        request_response::Config::default(),
+                        request_response::Config::default()
+                            .with_request_timeout(Duration::from_secs(100)),
                     ),
                     #[cfg(feature = "gossipsub")]
                     gossipsub: gossipsub::Behaviour::new(
@@ -155,6 +166,7 @@ impl Network {
                         key.public(),
                     )),
                     ping: ping::Behaviour::new(ping::Config::new()),
+                    stream: libp2p_stream::Behaviour::new(),
                 })
             })
             .map_err(|e| P2PNetworkError::NewBehaviourError(format!("{e}")))?
@@ -196,6 +208,8 @@ impl Network {
         self.swarm.add_external_address(multiaddr.clone());
 
         let mut inter = tokio::time::interval(Duration::from_secs(10));
+        let mut control = self.swarm.behaviour().stream.new_control();
+        let mut incomming_stream = control.accept(Self::stream_protocol()).unwrap();
         loop {
             tokio::select! {
                 event = self.swarm.select_next_some() => self.handle_event(event).await,
@@ -203,6 +217,9 @@ impl Network {
                     Some(c) => self.handle_action(c).await,
                     None=>  return,
                 },
+                Some((peer,stream)) = incomming_stream.next() => {
+                        self.handle_stream(peer,stream).await.unwrap();
+                },
                 _ = inter.tick() => {
                     self.list_peers();
                 }
@@ -210,6 +227,45 @@ impl Network {
         }
     }
 
+    // Handle chunk data sent from another peer, saved as file with data hash as filename.
+    async fn handle_stream(
+        &mut self,
+        peer_id: PeerId,
+        mut stream: Stream,
+    ) -> Result<(), P2PNetworkError> {
+        let mut temp_file = NamedTempFile::new().unwrap();
+        let mut buffer = [0u8; 1024 * 8];
+        let mut hasher = blake3::Hasher::new();
+        loop {
+            match stream.read(&mut buffer).await {
+                Ok(bytes_read) => {
+                    info!("read bytes {bytes_read}");
+                    if bytes_read == 0 {
+                        break;
+                    }
+                    hasher.update(&buffer[..bytes_read]);
+                    temp_file.write_all(&buffer[..bytes_read]).unwrap();
+                }
+                Err(e) => {
+                    error!("handle stream {e:?}");
+                    break;
+                }
+            }
+        }
+        let hash = hasher.finalize();
+
+        temp_file.persist(format!("{}", hash))?;
+        stream.write_all(b"hello").await?;
+        self.event_sender
+            .send(Event::ChunkReceived {
+                peer_id,
+                chunk_id: ChunkID(hash.as_bytes().to_vec()),
+            })
+            .await
+            .unwrap();
+        Ok(())
+    }
+
     async fn handle_event(&mut self, event: SwarmEvent<BehaviourEvent>) {
         match event {
             SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
@@ -458,4 +514,12 @@ impl Network {
             }
         }
     }
+
+    pub fn stream_protocol() -> libp2p::StreamProtocol {
+        libp2p::StreamProtocol::new("/entropy_stream")
+    }
+
+    pub fn control(&self) -> libp2p_stream::Control {
+        self.swarm.behaviour().stream.new_control()
+    }
 }
diff --git a/beta/src/p2p/client.rs b/beta/src/p2p/client.rs
new file mode 100644
index 0000000..8d2969e
--- /dev/null
+++ b/beta/src/p2p/client.rs
@@ -0,0 +1,235 @@
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+};
+
+use crate::{
+    p2p::{behaviour::Network, utils::random_req_id},
+    reqres_proto::{PeerRequestMessage, PeerResponseMessage},
+};
+use futures::{
+    channel::{mpsc, oneshot},
+    AsyncWriteExt, SinkExt, StreamExt,
+};
+use tokio::sync::Mutex;
+
+use libp2p::{request_response::ResponseChannel, Multiaddr, PeerId};
+use libp2p_stream::Control;
+use log::{debug, info};
+
+use super::{
+    behaviour::{Action, ChunkID, Event, PeerRequest, PeerResponse},
+    error::P2PNetworkError,
+    utils::bootstrap_peer,
+};
+
+const CHUNK_RECEIVED_ACK: &str = "chunk_ack";
+const CHUNK_SEND_CMD: &str = "chunk";
+
+pub enum ChunkState {
+    WaitToReceive((ChunkID, ResponseChannel<PeerResponse>, String)),
+    Saved(ChunkID),
+}
+
+/// Client encapsulates the interface for interacting with the Network, handling network events.
+#[derive(Clone)]
+pub struct Client {
+    pub local_multi_address: Multiaddr,
+    pub action_sender: mpsc::Sender<Action>,
+    pub event_receiver: Arc<Mutex<mpsc::Receiver<Event>>>,
+    pub connected_peers: Arc<Mutex<HashSet<PeerId>>>,
+    pub pending_chunks: Arc<Mutex<HashMap<ChunkID, ChunkState>>>,
+    control: Control,
+}
+
+impl Client {
+    pub fn new(
+        local_addr: Multiaddr,
+        action_sender: mpsc::Sender<Action>,
+        event_receiver: Arc<Mutex<mpsc::Receiver<Event>>>,
+        control: Control,
+    ) -> Self {
+        Self {
+            local_multi_address: local_addr,
+            action_sender,
+            event_receiver,
+            connected_peers: Arc::new(Mutex::new(HashSet::new())),
+            pending_chunks: Arc::new(Mutex::new(HashMap::new())),
+            control,
+        }
+    }
+
+    pub async fn get_connected_peers(&self) -> Vec<PeerId> {
+        let peers = self.connected_peers.lock().await;
+        let mut res = Vec::new();
+        for peer in peers.iter() {
+            res.push(peer.to_owned());
+        }
+        res
+    }
+
+    pub async fn bootstrap_light(&self, full_node_addr: Multiaddr) -> Result<(), P2PNetworkError> {
+        let action_sender = self.action_sender.clone();
+        bootstrap_peer(
+            action_sender,
+            full_node_addr,
+            self.local_multi_address.clone(),
+        )
+        .await
+    }
+
+    pub async fn request(
+        &self,
+        peer_id: PeerId,
+        msg: PeerRequestMessage,
+    ) -> Result<PeerResponse, P2PNetworkError> {
+        let mut action_sender = self.action_sender.clone();
+        let (sender, receiver) = oneshot::channel();
+        action_sender
+            .send(Action::SendRequest {
+                peer_id,
+                msg: PeerRequest(msg),
+                sender,
+            })
+            .await
+            .unwrap();
+
+        receiver.await.unwrap()
+    }
+
+    pub async fn get_closest_peers(&self, key: PeerId) -> Result<Vec<PeerId>, P2PNetworkError> {
+        let mut action_sender = self.action_sender.clone();
+        let (sender, receiver) = oneshot::channel();
+        action_sender
+            .send(Action::GetPeers {
+                key: key.into(),
+                sender,
+            })
+            .await
+            .unwrap();
+        receiver.await.unwrap()
+    }
+
+    // Send chunk data to a peer
+    pub async fn send_chunk(&self, peer_id: PeerId, chunk: Vec<u8>) -> Result<(), P2PNetworkError> {
+        let mut control = self.control.clone();
+        let mut action_sender_dup = self.action_sender.clone();
+        let (sender, receiver) = oneshot::channel();
+
+        let mut hasher = blake3::Hasher::new();
+        hasher.update(&chunk);
+        let hash = hasher.finalize();
+        let msg = PeerRequest(PeerRequestMessage {
+            id: random_req_id(),
+            command: CHUNK_SEND_CMD.to_string(),
+            data: hash.as_bytes().to_vec(),
+        });
+
+        action_sender_dup
+            .send(Action::SendRequest {
+                peer_id: peer_id.to_owned(),
+                msg: msg.clone(),
+                sender,
+            })
+            .await
+            .unwrap();
+
+        let mut stream = control
+            .open_stream(peer_id.to_owned(), Network::stream_protocol())
+            .await
+            .map_err(|error| P2PNetworkError::OpenStreamError(format!("{peer_id},{error:?}")))?;
+        stream.write_all(&chunk).await.unwrap();
+        stream.close().await.unwrap();
+        receiver.await.unwrap()?;
+        Ok(())
+    }
+
+    pub async fn handle_event(&self) {
+        let mut action_sender_dup = self.action_sender.clone();
+        let peers_clone = self.connected_peers.clone();
+        let mut event_receiver = self.event_receiver.lock().await;
+        while let Some(e) = event_receiver.next().await {
+            match e {
+                Event::InboundRequest { request, channel } => {
+                    debug!("handle inboud request");
+                    let chunk_cmd = CHUNK_SEND_CMD.to_string();
+                    if request.0.command == chunk_cmd {
+                        let mut pendind_chunks = self.pending_chunks.lock().await;
+                        let chunk_id = ChunkID(request.0.data.clone());
+                        if let Some(ChunkState::Saved(_chunk_id)) = pendind_chunks.remove(&chunk_id)
+                        {
+                            action_sender_dup
+                                .send(Action::SendResponse {
+                                    response: PeerResponse(PeerResponseMessage {
+                                        id: request.0.id,
+                                        command: CHUNK_RECEIVED_ACK.to_string(),
+                                        data: b"ack".to_vec(),
+                                    }),
+                                    channel,
+                                })
+                                .await
+                                .unwrap();
+                        } else {
+                            pendind_chunks.insert(
+                                chunk_id.clone(),
+                                ChunkState::WaitToReceive((chunk_id, channel, request.0.id)),
+                            );
+                        }
+                    } else {
+                        action_sender_dup
+                            .send(Action::SendResponse {
+                                response: PeerResponse(PeerResponseMessage {
+                                    id: request.0.id,
+                                    command: request.0.command,
+                                    data: b"ack".to_vec(),
+                                }),
+                                channel,
+                            })
+                            .await
+                            .unwrap();
+                    }
+                }
+                Event::ChunkReceived {
+                    peer_id: _,
+                    chunk_id,
+                } => {
+                    let mut pendind_chunks = self.pending_chunks.lock().await;
+                    if let Some(ChunkState::WaitToReceive((_chunk_id, channel, req_id))) =
+                        pendind_chunks.remove(&chunk_id)
+                    {
+                        action_sender_dup
+                            .send(Action::SendResponse {
+                                response: PeerResponse(PeerResponseMessage {
+                                    id: req_id,
+                                    command: CHUNK_RECEIVED_ACK.to_string(),
+                                    data: b"ack".to_vec(),
+                                }),
+                                channel,
+                            })
+                            .await
+                            .unwrap();
+                    } else {
+                        pendind_chunks.insert(chunk_id.clone(), ChunkState::Saved(chunk_id));
+                    }
+                }
+                Event::IncomeConnection {
+                    peer_id,
+                    connection_id,
+                } => {
+                    let mut peers = peers_clone.lock().await;
+                    peers.insert(peer_id);
+                    info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
+                }
+                Event::ConnectionClosed {
+                    peer_id,
+                    connection_id,
+                } => {
+                    let mut peers = peers_clone.lock().await;
+                    peers.remove(&peer_id);
+                    info!("connection closed peerID {peer_id} connectionID {connection_id}");
+                }
+                _ => {}
+            }
+        }
+    }
+}
diff --git a/beta/src/p2p/error.rs b/beta/src/p2p/error.rs
index e7be54e..bef8041 100644
--- a/beta/src/p2p/error.rs
+++ b/beta/src/p2p/error.rs
@@ -1,6 +1,10 @@
+use std::io;
+
 use libp2p::{
     kad::GetClosestPeersError, request_response::OutboundFailure, swarm::DialError, Multiaddr,
 };
+
+use tempfile::PersistError;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -17,4 +21,10 @@ pub enum P2PNetworkError {
     NewBehaviourError(String),
     #[error("multi addr format error")]
     MultiAddrFormatError(Multiaddr),
+    #[error("io error")]
+    IOError(#[from] io::Error),
+    #[error("persis file error")]
+    PersisError(#[from] PersistError),
+    #[error("open stream error")]
+    OpenStreamError(String),
 }
diff --git a/beta/src/p2p/mod.rs b/beta/src/p2p/mod.rs
index 62a2080..0e8b03c 100644
--- a/beta/src/p2p/mod.rs
+++ b/beta/src/p2p/mod.rs
@@ -1,3 +1,4 @@
 pub mod behaviour;
 pub mod utils;
-pub mod error;
\ No newline at end of file
+pub mod error;
+pub mod client;
\ No newline at end of file
diff --git a/beta/src/p2p/utils.rs b/beta/src/p2p/utils.rs
index 0cd2d30..c955683 100644
--- a/beta/src/p2p/utils.rs
+++ b/beta/src/p2p/utils.rs
@@ -1,3 +1,5 @@
+use rand::{rngs::OsRng, RngCore};
+
 use std::time::Duration;
 
 use crate::p2p::error::P2PNetworkError;
@@ -13,7 +15,7 @@ use crate::{p2p::behaviour::PeerRequest, reqres_proto::PeerRequestMessage};
 
 use super::behaviour::Action;
 
-// Bootstrap_peer dials an existing peer in the p2p network and get discoverable by other peers. 
+// Bootstrap_peer dials an existing peer in the p2p network and get discoverable by other peers.
 pub async fn bootstrap_peer(
     mut action_sender: mpsc::Sender<Action>,
     full_node: Multiaddr,
@@ -55,3 +57,10 @@ pub async fn bootstrap_peer(
         Err(P2PNetworkError::MultiAddrFormatError(full_node))
     }
 }
+
+pub fn random_req_id() -> String {
+    let mut rng = OsRng;
+    let mut buf = [0u8; 32];
+    rng.fill_bytes(&mut buf);
+    hex::encode(buf)
+}

From f38ca386f16bab3503acff1bf25ecf43c7b2b25d Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Tue, 26 Mar 2024 21:34:46 +0800
Subject: [PATCH 09/11] clear logs

---
 beta/src/p2p/behaviour.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index a0d54f5..11f3c52 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -239,7 +239,6 @@ impl Network {
         loop {
             match stream.read(&mut buffer).await {
                 Ok(bytes_read) => {
-                    info!("read bytes {bytes_read}");
                     if bytes_read == 0 {
                         break;
                     }

From adcdd66cbb987f34bf50569286d2434aa186a89b Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Mon, 1 Apr 2024 11:09:41 +0800
Subject: [PATCH 10/11] add fullnode get/set

---
 beta/Cargo.toml                  |   2 +-
 beta/src/bin/net.rs              |  44 ++-
 beta/src/bin/peer.rs             | 133 ++++++++
 beta/src/codec/wirehair_codec.rs |  24 +-
 beta/src/lib.rs                  |  26 +-
 beta/src/p2p/behaviour.rs        | 332 +++++++++++-------
 beta/src/p2p/client.rs           | 567 +++++++++++++++++++++++++------
 beta/src/p2p/error.rs            |   9 +-
 beta/src/p2p/utils.rs            |  22 +-
 beta/src/peer.rs                 | 333 ++++++++++++++++++
 beta/src/server.rs               |  13 +-
 11 files changed, 1253 insertions(+), 252 deletions(-)
 create mode 100644 beta/src/bin/peer.rs
 create mode 100644 beta/src/peer.rs

diff --git a/beta/Cargo.toml b/beta/Cargo.toml
index 581ea6d..330d241 100644
--- a/beta/Cargo.toml
+++ b/beta/Cargo.toml
@@ -22,7 +22,7 @@ env_logger = "0.11.1"
 futures = "0.3.30"
 getrandom = "^0.2.12"
 hex = "0.4.3"
-libp2p = {version = "0.53.2", features = ["tokio", "cbor", "dns", "kad", "noise", "macros", "request-response", "tcp", "yamux", "gossipsub", "identify", "ping", "quic"]}
+libp2p = {version = "0.53.2", features = ["tokio", "cbor", "dns", "kad", "noise", "macros", "request-response", "tcp", "yamux", "gossipsub", "identify", "ping"]}
 libp2p-mplex = "0.41.0"
 libp2p-stream = "0.1.0-alpha"
 log = "0.4.21"
diff --git a/beta/src/bin/net.rs b/beta/src/bin/net.rs
index 29f5666..78fcc47 100644
--- a/beta/src/bin/net.rs
+++ b/beta/src/bin/net.rs
@@ -3,14 +3,14 @@ use beta::p2p::behaviour::Network;
 use beta::p2p::client::Client;
 use beta::p2p::utils::random_req_id;
 use beta::reqres_proto::PeerRequestMessage;
-use beta::server;
+use beta::{cid, server};
 use clap::Parser;
 use env_logger::{Builder, Env};
 use futures::future;
 use futures::{channel::mpsc, future::join_all};
 use libp2p::PeerId;
 use libp2p::{multiaddr::Protocol, Multiaddr};
-use log::info;
+use log::{error, info};
 use rand::distributions::Uniform;
 use rand::{random, Rng};
 use tokio::sync::Mutex;
@@ -76,34 +76,40 @@ async fn main() -> Result<(), Box<dyn Error>> {
         return Ok(());
     }
 
-    let bench_light_node_cnt: usize = cli.bench_peers.unwrap_or(DEFAULT_BENCH_CNT);
+    let bench_light_node_cnt: usize =
+        cli.bench_peers.unwrap_or(DEFAULT_BENCH_CNT);
     let payload_size: usize = cli.payload_size.unwrap_or(1024 * 1024);
 
     // Init keystore
     let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
-    let keystore = KeyStore::generate_from_file(path.clone()).unwrap_or(KeyStore::generate());
+    let keystore = KeyStore::generate_from_file(path.clone())
+        .unwrap_or(KeyStore::generate());
     keystore.save_to(path).unwrap();
 
     // Demo p2p
-    let (action_sender, action_receiver) = mpsc::channel(0);
-    let (event_sender, event_receiver) = mpsc::channel(0);
+    let (action_sender, action_receiver) = mpsc::channel(100);
+    let (event_sender, event_receiver) = mpsc::channel(100);
 
     // New p2p Network
-    let network = Network::new(Some(keystore.seed), action_receiver, event_sender)
-        .await
-        .unwrap();
+    let network =
+        Network::new(Some(keystore.seed), action_receiver, event_sender)
+            .await
+            .unwrap();
 
     let control = network.control();
 
     let port = cli.port.unwrap_or_else(|| 6000 + random::<u16>() % 100);
     info!("start p2p at {port}");
-    let mut address_local: Multiaddr = format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap();
+    let mut address_local: Multiaddr =
+        format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap();
 
     let local_peer_id = network.local_peer_id();
     address_local = address_local.with_p2p(local_peer_id).unwrap();
+    info!("local peer id {}", local_peer_id.clone().to_string());
 
     // New client
     let client = Client::new(
+        local_peer_id,
         address_local.clone(),
         action_sender.clone(),
         Arc::new(Mutex::new(event_receiver)),
@@ -139,7 +145,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
                 // Demo discovery query random peers
                 let key = PeerId::random();
-                let res = client.get_closest_peers(key).await.unwrap();
+                let res = client.get_closest_peers(key.into()).await.unwrap();
                 info!("get closest peers {:?}", res);
             }
         } else {
@@ -149,7 +155,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
         // Start as full node demo
         let mut rng = rand::thread_rng();
         let between = Uniform::from(0..=255);
-        let payload: Vec<u8> = (0..payload_size).map(|_| rng.sample(between)).collect();
+        let payload: Vec<u8> =
+            (0..payload_size).map(|_| rng.sample(between)).collect();
 
         // Bench send data large chunk
         let mut inter = time::interval(Duration::from_secs(1));
@@ -161,12 +168,21 @@ async fn main() -> Result<(), Box<dyn Error>> {
                 let start_time = Instant::now();
                 let mut tasks = Vec::new();
                 for peer_id in peers.iter() {
-                    let task = client.send_chunk(peer_id.to_owned(), payload.clone());
+                    let task =
+                        client.send_chunk(peer_id.to_owned(), payload.clone());
                     tasks.push(task);
                 }
                 join_all(tasks).await;
                 let dur = Instant::now() - start_time;
-                info!("send chunks and wait ack takes {:?} ms", dur.as_millis());
+                info!(
+                    "send chunks and wait ack takes {:?} ms",
+                    dur.as_millis()
+                );
+                // test get chunk
+                if let Err(e) = client.get_chunk(peers[0], cid(&payload)).await
+                {
+                    error!("get chunk error {e:?}");
+                };
                 break;
             }
         }
diff --git a/beta/src/bin/peer.rs b/beta/src/bin/peer.rs
new file mode 100644
index 0000000..c83e4da
--- /dev/null
+++ b/beta/src/bin/peer.rs
@@ -0,0 +1,133 @@
+use beta::keystore::KeyStore;
+use beta::p2p::behaviour::Network;
+use beta::p2p::client::Client;
+
+use beta::peer::{self, PeerOpt};
+use clap::Parser;
+use env_logger::{Builder, Env};
+use futures::channel::mpsc;
+use futures::future;
+
+use libp2p::{multiaddr::Protocol, Multiaddr};
+use log::info;
+
+use rand::random;
+use tokio::sync::Mutex;
+
+use std::error::Error;
+use std::path::PathBuf;
+
+use tokio::task::spawn;
+
+use std::sync::Arc;
+
+#[derive(clap::Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Sets a p2p network port
+    #[arg(long, value_name = "P2PPORT")]
+    pport: Option<u16>,
+
+    /// Sets a full node server listen port
+    #[arg(long, value_name = "SERVERPORT")]
+    sport: Option<u16>,
+
+    /// Sets full node muldiaddress
+    #[arg(short, long, value_name = "FUllADDR")]
+    full: Option<String>,
+
+    /// Path to keystore
+    #[arg(short, long, value_name = "KEY")]
+    key: Option<PathBuf>,
+
+    /// Max inbound streams
+    #[arg(short, long, value_name = "INBOUND_STREAM_MAX")]
+    inbound_stream_max: Option<usize>,
+
+    /// Max outbound streams
+    #[arg(short, long, value_name = "OUTBOUND_STREAM_MAX")]
+    outbound_stream_max: Option<usize>,
+
+    /// Turn debugging information on
+    #[arg(short, long, value_name = "LOGLEVL")]
+    log_level: Option<String>,
+}
+
+#[tokio::main(flavor = "multi_thread")]
+async fn main() -> Result<(), Box<dyn Error>> {
+    let cli = Cli::parse();
+    let log_level = cli.log_level.unwrap_or("info".into());
+    let env = Env::default().default_filter_or(&log_level);
+    Builder::from_env(env).format_timestamp_millis().init();
+
+    // Init keystore
+    let path = cli.key.unwrap_or(PathBuf::from("./keystore"));
+    let keystore = KeyStore::generate_from_file(path.clone())
+        .unwrap_or(KeyStore::generate());
+    keystore.save_to(path).unwrap();
+
+    // Demo p2p
+    let (action_sender, action_receiver) = mpsc::channel(100);
+    let (event_sender, event_receiver) = mpsc::channel(100);
+
+    // New p2p Network
+    let network =
+        Network::new(Some(keystore.seed), action_receiver, event_sender)
+            .await
+            .unwrap();
+
+    let port = cli.pport.unwrap_or_else(|| 6000 + random::<u16>() % 100);
+    info!("start p2p at {port}");
+    let mut address_local: Multiaddr =
+        format!("/ip4/127.0.0.1/tcp/{}", port).parse().unwrap();
+
+    let local_peer_id = network.local_peer_id();
+    address_local = address_local.with_p2p(local_peer_id).unwrap();
+    info!("local peer id {}", local_peer_id.clone().to_string());
+
+    // New client
+    let p2p_client = Client::new(
+        local_peer_id,
+        address_local.clone(),
+        action_sender.clone(),
+        Arc::new(Mutex::new(event_receiver)),
+        network.control(),
+    );
+
+    // Start P2P network listener
+    spawn(network.start(address_local.clone()));
+
+    let client_clone = p2p_client.clone();
+    spawn(async move { client_clone.handle_event().await });
+
+    match cli.full {
+        Some(full_node_addr_str) => {
+            // Start light node demo
+            let full_node: Multiaddr = full_node_addr_str.parse().unwrap();
+            if let Some(Protocol::P2p(_)) = full_node.iter().last() {
+                // Bootstrap light node
+                p2p_client.bootstrap_light(full_node).await.unwrap();
+            } else {
+                panic!("peer addr format error");
+            }
+        },
+        None => {
+            // Start full node demo
+            let mut peer = peer::FullNodeService::default();
+            peer.start(
+                "0.0.0.0".to_string(),
+                cli.sport.unwrap(),
+                PeerOpt {
+                    codec_opt: None,
+                    max_inbound_stream: cli.inbound_stream_max,
+                    max_oubound_stream: cli.outbound_stream_max,
+                },
+                p2p_client,
+            )
+            .await;
+        },
+    }
+    let p = future::pending();
+    let () = p.await;
+    Ok(())
+}
diff --git a/beta/src/codec/wirehair_codec.rs b/beta/src/codec/wirehair_codec.rs
index 2de3f9d..ab1240b 100644
--- a/beta/src/codec/wirehair_codec.rs
+++ b/beta/src/codec/wirehair_codec.rs
@@ -161,6 +161,8 @@ impl WirehairCodec {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use rand::RngCore;
 
     use crate::codec::wirehair_codec::WirehairCodec;
@@ -174,7 +176,27 @@ mod tests {
 
         let codec = WirehairCodec::new();
         let links = codec.encode(data.clone());
-        let recovered = codec.decode(links, data.len());
+        let mut rm_links = HashMap::new();
+        let max_chunks_cnt = 8;
+        let max_fragment_cnt = 40;
+        let mut chunks_cnt = 0;
+        for (chunk_id, chunk) in links.into_iter() {
+            chunks_cnt += 1;
+            if chunks_cnt > max_chunks_cnt {
+                break;
+            }
+            let mut fragment_cnt = 0;
+            let mut rm_chunk = HashMap::new();
+            for (fragment_id, fragment) in chunk {
+                fragment_cnt += 1;
+                if fragment_cnt > max_fragment_cnt {
+                    break;
+                }
+                rm_chunk.insert(fragment_id, fragment);
+            }
+            rm_links.insert(chunk_id, rm_chunk);
+        }
+        let recovered = codec.decode(rm_links, data.len());
 
         assert_eq!(data, recovered);
     }
diff --git a/beta/src/lib.rs b/beta/src/lib.rs
index 3065055..e0235b3 100644
--- a/beta/src/lib.rs
+++ b/beta/src/lib.rs
@@ -1,13 +1,35 @@
+use serde::{Deserialize, Serialize};
+
 pub mod codec;
-pub mod server;
-pub mod p2p;
 pub mod keystore;
+pub mod p2p;
+pub mod peer;
+pub mod server;
 mod vrf;
 
 pub mod reqres_proto {
     include!(concat!(env!("OUT_DIR"), "/reqres_proto.rs"));
 }
 
+/// CID is an unique id (Sha3) of data content.
+#[derive(
+    Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash, Default,
+)]
+pub struct CID(String);
+
+pub fn cid(content: &[u8]) -> CID {
+    let mut hasher = blake3::Hasher::new();
+    hasher.update(content);
+    let hash = hasher.finalize();
+    CID(hash.to_string())
+}
+
+impl From<CID> for Vec<u8> {
+    fn from(val: CID) -> Self {
+        val.0.into_bytes().to_vec()
+    }
+}
+
 #[cfg(test)]
 pub mod test {
     use actix::Actor;
diff --git a/beta/src/p2p/behaviour.rs b/beta/src/p2p/behaviour.rs
index 11f3c52..7ee8417 100644
--- a/beta/src/p2p/behaviour.rs
+++ b/beta/src/p2p/behaviour.rs
@@ -1,7 +1,9 @@
 use crate::p2p::error::P2PNetworkError;
+use crate::CID;
 use futures::channel::{mpsc, oneshot};
 use futures::prelude::*;
 use futures::StreamExt;
+
 use tempfile::NamedTempFile;
 
 use libp2p::core::ConnectedPoint;
@@ -13,14 +15,17 @@ use libp2p::{
     core::Multiaddr,
     identify, identity, kad,
     multiaddr::Protocol,
-    noise, ping,
-    request_response::{self, OutboundRequestId, ProtocolSupport, ResponseChannel},
+    noise,
+    request_response::{
+        self, OutboundRequestId, ProtocolSupport, ResponseChannel,
+    },
     swarm::{NetworkBehaviour, Swarm, SwarmEvent},
     tcp, yamux, PeerId,
 };
+use tokio::time::interval;
 
 use crate::reqres_proto::{PeerRequestMessage, PeerResponseMessage};
-use libp2p::{Stream, StreamProtocol};
+use libp2p::{ping, Stream, StreamProtocol};
 use serde::{Deserialize, Serialize};
 use std::collections::{hash_map, HashMap};
 
@@ -30,9 +35,6 @@ use std::time::Duration;
 
 use log::{debug, error, info, warn};
 
-#[derive(Eq, Hash, PartialEq, Clone)]
-pub struct ChunkID(pub Vec<u8>);
-
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub struct PeerRequest(pub PeerRequestMessage);
 
@@ -41,7 +43,8 @@ pub struct PeerResponse(pub PeerResponseMessage);
 
 #[derive(NetworkBehaviour)]
 pub struct Behaviour {
-    request_response: request_response::cbor::Behaviour<PeerRequest, PeerResponse>,
+    request_response:
+        request_response::cbor::Behaviour<PeerRequest, PeerResponse>,
     kademlia: kad::Behaviour<kad::store::MemoryStore>,
     #[cfg(feature = "gossipsub")]
     gossipsub: gossipsub::Behaviour,
@@ -50,27 +53,27 @@ pub struct Behaviour {
     stream: libp2p_stream::Behaviour,
 }
 
-// Action contains commands Network handles from external.
+/// Action are commands Network handles from channel.
 pub enum Action {
-    // Command to dial another peer.
-    // peer_id is an unique idendity of a peer, peer_addr is a multiaddr including ip, port, transport protocol info.
+    /// Command to dial another peer.
+    /// peer_id is an unique idendity of a peer, peer_addr is a multiaddr including ip, port, transport protocol info.
     Dial {
         peer_id: PeerId,
         peer_addr: Multiaddr,
         sender: oneshot::Sender<Result<(), P2PNetworkError>>,
     },
-    // Command to send request to a peer
+    /// Command to send request to a peer
     SendRequest {
         peer_id: PeerId,
         msg: PeerRequest,
         sender: oneshot::Sender<Result<PeerResponse, P2PNetworkError>>,
     },
-    // Command to get peers closest to key in the DHT network. Can be used for peer discovery.
+    /// Command to get peers closest to key in the DHT network. Can be used for peer discovery.
     GetPeers {
         key: Vec<u8>,
         sender: oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>,
     },
-    // Command to send response, corresbonding to a request.
+    /// Command to send response, corresbonding to a request.
     SendResponse {
         response: PeerResponse,
         channel: ResponseChannel<PeerResponse>,
@@ -78,45 +81,49 @@ pub enum Action {
     Bootstrap {},
 }
 
-// Event contains events send from Network to external.
+/// Event are events send from Network to channel.
 pub enum Event {
-    // Event when received a request.
+    /// Event when received a request.
     InboundRequest {
         request: PeerRequest,
         channel: ResponseChannel<PeerResponse>,
     },
-    // Event when being dialed and successfully connected to another peer.
+    /// Event when being dialed and successfully connected to another peer.
     IncomeConnection {
         peer_id: PeerId,
         connection_id: ConnectionId,
     },
-    // Event when connection closed.
+    /// Event when connection closed.
     ConnectionClosed {
         peer_id: PeerId,
         connection_id: ConnectionId,
     },
-    // Chunk Received
+    /// Chunk Received
     ChunkReceived {
         peer_id: PeerId,
-        chunk_id: ChunkID,
+        chunk_id: CID,
     },
-    // Event when bootstrap done.
+    /// Event when bootstrap done.
     Bootstrap,
 }
 
+/// Network manages P2P network, handles action commands and pop network events through channel.
 pub struct Network {
     swarm: Swarm<Behaviour>,
     action_receiver: mpsc::Receiver<Action>,
     event_sender: mpsc::Sender<Event>,
-    pending_request:
-        HashMap<OutboundRequestId, oneshot::Sender<Result<PeerResponse, P2PNetworkError>>>,
-    pending_get_peers: HashMap<QueryId, oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>>,
+    pending_request: HashMap<
+        OutboundRequestId,
+        oneshot::Sender<Result<PeerResponse, P2PNetworkError>>,
+    >,
+    pending_get_peers:
+        HashMap<QueryId, oneshot::Sender<Result<Vec<PeerId>, P2PNetworkError>>>,
     pending_dial: HashMap<PeerId, oneshot::Sender<Result<(), P2PNetworkError>>>,
 }
 
 impl Network {
-    // New P2P network. action_receiver is for receiving external action commands, i.e. dial another peer, send resquest.
-    // event_sender is for sending network events to external, i.e. connection established/closed, request received.
+    /// New P2P network. action_receiver is for receiving external action commands, i.e. dial another peer, send resquest.
+    /// event_sender is for sending network events to external, i.e. connection established/closed, request received.
     pub async fn new(
         secret_key_seed: Option<[u8; 32]>,
         action_receiver: mpsc::Receiver<Action>,
@@ -127,13 +134,14 @@ impl Network {
             None => identity::Keypair::generate_ed25519(),
         };
         let peer_id = id_keys.public().to_peer_id();
-        let _yamux_config = yamux::Config::default();
-        let mut plex_config = libp2p_mplex::MplexConfig::default();
-        plex_config.set_split_send_size(1024 * 1024 * 10);
+        let mut yamux_config = yamux::Config::default();
+        yamux_config.set_max_num_streams(1000);
 
         let mut swarm = libp2p::SwarmBuilder::with_existing_identity(id_keys)
             .with_tokio()
-            .with_tcp(tcp::Config::default(), noise::Config::new, || plex_config)?
+            .with_tcp(tcp::Config::default(), noise::Config::new, || {
+                yamux_config
+            })?
             .with_behaviour(|key| {
                 #[cfg(feature = "gossipsub")]
                 let gossipsub_config = gossipsub::ConfigBuilder::default()
@@ -142,7 +150,9 @@ impl Network {
                     .unwrap();
                 let mut kad_config = Config::default();
                 kad_config
-                    .set_protocol_names(vec![StreamProtocol::new("/entropy_kad")])
+                    .set_protocol_names(vec![StreamProtocol::new(
+                        "/entropy_kad",
+                    )])
                     .set_caching(Caching::Enabled { max_peers: 10 });
                 Ok(Behaviour {
                     kademlia: kad::Behaviour::with_config(
@@ -151,9 +161,13 @@ impl Network {
                         kad_config,
                     ),
                     request_response: request_response::cbor::Behaviour::new(
-                        [(StreamProtocol::new("/reqres"), ProtocolSupport::Full)],
+                        [(
+                            StreamProtocol::new("/reqres"),
+                            ProtocolSupport::Full,
+                        )],
                         request_response::Config::default()
-                            .with_request_timeout(Duration::from_secs(100)),
+                            .with_request_timeout(Duration::from_secs(100))
+                            .with_max_concurrent_streams(1000),
                     ),
                     #[cfg(feature = "gossipsub")]
                     gossipsub: gossipsub::Behaviour::new(
@@ -165,12 +179,18 @@ impl Network {
                         "/entropy/0.1.0".into(),
                         key.public(),
                     )),
-                    ping: ping::Behaviour::new(ping::Config::new()),
+                    ping: ping::Behaviour::new(
+                        ping::Config::new()
+                            .with_interval(Duration::from_secs(5))
+                            .with_timeout(Duration::from_secs(10)),
+                    ),
                     stream: libp2p_stream::Behaviour::new(),
                 })
             })
             .map_err(|e| P2PNetworkError::NewBehaviourError(format!("{e}")))?
-            .with_swarm_config(|c| c.with_idle_connection_timeout(Duration::from_secs(60)))
+            .with_swarm_config(|c| {
+                c.with_idle_connection_timeout(Duration::from_secs(600))
+            })
             .build();
 
         swarm
@@ -195,7 +215,7 @@ impl Network {
         })
     }
 
-    // Start network listening to multiaddr.
+    /// Start network listening to multiaddr.
     pub async fn start(mut self, multiaddr: Multiaddr) {
         let peer_id = self.swarm.local_peer_id().to_owned();
         self.swarm
@@ -207,18 +227,33 @@ impl Network {
 
         self.swarm.add_external_address(multiaddr.clone());
 
-        let mut inter = tokio::time::interval(Duration::from_secs(10));
+        let _inter = tokio::time::interval(Duration::from_secs(10));
         let mut control = self.swarm.behaviour().stream.new_control();
-        let mut incomming_stream = control.accept(Self::stream_protocol()).unwrap();
+        let mut incomming_stream =
+            control.accept(Self::stream_protocol()).unwrap();
+
+        //handle stream
+        let event_sender = self.event_sender.clone();
+        tokio::spawn(async move {
+            while let Some((peer, stream)) = incomming_stream.next().await {
+                let event_sender = event_sender.clone();
+                tokio::spawn(async move {
+                    if let Err(e) =
+                        Self::handle_stream(peer, stream, event_sender).await
+                    {
+                        error!("handle stream from peer {} error {}", peer, e);
+                    };
+                });
+            }
+        });
+
+        let mut inter = interval(Duration::from_secs(10));
         loop {
             tokio::select! {
                 event = self.swarm.select_next_some() => self.handle_event(event).await,
                 action = self.action_receiver.next() => match action {
                     Some(c) => self.handle_action(c).await,
-                    None=>  return,
-                },
-                Some((peer,stream)) = incomming_stream.next() => {
-                        self.handle_stream(peer,stream).await.unwrap();
+                    None=> return ,
                 },
                 _ = inter.tick() => {
                     self.list_peers();
@@ -229,9 +264,9 @@ impl Network {
 
     // Handle chunk data sent from another peer, saved as file with data hash as filename.
     async fn handle_stream(
-        &mut self,
         peer_id: PeerId,
         mut stream: Stream,
+        mut event_sender: mpsc::Sender<Event>,
     ) -> Result<(), P2PNetworkError> {
         let mut temp_file = NamedTempFile::new().unwrap();
         let mut buffer = [0u8; 1024 * 8];
@@ -244,56 +279,56 @@ impl Network {
                     }
                     hasher.update(&buffer[..bytes_read]);
                     temp_file.write_all(&buffer[..bytes_read]).unwrap();
-                }
+                },
                 Err(e) => {
-                    error!("handle stream {e:?}");
+                    error!("handle stream error {e:?}");
                     break;
-                }
+                },
             }
         }
         let hash = hasher.finalize();
 
-        temp_file.persist(format!("{}", hash))?;
-        stream.write_all(b"hello").await?;
-        self.event_sender
+        temp_file.persist(hash.to_string())?;
+        event_sender
             .send(Event::ChunkReceived {
                 peer_id,
-                chunk_id: ChunkID(hash.as_bytes().to_vec()),
+                chunk_id: CID(hash.to_string()),
             })
             .await
             .unwrap();
         Ok(())
     }
 
+    // Hanlde network events and push wrapped events to channel.
     async fn handle_event(&mut self, event: SwarmEvent<BehaviourEvent>) {
         match event {
             SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
                 kad::Event::OutboundQueryProgressed {
                     id: _,
-                    result: kad::QueryResult::Bootstrap(res),
+                    result: kad::QueryResult::Bootstrap(_res),
                     ..
                 },
-            )) => {
-                debug!("bootstrap result {res:?}");
-            }
+            )) => {},
 
             SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
                 kad::Event::OutboundQueryProgressed {
                     id,
                     result:
-                        kad::QueryResult::GetClosestPeers(Ok(kad::GetClosestPeersOk { peers, key: _ })),
-                    // stats,
+                        kad::QueryResult::GetClosestPeers(Ok(
+                            kad::GetClosestPeersOk { peers, key: _ },
+                        )),
                     ..
                 },
             )) => {
                 if let Some(sender) = self.pending_get_peers.remove(&id) {
                     let _ = sender.send(Ok(peers.clone()));
-                    debug!("get peers progress {peers:?}");
-                    if let Some(mut q) = self.swarm.behaviour_mut().kademlia.query_mut(&id) {
+                    if let Some(mut q) =
+                        self.swarm.behaviour_mut().kademlia.query_mut(&id)
+                    {
                         q.finish();
                     }
                 }
-            }
+            },
             SwarmEvent::Behaviour(BehaviourEvent::Kademlia(
                 kad::Event::OutboundQueryProgressed {
                     id,
@@ -303,15 +338,17 @@ impl Network {
                 },
             )) => {
                 if let Some(sender) = self.pending_get_peers.remove(&id) {
-                    debug!("get closest peers error {e}");
                     if step.last {
-                        let _ = sender.send(Err(P2PNetworkError::KadQueryError(e)));
-                        if let Some(mut q) = self.swarm.behaviour_mut().kademlia.query_mut(&id) {
+                        let _ =
+                            sender.send(Err(P2PNetworkError::KadQueryError(e)));
+                        if let Some(mut q) =
+                            self.swarm.behaviour_mut().kademlia.query_mut(&id)
+                        {
                             q.finish();
                         }
                     }
                 }
-            }
+            },
             SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
                 request_response::Event::Message { message, .. },
             )) => match message {
@@ -320,65 +357,71 @@ impl Network {
                 } => {
                     let data = request.0.data.clone();
                     if request.0.command == *"multiaddress" {
-                        if let Ok(remote_address) =
-                            Multiaddr::from_str(String::from_utf8(data).unwrap().as_str())
-                        {
+                        if let Ok(remote_address) = Multiaddr::from_str(
+                            String::from_utf8(data).unwrap().as_str(),
+                        ) {
                             if let Some(Protocol::P2p(remote_peer_id)) =
                                 remote_address.iter().last()
                             {
-                                debug!(
-                                    "add address from ack {} {}",
-                                    remote_peer_id.clone(),
-                                    remote_address.clone()
-                                );
                                 self.swarm
                                     .behaviour_mut()
                                     .kademlia
-                                    .add_address(&remote_peer_id, remote_address);
+                                    .add_address(
+                                        &remote_peer_id,
+                                        remote_address,
+                                    );
                             }
                         }
                     }
-                    self.event_sender
-                        .send(Event::InboundRequest {
-                            request: PeerRequest(request.0),
-                            channel,
-                        })
-                        .await
-                        .expect("Event receiver not to be dropped.");
-                }
+                    let mut event_sender = self.event_sender.clone();
+                    tokio::spawn(async move {
+                        event_sender
+                            .send(Event::InboundRequest {
+                                request: PeerRequest(request.0),
+                                channel,
+                            })
+                            .await
+                            .expect("Event receiver not to be dropped.");
+                    });
+                },
                 request_response::Message::Response {
                     request_id,
                     response,
                 } => {
-                    let _ = self
+                    let sender = self
                         .pending_request
                         .remove(&request_id)
-                        .expect("Request to still be pending.")
-                        .send(Ok(response));
-                }
+                        .expect("Request to still be pending.");
+                    if !sender.is_canceled() {
+                        sender.send(Ok(response)).unwrap();
+                    }
+                },
             },
             SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
                 request_response::Event::OutboundFailure {
-                    request_id, error, ..
+                    request_id,
+                    error,
+                    ..
                 },
             )) => {
+                debug!("send request failure error {error}");
                 let _ = self
                     .pending_request
                     .remove(&request_id)
                     .expect("Request to still be pending.")
                     .send(Err(P2PNetworkError::SendRequestFailure(error)));
-            }
+            },
             SwarmEvent::Behaviour(BehaviourEvent::RequestResponse(
                 request_response::Event::ResponseSent { .. },
-            )) => {}
+            )) => {},
             SwarmEvent::NewListenAddr { address, .. } => {
                 let local_peer_id = *self.swarm.local_peer_id();
                 info!(
                     "Local node is listening on {:?}",
                     address.with(Protocol::P2p(local_peer_id))
                 );
-            }
-            SwarmEvent::IncomingConnection { .. } => {}
+            },
+            SwarmEvent::IncomingConnection { .. } => {},
             SwarmEvent::ConnectionEstablished {
                 peer_id,
                 connection_id,
@@ -387,7 +430,8 @@ impl Network {
             } => {
                 match endpoint {
                     ConnectedPoint::Dialer { address, .. } => {
-                        if let Some(sender) = self.pending_dial.remove(&peer_id) {
+                        if let Some(sender) = self.pending_dial.remove(&peer_id)
+                        {
                             let _ = sender.send(Ok(()));
                             self.swarm
                                 .behaviour_mut()
@@ -395,20 +439,23 @@ impl Network {
                                 .add_address(&peer_id, address.clone());
                         }
                         debug!("connection dialer address {address}");
-                    }
+                    },
                     ConnectedPoint::Listener { send_back_addr, .. } => {
-                        self.event_sender
-                            .send(Event::IncomeConnection {
-                                peer_id,
-                                connection_id,
-                            })
-                            .await
-                            .expect("Event receiver not to be dropped.");
-                        debug!("connection listener send back address {send_back_addr}");
-                    }
+                        let mut event_sender = self.event_sender.clone();
+                        tokio::spawn(async move {
+                            event_sender
+                                .send(Event::IncomeConnection {
+                                    peer_id,
+                                    connection_id,
+                                })
+                                .await
+                                .expect("Event receiver not to be dropped.");
+                            debug!("connection listener send back address {send_back_addr}");
+                        });
+                    },
                 }
                 debug!("connected to {peer_id}");
-            }
+            },
             SwarmEvent::ConnectionClosed {
                 peer_id,
                 connection_id,
@@ -416,23 +463,27 @@ impl Network {
                 endpoint: _,
                 cause,
             } => {
-                self.event_sender
-                    .send(Event::ConnectionClosed {
-                        peer_id,
-                        connection_id,
-                    })
-                    .await
-                    .expect("Event receiver not to be dropped.");
-                debug!("connection closed {peer_id}, cause {cause:?}");
-            }
+                let mut event_sender = self.event_sender.clone();
+                tokio::spawn(async move {
+                    event_sender
+                        .send(Event::ConnectionClosed {
+                            peer_id,
+                            connection_id,
+                        })
+                        .await
+                        .expect("Event receiver not to be dropped.");
+                    debug!("connection closed {peer_id}, cause {cause:?}");
+                });
+            },
             SwarmEvent::OutgoingConnectionError { peer_id, error, .. } => {
                 if let Some(peer_id) = peer_id {
                     if let Some(sender) = self.pending_dial.remove(&peer_id) {
-                        let _ = sender.send(Err(P2PNetworkError::DialError(error)));
+                        let _ =
+                            sender.send(Err(P2PNetworkError::DialError(error)));
                     }
                 }
-            }
-            SwarmEvent::IncomingConnectionError { .. } => {}
+            },
+            SwarmEvent::IncomingConnectionError { .. } => {},
             SwarmEvent::Dialing {
                 peer_id: Some(peer_id),
                 ..
@@ -441,38 +492,46 @@ impl Network {
         }
     }
 
+    // handle Action commands to P2P network
     async fn handle_action(&mut self, command: Action) {
         match command {
             Action::Bootstrap {} => {
-                let _ = self.swarm.behaviour_mut().kademlia.bootstrap().unwrap();
-            }
+                let _ =
+                    self.swarm.behaviour_mut().kademlia.bootstrap().unwrap();
+            },
             Action::GetPeers { key, sender } => {
-                let query_id = self.swarm.behaviour_mut().kademlia.get_closest_peers(key);
+                let query_id =
+                    self.swarm.behaviour_mut().kademlia.get_closest_peers(key);
                 self.pending_get_peers.insert(query_id, sender);
-            }
+            },
             Action::Dial {
                 peer_id,
                 peer_addr,
                 sender,
             } => {
-                if let hash_map::Entry::Vacant(e) = self.pending_dial.entry(peer_id) {
+                if let hash_map::Entry::Vacant(e) =
+                    self.pending_dial.entry(peer_id)
+                {
                     self.swarm
                         .behaviour_mut()
                         .kademlia
                         .add_address(&peer_id, peer_addr.clone());
-                    match self.swarm.dial(peer_addr.with(Protocol::P2p(peer_id))) {
+                    match self
+                        .swarm
+                        .dial(peer_addr.with(Protocol::P2p(peer_id)))
+                    {
                         Ok(()) => {
                             e.insert(sender);
-                        }
+                        },
                         Err(e) => {
-                            let _ = sender.send(Err(P2PNetworkError::DialError(e)));
-                        }
+                            let _ =
+                                sender.send(Err(P2PNetworkError::DialError(e)));
+                        },
                     }
                 } else {
-                    // already dialing
                     warn!("already dialing {peer_id}")
                 }
-            }
+            },
             Action::SendRequest {
                 peer_id,
                 msg,
@@ -484,14 +543,24 @@ impl Network {
                     .request_response
                     .send_request(&peer_id, msg);
                 self.pending_request.insert(req_id, sender);
-            }
+            },
             Action::SendResponse { response, channel } => {
-                self.swarm
-                    .behaviour_mut()
-                    .request_response
-                    .send_response(channel, response)
-                    .unwrap();
-            }
+                if channel.is_open() {
+                    if let Err(e) = self
+                        .swarm
+                        .behaviour_mut()
+                        .request_response
+                        .send_response(channel, response.clone())
+                    {
+                        error!(
+                            "send response error cmd {} error {:?}",
+                            response.0.command, e,
+                        );
+                    }
+                } else {
+                    debug!("channel closed");
+                }
+            },
         }
     }
 
@@ -504,11 +573,12 @@ impl Network {
         gossipsub::IdentTopic::new("entropy_gossip")
     }
 
+    // list peers in local DHT bucket.
     fn list_peers(&mut self) {
         for bucket in self.swarm.behaviour_mut().kademlia.kbuckets() {
             if bucket.num_entries() > 0 {
                 for item in bucket.iter() {
-                    debug!("Peer ID: {:?}", item.node.key);
+                    debug!("local bucket peers peerid: {:?}", item.node.key);
                 }
             }
         }
diff --git a/beta/src/p2p/client.rs b/beta/src/p2p/client.rs
index 8d2969e..65425e0 100644
--- a/beta/src/p2p/client.rs
+++ b/beta/src/p2p/client.rs
@@ -1,64 +1,122 @@
 use std::{
-    collections::{HashMap, HashSet},
+    collections::{hash_map::Entry, HashMap, HashSet},
+    fmt::Debug,
+    io::Read,
     sync::Arc,
 };
 
 use crate::{
+    cid,
     p2p::{behaviour::Network, utils::random_req_id},
     reqres_proto::{PeerRequestMessage, PeerResponseMessage},
+    CID,
 };
+
 use futures::{
     channel::{mpsc, oneshot},
     AsyncWriteExt, SinkExt, StreamExt,
 };
+use serde::{Deserialize, Serialize};
 use tokio::sync::Mutex;
 
 use libp2p::{request_response::ResponseChannel, Multiaddr, PeerId};
 use libp2p_stream::Control;
-use log::{debug, info};
+use log::{debug, error, info};
 
 use super::{
-    behaviour::{Action, ChunkID, Event, PeerRequest, PeerResponse},
+    behaviour::{Action, Event, PeerRequest, PeerResponse},
     error::P2PNetworkError,
     utils::bootstrap_peer,
 };
+use rand::{seq::SliceRandom, thread_rng};
+use tokio::sync::Semaphore;
+
+const PUSH_CHUNK_CMD: &str = "chunk_push";
+const GET_CHUNK_CMD: &str = "chunk_get";
+const PUSH_CHUNK_ACK_CMD: &str = "chunk_push_ack";
+const GET_CHUNK_ACK_CMD: &str = "chunk_get_ack";
+
+const MAX_PENDING_SEND_PER_PEER: usize = 3;
+const MAX_PENDING_GET_PER_PEER: usize = 1;
+
+const CHUNK_SEND_TIMEOUT_DURATION: tokio::time::Duration =
+    tokio::time::Duration::from_millis(50);
 
-const CHUNK_RECEIVED_ACK: &str = "chunk_ack";
-const CHUNK_SEND_CMD: &str = "chunk";
+/// P2P network commands send through request_response protocol.
+pub enum PeerCommand {
+    /// Send PushChunk request to a peer then send chunk data with stream protocol.
+    PushChunk,
+    /// Send PushChunkAck response when get certain chunk data with stream protocol.
+    PushChunkAck,
+    /// Send GetChunk request to a peer, ask that peer to send back certain data chunk with stream protocol.
+    GetChunk,
+    /// Send GetChunkAck response when get chunk data asked for.
+    GetChunkAck,
+    /// None specific command.
+    Other,
+}
+
+impl From<String> for PeerCommand {
+    fn from(cmd: String) -> Self {
+        match cmd.as_str() {
+            PUSH_CHUNK_ACK_CMD => PeerCommand::PushChunkAck,
+            PUSH_CHUNK_CMD => PeerCommand::PushChunk,
+            GET_CHUNK_CMD => PeerCommand::GetChunk,
+            GET_CHUNK_ACK_CMD => PeerCommand::GetChunkAck,
+            _ => PeerCommand::Other,
+        }
+    }
+}
 
+#[derive(Debug)]
 pub enum ChunkState {
-    WaitToReceive((ChunkID, ResponseChannel<PeerResponse>, String)),
-    Saved(ChunkID),
+    WaitToReceive(
+        (
+            CID,
+            ResponseChannel<PeerResponse>,
+            String,
+            Option<oneshot::Sender<()>>,
+        ),
+    ), //(cid, response channel, request id)
+    Saved(CID),
 }
 
 /// Client encapsulates the interface for interacting with the Network, handling network events.
 #[derive(Clone)]
 pub struct Client {
     pub local_multi_address: Multiaddr,
+    pub local_peer_id: PeerId,
     pub action_sender: mpsc::Sender<Action>,
     pub event_receiver: Arc<Mutex<mpsc::Receiver<Event>>>,
     pub connected_peers: Arc<Mutex<HashSet<PeerId>>>,
-    pub pending_chunks: Arc<Mutex<HashMap<ChunkID, ChunkState>>>,
+    pub pending_chunks: Arc<Mutex<HashMap<CID, ChunkState>>>,
+    pending_sends: Arc<Mutex<HashMap<PeerId, Arc<Semaphore>>>>,
+    pending_gets: Arc<Mutex<HashMap<PeerId, Arc<Semaphore>>>>,
     control: Control,
 }
 
 impl Client {
     pub fn new(
+        local_peer_id: PeerId,
         local_addr: Multiaddr,
         action_sender: mpsc::Sender<Action>,
         event_receiver: Arc<Mutex<mpsc::Receiver<Event>>>,
         control: Control,
     ) -> Self {
         Self {
+            local_peer_id,
             local_multi_address: local_addr,
             action_sender,
             event_receiver,
             connected_peers: Arc::new(Mutex::new(HashSet::new())),
             pending_chunks: Arc::new(Mutex::new(HashMap::new())),
+            pending_sends: Arc::new(Mutex::new(HashMap::new())),
+            pending_gets: Arc::new(Mutex::new(HashMap::new())),
             control,
         }
     }
 
+    /// Get connected peers
     pub async fn get_connected_peers(&self) -> Vec<PeerId> {
         let peers = self.connected_peers.lock().await;
         let mut res = Vec::new();
@@ -68,7 +126,11 @@ impl Client {
         res
     }
 
-    pub async fn bootstrap_light(&self, full_node_addr: Multiaddr) -> Result<(), P2PNetworkError> {
+    /// Bootstrap light node when startup. Dial full node and broadcast self multiaddress.
+    pub async fn bootstrap_light(
+        &self,
+        full_node_addr: Multiaddr,
+    ) -> Result<(), P2PNetworkError> {
         let action_sender = self.action_sender.clone();
         bootstrap_peer(
             action_sender,
@@ -78,6 +140,7 @@ impl Client {
         .await
     }
 
+    /// Send request and wait for response
     pub async fn request(
         &self,
         peer_id: PeerId,
@@ -97,53 +160,122 @@ impl Client {
         receiver.await.unwrap()
     }
 
-    pub async fn get_closest_peers(&self, key: PeerId) -> Result<Vec<PeerId>, P2PNetworkError> {
+    pub async fn get_closest_peers(
+        &self,
+        key: Vec<u8>,
+    ) -> Result<Vec<PeerId>, P2PNetworkError> {
         let mut action_sender = self.action_sender.clone();
         let (sender, receiver) = oneshot::channel();
         action_sender
-            .send(Action::GetPeers {
-                key: key.into(),
-                sender,
-            })
+            .send(Action::GetPeers { key, sender })
             .await
             .unwrap();
         receiver.await.unwrap()
     }
 
-    // Send chunk data to a peer
-    pub async fn send_chunk(&self, peer_id: PeerId, chunk: Vec<u8>) -> Result<(), P2PNetworkError> {
-        let mut control = self.control.clone();
-        let mut action_sender_dup = self.action_sender.clone();
-        let (sender, receiver) = oneshot::channel();
+    /// Ask for chunk data from a peer
+    pub async fn get_chunk(
+        &self,
+        peer_id: PeerId,
+        chunk_id: CID,
+    ) -> Result<(), P2PNetworkError> {
+        let (done_sender, done_recv) = oneshot::channel();
+        let action_sender = self.action_sender.clone();
+        let local_peer_id = self.local_peer_id;
 
-        let mut hasher = blake3::Hasher::new();
-        hasher.update(&chunk);
-        let hash = hasher.finalize();
-        let msg = PeerRequest(PeerRequestMessage {
-            id: random_req_id(),
-            command: CHUNK_SEND_CMD.to_string(),
-            data: hash.as_bytes().to_vec(),
-        });
+        let mut pending_gets = self.pending_gets.lock().await;
+        let sema = match pending_gets.entry(peer_id) {
+            Entry::Occupied(o) => {
+                let sema = o.into_mut().clone();
+                sema
+            },
+            Entry::Vacant(v) => {
+                let sema = Arc::new(Semaphore::new(MAX_PENDING_GET_PER_PEER));
+                let sema_clone = sema.clone();
+                v.insert(sema);
+                sema_clone
+            },
+        };
+        drop(pending_gets);
 
-        action_sender_dup
-            .send(Action::SendRequest {
-                peer_id: peer_id.to_owned(),
-                msg: msg.clone(),
-                sender,
-            })
-            .await
-            .unwrap();
+        let _ = tokio::spawn(async move {
+            let permit = sema.acquire().await.unwrap();
+            get_chunk_inner(
+                local_peer_id,
+                action_sender,
+                peer_id,
+                chunk_id,
+                done_sender,
+            )
+            .await;
+            drop(permit);
+        })
+        .await;
+        done_recv.await.unwrap()
+    }
 
-        let mut stream = control
-            .open_stream(peer_id.to_owned(), Network::stream_protocol())
-            .await
-            .map_err(|error| P2PNetworkError::OpenStreamError(format!("{peer_id},{error:?}")))?;
-        stream.write_all(&chunk).await.unwrap();
-        stream.close().await.unwrap();
-        receiver.await.unwrap()?;
-        Ok(())
+    /// Send chunk data to a peer
+    pub async fn send_chunk(
+        &self,
+        peer_id: PeerId,
+        chunk: Vec<u8>,
+    ) -> Result<(), P2PNetworkError> {
+        let (done_sender, done_recv) = oneshot::channel();
+        let control = self.control.clone();
+        let action_sender = self.action_sender.clone();
+
+        let mut pending_sends = self.pending_sends.lock().await;
+        let sema = match pending_sends.entry(peer_id) {
+            Entry::Occupied(o) => {
+                let sema = o.into_mut().clone();
+                sema
+            },
+            Entry::Vacant(v) => {
+                let sema = Arc::new(Semaphore::new(MAX_PENDING_SEND_PER_PEER));
+                let sema_clone = sema.clone();
+                v.insert(sema);
+                sema_clone
+            },
+        };
+        drop(pending_sends);
+
+        let _ = tokio::spawn(async move {
+            let permit = sema.acquire().await.unwrap();
+            send_chunk_inner(
+                control,
+                action_sender,
+                peer_id,
+                chunk,
+                done_sender,
+            )
+            .await;
+            drop(permit);
+        })
+        .await;
+        done_recv.await.unwrap()
     }
 
+    /// Choose random peers from DHT closest to key.
+    pub async fn random_closest_peer(
+        &self,
+        _key: Vec<u8>,
+        cnt: usize,
+    ) -> Result<Vec<PeerId>, P2PNetworkError> {
+        let connected_peers = self.connected_peers.lock().await;
+        // todo: vrf select
+        let v = connected_peers
+            .iter()
+            .map(|x| x.to_owned())
+            .collect::<Vec<PeerId>>();
+        let mut rng = thread_rng();
+        let chosen = v
+            .choose_multiple(&mut rng, cnt)
+            .map(|x| x.to_owned())
+            .collect::<Vec<PeerId>>();
+        Ok(chosen)
+    }
+
+    /// Handle network events
     pub async fn handle_event(&self) {
         let mut action_sender_dup = self.action_sender.clone();
         let peers_clone = self.connected_peers.clone();
@@ -151,67 +283,177 @@ impl Client {
         while let Some(e) = event_receiver.next().await {
             match e {
                 Event::InboundRequest { request, channel } => {
-                    debug!("handle inboud request");
-                    let chunk_cmd = CHUNK_SEND_CMD.to_string();
-                    if request.0.command == chunk_cmd {
-                        let mut pendind_chunks = self.pending_chunks.lock().await;
-                        let chunk_id = ChunkID(request.0.data.clone());
-                        if let Some(ChunkState::Saved(_chunk_id)) = pendind_chunks.remove(&chunk_id)
-                        {
+                    match PeerCommand::from(request.0.command.clone()) {
+                        PeerCommand::PushChunkAck => {
+                            debug!("PushChunkAck req");
+                        },
+                        PeerCommand::GetChunkAck => {
+                            debug!("PushChunkAck req");
+                        },
+                        PeerCommand::PushChunk => {
+                            debug!("push chunk req");
+                            let mut pendind_chunks =
+                                self.pending_chunks.lock().await;
+
+                            let chunk_id =
+                                match serde_json::from_slice::<PushChunkReq>(
+                                    &request.0.data,
+                                ) {
+                                    Ok(req) => req.chunk_id,
+                                    Err(e) => {
+                                        error!("parse error {:?}", e);
+                                        continue;
+                                    },
+                                };
+                            if pendind_chunks.contains_key(&chunk_id) {
+                                action_sender_dup
+                                    .send(Action::SendResponse {
+                                        response: PeerResponse(
+                                            PeerResponseMessage {
+                                                id: request.0.id,
+                                                command: PUSH_CHUNK_ACK_CMD
+                                                    .to_string(),
+                                                data: b"ack".to_vec(),
+                                            },
+                                        ),
+                                        channel,
+                                    })
+                                    .await
+                                    .unwrap();
+                            } else {
+                                pendind_chunks.insert(
+                                    chunk_id.clone(),
+                                    ChunkState::WaitToReceive((
+                                        chunk_id,
+                                        channel,
+                                        request.0.id,
+                                        None,
+                                    )),
+                                );
+                            }
+                        },
+                        PeerCommand::GetChunk => {
+                            let req = match serde_json::from_slice::<GetChunkReq>(
+                                &request.0.data,
+                            ) {
+                                Ok(req) => req,
+                                Err(_) => {
+                                    continue;
+                                },
+                            };
+
+                            let peer_id = match PeerId::from_bytes(&req.peer_id)
+                            {
+                                Ok(id) => id,
+                                Err(_) => {
+                                    continue;
+                                },
+                            };
+                            //load chunk from file
+                            let mut file = match std::fs::File::open(
+                                req.chunk_id.0.as_str(),
+                            ) {
+                                Ok(file) => file,
+                                Err(e) => {
+                                    error!("open file {:?}", e);
+                                    continue;
+                                },
+                            };
+                            let mut chunk = Vec::new();
+                            if let Err(e) = file.read_to_end(&mut chunk) {
+                                error!("read file {:?}", e);
+                                continue;
+                            }
+                            //send chunk
+                            let control = self.control.clone();
+                            let action_sender = self.action_sender.clone();
+                            let (done_sender, done_recv) = oneshot::channel();
+                            tokio::spawn(async move {
+                                let mut action_sender_dup =
+                                    action_sender.clone();
+                                send_chunk_inner(
+                                    control,
+                                    action_sender,
+                                    peer_id,
+                                    chunk,
+                                    done_sender,
+                                )
+                                .await;
+                                if let Err(e) = done_recv.await.unwrap() {
+                                    error!(
+                                        "handle send chunk {} to {} error {}",
+                                        req.chunk_id.0.clone(),
+                                        peer_id,
+                                        e
+                                    );
+                                };
+                                //todo: handle error
+                                action_sender_dup
+                                    .send(Action::SendResponse {
+                                        response: PeerResponse(
+                                            PeerResponseMessage {
+                                                id: request.0.id,
+                                                command: GET_CHUNK_ACK_CMD
+                                                    .to_string(),
+                                                data: b"ack".to_vec(),
+                                            },
+                                        ),
+                                        channel,
+                                    })
+                                    .await
+                                    .unwrap();
+                            });
+                        },
+                        PeerCommand::Other => {
                             action_sender_dup
                                 .send(Action::SendResponse {
-                                    response: PeerResponse(PeerResponseMessage {
-                                        id: request.0.id,
-                                        command: CHUNK_RECEIVED_ACK.to_string(),
-                                        data: b"ack".to_vec(),
-                                    }),
+                                    response: PeerResponse(
+                                        PeerResponseMessage {
+                                            id: request.0.id,
+                                            command: request.0.command,
+                                            data: b"ack".to_vec(),
+                                        },
+                                    ),
                                     channel,
                                 })
                                 .await
                                 .unwrap();
-                        } else {
-                            pendind_chunks.insert(
-                                chunk_id.clone(),
-                                ChunkState::WaitToReceive((chunk_id, channel, request.0.id)),
-                            );
-                        }
-                    } else {
-                        action_sender_dup
-                            .send(Action::SendResponse {
-                                response: PeerResponse(PeerResponseMessage {
-                                    id: request.0.id,
-                                    command: request.0.command,
-                                    data: b"ack".to_vec(),
-                                }),
-                                channel,
-                            })
-                            .await
-                            .unwrap();
+                        },
                     }
-                }
+                },
                 Event::ChunkReceived {
                     peer_id: _,
                     chunk_id,
                 } => {
                     let mut pendind_chunks = self.pending_chunks.lock().await;
-                    if let Some(ChunkState::WaitToReceive((_chunk_id, channel, req_id))) =
-                        pendind_chunks.remove(&chunk_id)
-                    {
-                        action_sender_dup
-                            .send(Action::SendResponse {
-                                response: PeerResponse(PeerResponseMessage {
-                                    id: req_id,
-                                    command: CHUNK_RECEIVED_ACK.to_string(),
-                                    data: b"ack".to_vec(),
-                                }),
-                                channel,
-                            })
-                            .await
-                            .unwrap();
-                    } else {
-                        pendind_chunks.insert(chunk_id.clone(), ChunkState::Saved(chunk_id));
-                    }
-                }
+                    match pendind_chunks.remove(&chunk_id) {
+                        Some(ChunkState::WaitToReceive((
+                            _chunk_id,
+                            channel,
+                            req_id,
+                            ..,
+                        ))) => {
+                            action_sender_dup
+                                .send(Action::SendResponse {
+                                    response: PeerResponse(
+                                        PeerResponseMessage {
+                                            id: req_id,
+                                            command: PUSH_CHUNK_ACK_CMD
+                                                .to_string(),
+                                            data: b"ack".to_vec(),
+                                        },
+                                    ),
+                                    channel,
+                                })
+                                .await
+                                .unwrap();
+                        },
+                        Some(ChunkState::Saved(_)) => {},
+                        None => {},
+                    };
+                    pendind_chunks
+                        .insert(chunk_id.clone(), ChunkState::Saved(chunk_id));
+                },
                 Event::IncomeConnection {
                     peer_id,
                     connection_id,
@@ -219,17 +461,148 @@ impl Client {
                     let mut peers = peers_clone.lock().await;
                     peers.insert(peer_id);
                     info!("inbound connected to peerID {peer_id}, connectionID {connection_id}");
-                }
+                },
                 Event::ConnectionClosed {
                     peer_id,
                     connection_id,
                 } => {
-                    let mut peers = peers_clone.lock().await;
-                    peers.remove(&peer_id);
                     info!("connection closed peerID {peer_id} connectionID {connection_id}");
-                }
-                _ => {}
+                },
+                _ => {},
             }
         }
     }
 }
+
+async fn get_chunk_inner(
+    local_peer_id: PeerId,
+    action_sender: mpsc::Sender<Action>,
+    peer_id: PeerId,
+    chunk_id: CID,
+    done_sender: oneshot::Sender<Result<(), P2PNetworkError>>,
+) {
+    let mut action_sender_dup = action_sender.clone();
+    let (sender, receiver) = oneshot::channel();
+    let data = match serde_json::to_vec(&GetChunkReq {
+        chunk_id: chunk_id.clone(),
+        peer_id: local_peer_id.to_bytes(),
+    }) {
+        Ok(data) => data,
+        Err(e) => {
+            done_sender.send(Err(e.into())).unwrap();
+            return;
+        },
+    };
+    let msg = PeerRequest(PeerRequestMessage {
+        id: random_req_id(),
+        command: GET_CHUNK_CMD.to_string(),
+        data,
+    });
+    action_sender_dup
+        .send(Action::SendRequest {
+            peer_id: peer_id.to_owned(),
+            msg: msg.clone(),
+            sender,
+        })
+        .await
+        .unwrap();
+    match tokio::time::timeout(CHUNK_SEND_TIMEOUT_DURATION, receiver).await {
+        Ok(received) => {
+            let _ = match received.unwrap() {
+                Ok(_) => done_sender.send(Ok(())),
+                Err(e) => done_sender.send(Err(e)),
+            };
+        },
+        Err(_) => {
+            done_sender
+                .send(Err(P2PNetworkError::ChunkTimeout))
+                .unwrap();
+        },
+    };
+}
+
+async fn send_chunk_inner(
+    control: Control,
+    action_sender: mpsc::Sender<Action>,
+    peer_id: PeerId,
+    chunk: Vec<u8>,
+    done_sender: oneshot::Sender<Result<(), P2PNetworkError>>,
+) {
+    let mut control = control.clone();
+    let mut action_sender_dup = action_sender.clone();
+    let (sender, receiver) = oneshot::channel();
+
+    let cid = cid(&chunk);
+    let data = serde_json::to_vec(&PushChunkReq {
+        chunk_id: cid.clone(),
+    })
+    .unwrap();
+    let msg = PeerRequest(PeerRequestMessage {
+        id: random_req_id(),
+        command: PUSH_CHUNK_CMD.to_string(),
+        data,
+    });
+
+    action_sender_dup
+        .send(Action::SendRequest {
+            peer_id: peer_id.to_owned(),
+            msg: msg.clone(),
+            sender,
+        })
+        .await
+        .unwrap();
+
+    let mut stream = match control
+        .open_stream(peer_id.to_owned(), Network::stream_protocol())
+        .await
+        .map_err(|error| {
+            error!("{error}");
+            P2PNetworkError::OpenStreamError(format!("{peer_id},{error:?}"))
+        }) {
+        Ok(stream) => stream,
+        Err(e) => {
+            done_sender.send(Err(e)).unwrap();
+            return;
+        },
+    };
+    if let Err(e) = stream.write_all(&chunk).await {
+        error!("write to stream error {}", e);
+    };
+    stream.close().await.unwrap();
+    match tokio::time::timeout(CHUNK_SEND_TIMEOUT_DURATION, receiver).await {
+        Ok(received) => {
+            let _ = match received.unwrap() {
+                Ok(_) => done_sender.send(Ok(())),
+                Err(e) => done_sender.send(Err(e)),
+            };
+        },
+        Err(_) => {
+            done_sender
+                .send(Err(P2PNetworkError::ChunkTimeout))
+                .unwrap();
+        },
+    };
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct GetChunkReq {
+    chunk_id: CID,
+    peer_id: Vec<u8>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+struct PushChunkReq {
+    chunk_id: CID,
+}
+
+impl Debug for Client {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Client")
+            .field("local_multi_address", &self.local_multi_address)
+            .field("action_sender", &self.action_sender)
+            .field("event_receiver", &self.event_receiver)
+            .field("connected_peers", &self.connected_peers)
+            .field("pending_chunks", &self.pending_chunks)
+            .finish()
+    }
+}
diff --git a/beta/src/p2p/error.rs b/beta/src/p2p/error.rs
index bef8041..eaf239d 100644
--- a/beta/src/p2p/error.rs
+++ b/beta/src/p2p/error.rs
@@ -1,7 +1,8 @@
 use std::io;
 
 use libp2p::{
-    kad::GetClosestPeersError, request_response::OutboundFailure, swarm::DialError, Multiaddr,
+    kad::GetClosestPeersError, request_response::OutboundFailure,
+    swarm::DialError, Multiaddr,
 };
 
 use tempfile::PersistError;
@@ -27,4 +28,10 @@ pub enum P2PNetworkError {
     PersisError(#[from] PersistError),
     #[error("open stream error")]
     OpenStreamError(String),
+    #[error("serde json error")]
+    SerializeError(#[from] serde_json::Error),
+    #[error("chunk transfer timeout")]
+    ChunkTimeout,
+    #[error("other error")]
+    Other(String),
 }
diff --git a/beta/src/p2p/utils.rs b/beta/src/p2p/utils.rs
index c955683..27ae856 100644
--- a/beta/src/p2p/utils.rs
+++ b/beta/src/p2p/utils.rs
@@ -8,7 +8,7 @@ use futures::{
     SinkExt,
 };
 use libp2p::{multiaddr::Protocol, Multiaddr};
-use log::debug;
+
 use rand::random;
 
 use crate::{p2p::behaviour::PeerRequest, reqres_proto::PeerRequestMessage};
@@ -31,7 +31,6 @@ pub async fn bootstrap_peer(
         };
         action_sender.send(action).await.unwrap();
         receiver.await.unwrap().unwrap();
-        debug!("dial {peer_id} done");
 
         // Bootstrap
         action_sender.send(Action::Bootstrap {}).await.unwrap();
@@ -58,9 +57,28 @@ pub async fn bootstrap_peer(
     }
 }
 
+/// Generate random bytes and hex it for request id. Response id is the same as it's request.
 pub fn random_req_id() -> String {
     let mut rng = OsRng;
     let mut buf = [0u8; 32];
     rng.fill_bytes(&mut buf);
     hex::encode(buf)
 }
+
+pub struct Defer<F: FnOnce()> {
+    cleanup: Option<F>,
+}
+
+impl<F: FnOnce()> Defer<F> {
+    pub fn new(f: F) -> Defer<F> {
+        Defer { cleanup: Some(f) }
+    }
+}
+
+impl<F: FnOnce()> Drop for Defer<F> {
+    fn drop(&mut self) {
+        if let Some(cleanup) = self.cleanup.take() {
+            cleanup();
+        }
+    }
+}
diff --git a/beta/src/peer.rs b/beta/src/peer.rs
new file mode 100644
index 0000000..063ba42
--- /dev/null
+++ b/beta/src/peer.rs
@@ -0,0 +1,333 @@
+use std::{
+    collections::{hash_map::Entry, HashMap},
+    io::Read,
+    sync::Arc,
+};
+
+use actix_web::web::{self};
+use libp2p::PeerId;
+use log::{debug, error, info};
+use tokio::{sync::Semaphore, task::JoinSet};
+
+use crate::{
+    cid,
+    codec::wirehair_codec::{WirehairCodec, WirehairCodecOptions},
+    p2p::{client::Client, error::P2PNetworkError},
+    CID,
+};
+
+use {
+    actix_multipart::form::{bytes::Bytes, MultipartForm, MultipartFormConfig},
+    actix_web::{
+        get,
+        middleware::Logger,
+        web::{Data, JsonConfig},
+        App, HttpResponse, HttpServer, Responder,
+    },
+};
+use {
+    serde::{Deserialize, Serialize},
+    tokio::sync::Mutex,
+};
+
+/// Default max outbound streams
+const DEFAULT_MAX_OUTBOUND_STREAM: usize = 30;
+
+/// Default max inbound streams
+const DEFAULT_MAX_INBOUND_STREAM: usize = 10;
+
+/// Save fragment save in FRAGMENT_REPLIC_CNT peers
+const FRAGMENT_REPLIC_CNT: usize = 1;
+
+#[derive(Debug, Clone)]
+struct ContentMeta {
+    pub chunk_meta: HashMap<u32, HashMap<u32, (CID, Vec<PeerId>)>>,
+    pub size: usize,
+}
+
+#[derive(Default)]
+pub struct FullNodeService {}
+
+#[derive(Debug)]
+struct PeerState {
+    pub content_log: Mutex<HashMap<CID, ContentMeta>>,
+    pub codec: WirehairCodec,
+    pub p2p_client: Client,
+    outbound_stream_sema: Arc<Semaphore>, // outbound stream restriction
+    inbound_stream_sema: Arc<Semaphore>,  // inbound stream restriction
+}
+
+impl PeerState {
+    pub fn new(peer_opt: PeerOpt, p2p_client: Client) -> Self {
+        let codec = match peer_opt.codec_opt {
+            Some(option) => WirehairCodec::new_with_options(option),
+            None => WirehairCodec::new(),
+        };
+        PeerState {
+            content_log: Mutex::new(HashMap::new()),
+            codec,
+            p2p_client,
+            outbound_stream_sema: Arc::new(Semaphore::new(
+                peer_opt
+                    .max_oubound_stream
+                    .unwrap_or(DEFAULT_MAX_OUTBOUND_STREAM),
+            )),
+            inbound_stream_sema: Arc::new(Semaphore::new(
+                peer_opt
+                    .max_inbound_stream
+                    .unwrap_or(DEFAULT_MAX_INBOUND_STREAM),
+            )),
+        }
+    }
+}
+
+#[get("/healthcheck")]
+async fn healthcheck() -> impl Responder {
+    HttpResponse::Ok().body("ok".to_string())
+}
+
+#[derive(Debug, MultipartForm)]
+struct PutForm {
+    #[multipart(rename = "file")]
+    files: Vec<Bytes>,
+}
+
+#[actix_web::post("/put")]
+async fn put(
+    data: Data<PeerState>,
+    MultipartForm(form): MultipartForm<PutForm>,
+) -> impl Responder {
+    let msg = form.files[0].data.as_ref().to_vec();
+    let object_size = msg.len();
+    let msg_cid = cid(&msg);
+
+    if data.content_log.lock().await.contains_key(&msg_cid.clone()) {
+        return HttpResponse::Ok().body(msg_cid.0);
+    }
+
+    info!(
+        "put content size {}, cid {}",
+        object_size,
+        msg_cid.0.clone()
+    );
+    let chunks = data.codec.encode(msg);
+    let mut content_meta = HashMap::new();
+    let mut set = JoinSet::new();
+    for (chunk_id, chunk) in chunks.into_iter() {
+        let mut chunk_meta = HashMap::new();
+        for (fragment_id, fragment) in chunk.into_iter() {
+            let fragment_cid = cid(&fragment);
+            // todo: consider full node save local copy
+            // let mut file = fs::File::create(fragment_cid.clone().0).unwrap();
+            // file.write_all(&fragment).unwrap();
+            // file.flush().unwrap();
+
+            // send fragment to random selected closest peer
+            let closest = data
+                .p2p_client
+                .random_closest_peer(
+                    fragment_cid.clone().into(),
+                    FRAGMENT_REPLIC_CNT,
+                )
+                .await
+                .unwrap();
+            for peer in closest.clone().into_iter() {
+                let client = data.p2p_client.clone();
+                let fragment_clone = fragment.clone();
+                let sema = data.outbound_stream_sema.clone();
+                set.spawn(async move {
+                    let permit = sema.acquire().await.unwrap();
+                    let _fragment_cid = cid(&fragment_clone);
+                    let res = client.send_chunk(peer, fragment_clone).await;
+                    drop(permit);
+                    res
+                });
+            }
+
+            chunk_meta.insert(fragment_id.to_owned(), (fragment_cid, closest));
+        }
+        content_meta.insert(chunk_id.to_owned(), chunk_meta);
+    }
+
+    while let Some(res) = set.join_next().await {
+        if let Err(e) = res.unwrap() {
+            error!("get fragment error {e:?}");
+        };
+    }
+
+    data.content_log.lock().await.insert(
+        msg_cid.clone(),
+        ContentMeta {
+            chunk_meta: content_meta,
+            size: object_size,
+        },
+    );
+    HttpResponse::Ok().body(msg_cid.0)
+}
+
+#[actix_web::get("/get/{chunk_id}")]
+async fn get(
+    data: Data<PeerState>,
+    chunk_id: web::Path<String>,
+) -> impl Responder {
+    let cid = CID(chunk_id.into_inner());
+    info!("get content cid {}", cid.0.clone());
+    let content_log = data.content_log.lock().await;
+    let content_meta = content_log.get(&cid);
+    if content_meta.is_none() {
+        return HttpResponse::BadRequest()
+            .body(format!("content not exist cid {}", cid.0));
+    }
+    let content_meta = content_meta.unwrap().to_owned();
+
+    let object_size = content_meta.size as usize;
+    let mut links: HashMap<u32, HashMap<u32, Vec<u8>>> = HashMap::new();
+    let mut set = JoinSet::new();
+    for (chunk_id, chunk_meta) in content_meta.chunk_meta.into_iter() {
+        for (fragment_id, (fragment_cid, light_peers)) in chunk_meta.into_iter()
+        {
+            let client = data.p2p_client.clone();
+            let sema = data.inbound_stream_sema.clone();
+            set.spawn(async move {
+                if read_local_content(fragment_cid.clone()).is_ok() {
+                    return Ok((chunk_id, fragment_id, fragment_cid.clone()));
+                };
+                let permit = sema.acquire().await.unwrap();
+                for light_peer in light_peers.into_iter() {
+                    match client
+                        .get_chunk(light_peer, fragment_cid.clone())
+                        .await
+                    {
+                        Ok(()) => {
+                            drop(permit);
+                            return Ok((
+                                chunk_id,
+                                fragment_id,
+                                fragment_cid.clone(),
+                            ));
+                        },
+                        Err(e) => {
+                            error!(
+                                "get chunk {} from peer {} error {}",
+                                fragment_cid.0.clone(),
+                                light_peer.clone(),
+                                e
+                            );
+                            continue;
+                        },
+                    };
+                }
+                drop(permit);
+                Err((chunk_id, fragment_id, fragment_cid.clone()))
+            });
+        }
+    }
+
+    while let Some(res) = set.join_next().await {
+        let res = res.unwrap();
+        let (chunk_id, fragment_id, fragment_cid) = res.unwrap_or_else(|x| x);
+        let buf = match read_local_content(fragment_cid.clone()) {
+            Ok(buf) => buf,
+            Err(e) => {
+                debug!("read local fragment content error {}", e);
+                continue;
+            },
+        };
+        match links.entry(chunk_id) {
+            Entry::Occupied(o) => {
+                let o = o.into_mut();
+                o.insert(fragment_id, buf);
+            },
+            Entry::Vacant(v) => {
+                let mut chunk_links: HashMap<u32, Vec<u8>> = HashMap::new();
+                chunk_links.insert(fragment_id, buf);
+                v.insert(chunk_links);
+            },
+        }
+    }
+
+    let object = data.codec.decode(links, object_size);
+    HttpResponse::Ok()
+        .content_type("text/markdown")
+        .body(object)
+}
+
+#[derive(Serialize, Deserialize)]
+struct Fragment {
+    id: u32,
+    #[serde(with = "serde_bytes")]
+    data: Vec<u8>,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct PeerOpt {
+    pub codec_opt: Option<WirehairCodecOptions>,
+    pub max_inbound_stream: Option<usize>,
+    pub max_oubound_stream: Option<usize>,
+}
+
+impl Default for PeerOpt {
+    fn default() -> Self {
+        Self {
+            codec_opt: Some(WirehairCodecOptions::default()),
+            max_inbound_stream: Some(DEFAULT_MAX_INBOUND_STREAM),
+            max_oubound_stream: Some(DEFAULT_MAX_OUTBOUND_STREAM),
+        }
+    }
+}
+
+impl FullNodeService {
+    pub async fn start(
+        &mut self,
+        host: String,
+        port: u16,
+        peer_opt: PeerOpt,
+        client: Client,
+    ) {
+        let addr = (host, port);
+        let peer_state = PeerState::new(peer_opt, client);
+
+        let data = Data::new(peer_state);
+        let _ = HttpServer::new(move || {
+            App::new()
+                .wrap(Logger::default())
+                .app_data(
+                    MultipartFormConfig::default()
+                        .total_limit(1 << 30)
+                        .memory_limit(1 << 30),
+                )
+                .app_data(JsonConfig::default().limit(1 << 30))
+                .app_data(data.clone())
+                .service(healthcheck)
+                .service(put)
+                .service(get)
+        })
+        .bind(addr)
+        .expect("failed to bind address")
+        .run()
+        .await;
+    }
+}
+
+fn read_local_content(fragment_cid: CID) -> Result<Vec<u8>, P2PNetworkError> {
+    let mut file = match std::fs::File::open(fragment_cid.0.clone()) {
+        Ok(file) => file,
+        Err(e) => {
+            debug!("open file {} error {:?}", e, fragment_cid.0.clone());
+            return Err(e.into());
+        },
+    };
+    let mut buf = Vec::new();
+    if let Err(e) = file.read_to_end(&mut buf) {
+        debug!("read file {} error {:?}", fragment_cid.0.clone(), e);
+        return Err(e.into());
+    }
+    if super::cid(&buf).0 != fragment_cid.0.clone() {
+        error!("fragment content inconsistent {}", fragment_cid.0.clone());
+        return Err(P2PNetworkError::Other(format!(
+            "fragment content inconsistent {}",
+            fragment_cid.0.clone()
+        )));
+    }
+    Ok(buf)
+}
diff --git a/beta/src/server.rs b/beta/src/server.rs
index 09d023d..6905561 100644
--- a/beta/src/server.rs
+++ b/beta/src/server.rs
@@ -45,6 +45,12 @@ pub struct Server {
     sessions: HashMap<usize, String>,
 }
 
+impl Default for Server {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl Server {
     pub fn new() -> Server {
         Server {
@@ -125,13 +131,14 @@ impl Handler<Put> for Server {
             }
             chunks.insert(chunk_id, fragments);
         }
-        ()
     }
 }
 
 #[derive(Default)]
 pub struct Peer {}
 
+type ChunkInfoMap = Mutex<HashMap<u64, (u64, u64, Vec<ChunkInfo>)>>;
+
 #[derive(Debug, Default)]
 struct PeerState {
     pub peers: Arc<Mutex<Vec<(String, u16)>>>,
@@ -139,7 +146,7 @@ struct PeerState {
     pub chunk_config: ChunkConfig,
 
     pub fragments: Mutex<HashMap<u32, Vec<u8>>>,
-    pub chunk_infos: Mutex<HashMap<u64, (u64, u64, Vec<ChunkInfo>)>>,
+    pub chunk_infos: ChunkInfoMap,
 }
 
 #[derive(Debug, Clone, Default)]
@@ -196,7 +203,7 @@ async fn put(
 
     let mut sum = 0u64;
     for n in msg.iter() {
-        sum = sum + *n as u64;
+        sum += *n as u64;
     }
 
     let encoder_start = std::time::Instant::now();

From 861ea5cf3ad2058be6048422bc3af1af85e590e2 Mon Sep 17 00:00:00 2001
From: carlhou369 <carlhou369@gmail.com>
Date: Thu, 11 Apr 2024 21:05:53 +0800
Subject: [PATCH 11/11] fix full node can't graceful close

---
 beta/src/bin/peer.rs | 4 ++--
 beta/src/peer.rs     | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/beta/src/bin/peer.rs b/beta/src/bin/peer.rs
index c83e4da..05db327 100644
--- a/beta/src/bin/peer.rs
+++ b/beta/src/bin/peer.rs
@@ -110,6 +110,8 @@ async fn main() -> Result<(), Box<dyn Error>> {
             } else {
                 panic!("peer addr format error");
             }
+            let p = future::pending();
+            let () = p.await;
         },
         None => {
             // Start full node demo
@@ -127,7 +129,5 @@ async fn main() -> Result<(), Box<dyn Error>> {
             .await;
         },
     }
-    let p = future::pending();
-    let () = p.await;
     Ok(())
 }
diff --git a/beta/src/peer.rs b/beta/src/peer.rs
index 063ba42..878a82d 100644
--- a/beta/src/peer.rs
+++ b/beta/src/peer.rs
@@ -207,7 +207,7 @@ async fn get(
                             ));
                         },
                         Err(e) => {
-                            error!(
+                            debug!(
                                 "get chunk {} from peer {} error {}",
                                 fragment_cid.0.clone(),
                                 light_peer.clone(),