From a112bf2884514226941402edfe856e110dfc1f85 Mon Sep 17 00:00:00 2001 From: ismellike Date: Tue, 10 Feb 2026 14:07:51 -0600 Subject: [PATCH 01/10] try fix hypercore --- packages/layer-tests/src/e2e.rs | 8 +- packages/layer-tests/src/e2e/handles.rs | 74 +++++---- .../layer-tests/src/e2e/handles/hypercore.rs | 1 + packages/layer-tests/src/e2e/matrix.rs | 10 +- .../layer-tests/src/e2e/test_definition.rs | 2 +- packages/layer-tests/src/e2e/test_registry.rs | 143 +++++++++--------- packages/wavs/src/subsystems/trigger.rs | 20 ++- .../trigger/streams/hypercore_protocol.rs | 17 ++- .../trigger/streams/hypercore_stream.rs | 24 ++- 9 files changed, 163 insertions(+), 136 deletions(-) diff --git a/packages/layer-tests/src/e2e.rs b/packages/layer-tests/src/e2e.rs index 0d0c0a1a1..dbd01e9b3 100644 --- a/packages/layer-tests/src/e2e.rs +++ b/packages/layer-tests/src/e2e.rs @@ -150,10 +150,10 @@ async fn _run( configs.chains.clone(), &clients, &cosmos_code_map, - // configs - // .wavs_configs - // .first() - // .and_then(|config| config.hyperswarm_bootstrap.clone()), + configs + .wavs_configs + .first() + .and_then(|config| config.hyperswarm_bootstrap.clone()), ) .await; diff --git a/packages/layer-tests/src/e2e/handles.rs b/packages/layer-tests/src/e2e/handles.rs index a321fe201..5cb468c6f 100644 --- a/packages/layer-tests/src/e2e/handles.rs +++ b/packages/layer-tests/src/e2e/handles.rs @@ -2,7 +2,7 @@ mod cosmos; mod evm; pub mod hypercore; -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration}; use cosmos::CosmosInstance; use evm::EvmInstance; @@ -22,9 +22,9 @@ use wavs_types::{ChainKey, ChainKeyNamespace}; use crate::config::TestP2pMode; /// Default port for the hyperswarm bootstrap node -//const HYPERSWARM_BOOTSTRAP_PORT: u16 = 49737; +const HYPERSWARM_BOOTSTRAP_PORT: u16 = 49737; use super::config::Configs; -//use super::matrix::EvmService; +use super::matrix::EvmService; pub struct AppHandles { /// One handle per WAVS operator instance @@ -44,15 +44,11 @@ impl AppHandles { Option, Option>>, ) = { - // #[cfg(feature = "hypercore-tests")] - // { - // if configs.matrix.evm.contains(&EvmService::HypercoreEchoData) { - // Self::start_hyperswarm_bootstrap() - // } else { - // (None, None) - // } - // } - (None, None) + if configs.matrix.evm.contains(&EvmService::HypercoreEchoData) { + Self::start_hyperswarm_bootstrap() + } else { + (None, None) + } }; if let Some(addr) = bootstrap_addr { let addr = addr.to_string(); @@ -271,31 +267,31 @@ impl AppHandles { Ok(handles) } - // fn start_hyperswarm_bootstrap() -> ( - // Option, - // Option>>, - // ) { - // let bind_addr = SocketAddr::new( - // std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)), - // HYPERSWARM_BOOTSTRAP_PORT, - // ); - - // match async_std::task::block_on(hyperswarm::run_bootstrap_node(Some(bind_addr))) { - // Ok((addr, handle)) => { - // tracing::info!( - // "Bootstrap node bound to {}, listening for peer connections", - // addr - // ); - - // // Give the bootstrap node time to bind and initialize its DHT - // std::thread::sleep(Duration::from_secs(5)); - - // (Some(addr), Some(handle)) - // } - // Err(err) => { - // tracing::warn!("Failed to start hyperswarm bootstrap node: {err}"); - // (None, None) - // } - // } - // } + fn start_hyperswarm_bootstrap() -> ( + Option, + Option>>, + ) { + let bind_addr = SocketAddr::new( + std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)), + HYPERSWARM_BOOTSTRAP_PORT, + ); + + match async_std::task::block_on(hyperswarm::run_bootstrap_node(Some(bind_addr))) { + Ok((addr, handle)) => { + tracing::info!( + "Bootstrap node bound to {}, listening for peer connections", + addr + ); + + // Give the bootstrap node time to bind and initialize its DHT + std::thread::sleep(Duration::from_secs(5)); + + (Some(addr), Some(handle)) + } + Err(err) => { + tracing::warn!("Failed to start hyperswarm bootstrap node: {err}"); + (None, None) + } + } + } } diff --git a/packages/layer-tests/src/e2e/handles/hypercore.rs b/packages/layer-tests/src/e2e/handles/hypercore.rs index 446aea159..20ae048d0 100644 --- a/packages/layer-tests/src/e2e/handles/hypercore.rs +++ b/packages/layer-tests/src/e2e/handles/hypercore.rs @@ -159,6 +159,7 @@ impl HypercoreTestClient { is_initiator, feed, feed_key_bytes, + None, ) .await; diff --git a/packages/layer-tests/src/e2e/matrix.rs b/packages/layer-tests/src/e2e/matrix.rs index 79682a0fd..19f50cccd 100644 --- a/packages/layer-tests/src/e2e/matrix.rs +++ b/packages/layer-tests/src/e2e/matrix.rs @@ -21,8 +21,7 @@ pub enum EvmService { CosmosQuery, EchoData, AtprotoEchoData, - // #[cfg(feature = "hypercore-tests")] - // HypercoreEchoData, + HypercoreEchoData, ChangeWorkflow, EchoDataSecondaryChain, KvStore, @@ -150,10 +149,9 @@ impl From for Vec { EvmService::AtprotoEchoData => { vec![ComponentName::Operator(OperatorComponent::EchoData)] } - // #[cfg(feature = "hypercore-tests")] - // EvmService::HypercoreEchoData => { - // vec![ComponentName::Operator(OperatorComponent::EchoData)] - // } + EvmService::HypercoreEchoData => { + vec![ComponentName::Operator(OperatorComponent::EchoData)] + } EvmService::ChangeWorkflow => vec![ ComponentName::Operator(OperatorComponent::Square), ComponentName::Operator(OperatorComponent::EchoData), diff --git a/packages/layer-tests/src/e2e/test_definition.rs b/packages/layer-tests/src/e2e/test_definition.rs index 54fb9679b..2f1a3537d 100644 --- a/packages/layer-tests/src/e2e/test_definition.rs +++ b/packages/layer-tests/src/e2e/test_definition.rs @@ -17,7 +17,7 @@ use crate::e2e::components::{ #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, PartialOrd, Ord)] pub enum TestGroupId { Default, - //Hypercore, + Hypercore, EvmInterval, EvmIntervalStartStop, CronInterval, diff --git a/packages/layer-tests/src/e2e/test_registry.rs b/packages/layer-tests/src/e2e/test_registry.rs index 1edaccfbd..3169cf99f 100644 --- a/packages/layer-tests/src/e2e/test_registry.rs +++ b/packages/layer-tests/src/e2e/test_registry.rs @@ -6,6 +6,7 @@ use example_types::{ use serde_json::json; use std::collections::BTreeMap; use std::sync::{Arc, RwLock}; +use std::time::Duration; use wavs_types::AtProtoAction; use super::clients::Clients; @@ -106,14 +107,14 @@ impl TestRegistry { self.hypercore_clients.get(test_name).map(|v| v.clone()) } - // Store a hypercore client factory for a test (client will be created later) - // pub fn insert_hypercore_client_factory( - // &self, - // test_name: String, - // factory: HypercoreClientFactory, - // ) { - // self.hypercore_client_factories.insert(test_name, factory); - // } + /// Store a hypercore client factory for a test (client will be created later) + pub fn insert_hypercore_client_factory( + &self, + test_name: String, + factory: HypercoreClientFactory, + ) { + self.hypercore_client_factories.insert(test_name, factory); + } /// Create hypercore clients from factories for all tests that need them /// Called right before tests run to ensure DHT announcements are fresh @@ -145,7 +146,7 @@ impl TestRegistry { chain_configs: Arc>, clients: &Clients, cosmos_code_map: &CosmosCodeMap, - //hyperswarm_bootstrap: Option, + hyperswarm_bootstrap: Option, ) -> Self { // Convert TestMode to TestMatrix let matrix: TestMatrix = test_mode.into(); @@ -166,12 +167,11 @@ impl TestRegistry { EvmService::AtprotoEchoData => { registry.register_evm_atproto_echo_data_test(chain); } - // #[cfg(feature = "hypercore-tests")] - // EvmService::HypercoreEchoData => { - // registry - // .register_evm_hypercore_echo_data_test(chain, hyperswarm_bootstrap.clone()) - // .await; - // } + EvmService::HypercoreEchoData => { + registry + .register_evm_hypercore_echo_data_test(chain, hyperswarm_bootstrap.clone()) + .await; + } EvmService::EchoDataSecondaryChain => { let secondary = chains.secondary_evm().unwrap(); registry.register_evm_echo_data_secondary_chain_test(secondary); @@ -356,63 +356,62 @@ impl TestRegistry { ) } - // #[cfg(feature = "hypercore-tests")] - // async fn register_evm_hypercore_echo_data_test( - // &mut self, - // chain: &ChainKey, - // hyperswarm_bootstrap: Option, - // ) -> &mut Self { - // // Generate signing key now to get feed_key for the trigger, - // // but delay creating the full client until right before test runs - // let test_name = "evm_hypercore_echo_data"; - // use hypercore::SigningKey; - // use rand_core::OsRng; - - // let signing_key = SigningKey::generate(&mut OsRng); - // let public_key_bytes = signing_key.verifying_key().to_bytes(); - // let feed_key = const_hex::encode(public_key_bytes); - // let signing_key_bytes = signing_key.to_bytes(); - - // tracing::info!( - // "Generated signing key for hypercore test '{}' with feed key: {}", - // test_name, - // feed_key - // ); - - // // Store the factory to create the client later - // self.insert_hypercore_client_factory( - // test_name.to_string(), - // HypercoreClientFactory { - // test_name: test_name.to_string(), - // hyperswarm_bootstrap, - // signing_key_bytes: signing_key_bytes.to_vec(), - // }, - // ); - - // self.register( - // TestBuilder::new(test_name) - // .with_description( - // "Tests the EchoData component with real Hypercore append triggers", - // ) - // .add_workflow( - // WorkflowId::new("hypercore_echo_data").unwrap(), - // WorkflowBuilder::new() - // .with_operator_component(OperatorComponent::EchoData) - // .with_aggregator_component(AggregatorComponent::SimpleAggregator) - // .with_trigger(TriggerDefinition::Existing(Trigger::HypercoreAppend { - // feed_key, - // })) - // .with_submit(SubmitDefinition::Aggregator(Self::simple_aggregator(chain))) - // .with_input_data(InputData::Text("hypercore-echo".to_string())) - // .with_expected_output(ExpectedOutput::Text("hypercore-echo".to_string())) - // .with_timeout(Duration::from_secs(60)) - // .build(), - // ) - // .with_service_manager_chain(chain) - // .with_group(TestGroupId::Hypercore) - // .build(), - // ) - // } + async fn register_evm_hypercore_echo_data_test( + &mut self, + chain: &ChainKey, + hyperswarm_bootstrap: Option, + ) -> &mut Self { + // Generate signing key now to get feed_key for the trigger, + // but delay creating the full client until right before test runs + let test_name = "evm_hypercore_echo_data"; + use hypercore::SigningKey; + use rand_core::OsRng; + + let signing_key = SigningKey::generate(&mut OsRng); + let public_key_bytes = signing_key.verifying_key().to_bytes(); + let feed_key = const_hex::encode(public_key_bytes); + let signing_key_bytes = signing_key.to_bytes(); + + tracing::info!( + "Generated signing key for hypercore test '{}' with feed key: {}", + test_name, + feed_key + ); + + // Store the factory to create the client later + self.insert_hypercore_client_factory( + test_name.to_string(), + HypercoreClientFactory { + test_name: test_name.to_string(), + hyperswarm_bootstrap, + signing_key_bytes: signing_key_bytes.to_vec(), + }, + ); + + self.register( + TestBuilder::new(test_name) + .with_description( + "Tests the EchoData component with real Hypercore append triggers", + ) + .add_workflow( + WorkflowId::new("hypercore_echo_data").unwrap(), + WorkflowBuilder::new() + .with_operator_component(OperatorComponent::EchoData) + .with_aggregator_component(AggregatorComponent::SimpleAggregator) + .with_trigger(TriggerDefinition::Existing(Trigger::HypercoreAppend { + feed_key, + })) + .with_submit(SubmitDefinition::Aggregator(Self::simple_aggregator(chain))) + .with_input_data(InputData::Text("hypercore-echo".to_string())) + .with_expected_output(ExpectedOutput::Text("hypercore-echo".to_string())) + .with_timeout(Duration::from_secs(60)) + .build(), + ) + .with_service_manager_chain(chain) + .with_group(TestGroupId::Hypercore) + .build(), + ) + } fn register_evm_echo_data_secondary_chain_test( &mut self, diff --git a/packages/wavs/src/subsystems/trigger.rs b/packages/wavs/src/subsystems/trigger.rs index 91c7c8b28..2c13e2294 100644 --- a/packages/wavs/src/subsystems/trigger.rs +++ b/packages/wavs/src/subsystems/trigger.rs @@ -721,14 +721,22 @@ impl TriggerManager { ) .await; match hypercore_start_result { - Ok(hypercore_stream) => { + Ok((hypercore_stream, peer_connected_rx)) => { multiplexed_stream.push(hypercore_stream); - // Mark as connected once stream starts successfully - hypercore_stream_states - .write() - .unwrap() - .insert(feed_key.clone(), StreamStartState::Connected); + // State stays as Connecting until a peer actually connects. + // Spawn a task that updates the state when the first peer + // connects via hyperswarm. + let states = Arc::clone(&hypercore_stream_states); + let fk = feed_key.clone(); + tokio::spawn(async move { + if peer_connected_rx.await.is_ok() { + states + .write() + .unwrap() + .insert(fk, StreamStartState::Connected); + } + }); } Err(err) => { tracing::error!("Failed to start hypercore stream: {:?}", err); diff --git a/packages/wavs/src/subsystems/trigger/streams/hypercore_protocol.rs b/packages/wavs/src/subsystems/trigger/streams/hypercore_protocol.rs index b09c26062..a7bc25dfd 100644 --- a/packages/wavs/src/subsystems/trigger/streams/hypercore_protocol.rs +++ b/packages/wavs/src/subsystems/trigger/streams/hypercore_protocol.rs @@ -46,6 +46,7 @@ pub async fn run_protocol( is_initiator: bool, hypercore: Arc>, feed_key: [u8; 32], + replication_ready: Option>, ) -> Result<()> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, @@ -58,6 +59,8 @@ where const_hex::encode(dkey) ); + let mut replication_ready = replication_ready; + while let Some(event) = protocol.next().await { let event = event.context("hypercore protocol event")?; match event { @@ -77,7 +80,7 @@ where ProtocolEvent::Channel(channel) => { if channel.discovery_key() == &dkey { tracing::info!("Hypercore protocol channel opened"); - spawn_peer(channel, hypercore.clone()); + spawn_peer(channel, hypercore.clone(), replication_ready.take()); } } _ => {} @@ -87,7 +90,11 @@ where Ok(()) } -fn spawn_peer(mut channel: Channel, hypercore: Arc>) { +fn spawn_peer( + mut channel: Channel, + hypercore: Arc>, + replication_ready: Option>, +) { tokio::spawn(async move { let mut peer_state = PeerState::default(); let mut receiver = { @@ -136,6 +143,12 @@ fn spawn_peer(mut channel: Channel, hypercore: Arc>) { info.contiguous_length ); + // Signal that replication is ready — the channel is open and + // the initial Synchronize has been sent to the remote peer. + if let Some(tx) = replication_ready { + let _ = tx.send(()); + } + loop { tokio::select! { message = channel.next() => { diff --git a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs index 86ff1a005..7a1012794 100644 --- a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs +++ b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs @@ -35,8 +35,13 @@ pub async fn start_hypercore_stream( config: HypercoreStreamConfig, metrics: TriggerMetrics, shutdown: tokio::sync::broadcast::Receiver<()>, -) -> Result> + Send>>, TriggerError> -{ +) -> Result< + ( + Pin> + Send>>, + tokio::sync::oneshot::Receiver<()>, + ), + TriggerError, +> { std::fs::create_dir_all(&config.storage_dir).map_err(|err| { TriggerError::Hypercore(format!( "create storage dir {}: {}", @@ -119,7 +124,7 @@ pub async fn start_hypercore_stream( } }; - start_swarm_replication( + let peer_connected_rx = start_swarm_replication( feed_key_bytes, Arc::clone(&core), shutdown, @@ -127,7 +132,7 @@ pub async fn start_hypercore_stream( ) .await?; - Ok(Box::pin(event_stream)) + Ok((Box::pin(event_stream), peer_connected_rx)) } async fn build_core_with_feed_key( @@ -161,7 +166,7 @@ async fn start_swarm_replication( core: Arc>, mut shutdown: tokio::sync::broadcast::Receiver<()>, hyperswarm_bootstrap: Option, -) -> Result<(), TriggerError> { +) -> Result, TriggerError> { let topic = discovery_key(&feed_key); tracing::info!( @@ -181,10 +186,14 @@ async fn start_swarm_replication( topic ); + let (replication_ready_tx, replication_ready_rx) = tokio::sync::oneshot::channel::<()>(); + // Hyperswarm is async-std based but exposes futures-compatible streams, so it // can be polled directly from the tokio runtime that owns hypercore. tokio::spawn(async move { tracing::info!("Hyperswarm task started, waiting for peer connections..."); + // Only the first peer's replication readiness is signalled. + let mut replication_ready_tx = Some(replication_ready_tx); loop { tokio::select! { @@ -214,6 +223,8 @@ async fn start_swarm_replication( let replication_core = Arc::clone(&core); let is_initiator = stream.is_initiator(); + // Pass the sender only to the first peer's protocol session. + let ready_tx = replication_ready_tx.take(); tokio::spawn(async move { if let Err(err) = hypercore_protocol::run_protocol( @@ -221,6 +232,7 @@ async fn start_swarm_replication( is_initiator, replication_core, feed_key, + ready_tx, ) .await { @@ -232,7 +244,7 @@ async fn start_swarm_replication( } }); - Ok(()) + Ok(replication_ready_rx) } fn build_swarm_config(hyperswarm_bootstrap: Option<&str>) -> SwarmConfig { From f0276f26e356588ca36b73ed398b3c5a2ed68bd2 Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 13 Feb 2026 13:05:55 -0600 Subject: [PATCH 02/10] Improve code structure --- packages/layer-tests/src/e2e.rs | 3 +- packages/layer-tests/src/e2e/handles.rs | 7 +- .../layer-tests/src/e2e/handles/hypercore.rs | 52 +++++++++ packages/layer-tests/src/e2e/runner.rs | 101 ++++++++---------- packages/layer-tests/src/e2e/test_registry.rs | 92 ++++------------ 5 files changed, 120 insertions(+), 135 deletions(-) diff --git a/packages/layer-tests/src/e2e.rs b/packages/layer-tests/src/e2e.rs index dbd01e9b3..e350ef98d 100644 --- a/packages/layer-tests/src/e2e.rs +++ b/packages/layer-tests/src/e2e.rs @@ -145,7 +145,7 @@ async fn _run( let cosmos_code_map = CosmosCodeMap::new(DashMap::new()); // Create test registry from test mode - let registry = test_registry::TestRegistry::from_test_mode( + let (registry, hypercore_clients) = test_registry::TestRegistry::from_test_mode( mode, configs.chains.clone(), &clients, @@ -183,6 +183,7 @@ async fn _run( clients, registry, component_sources, + hypercore_clients, service_managers, cosmos_code_map, report.clone(), diff --git a/packages/layer-tests/src/e2e/handles.rs b/packages/layer-tests/src/e2e/handles.rs index 5cb468c6f..c893f5969 100644 --- a/packages/layer-tests/src/e2e/handles.rs +++ b/packages/layer-tests/src/e2e/handles.rs @@ -21,8 +21,8 @@ use wavs_types::{ChainKey, ChainKeyNamespace}; use crate::config::TestP2pMode; -/// Default port for the hyperswarm bootstrap node -const HYPERSWARM_BOOTSTRAP_PORT: u16 = 49737; +/// Port for the hyperswarm bootstrap node (0 = OS-assigned to avoid conflicts in CI) +const HYPERSWARM_BOOTSTRAP_PORT: u16 = 0; use super::config::Configs; use super::matrix::EvmService; @@ -289,8 +289,7 @@ impl AppHandles { (Some(addr), Some(handle)) } Err(err) => { - tracing::warn!("Failed to start hyperswarm bootstrap node: {err}"); - (None, None) + panic!("Failed to start hyperswarm bootstrap node: {err}. All hypercore tests will fail without a bootstrap node."); } } } diff --git a/packages/layer-tests/src/e2e/handles/hypercore.rs b/packages/layer-tests/src/e2e/handles/hypercore.rs index 20ae048d0..2462ab2ff 100644 --- a/packages/layer-tests/src/e2e/handles/hypercore.rs +++ b/packages/layer-tests/src/e2e/handles/hypercore.rs @@ -6,6 +6,7 @@ use ::hypercore_protocol::discovery_key; use hypercore::{Hypercore, HypercoreBuilder, PartialKeypair, SigningKey, Storage, VerifyingKey}; use hyperswarm::{Config as SwarmConfig, Hyperswarm, TopicConfig}; +use std::collections::HashMap; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::{ @@ -17,6 +18,57 @@ use tokio::sync::Mutex; use tokio::task::JoinHandle; use wavs::subsystems::trigger::streams::hypercore_protocol; +/// Manages hypercore test client lifecycle (deferred creation and retrieval). +/// +/// During test registration, pending entries are inserted with the signing key +/// and bootstrap address. Clients are created lazily right before tests run +/// so DHT announcements stay fresh. +#[derive(Default)] +pub struct HypercoreClients { + /// Pending: (hyperswarm_bootstrap, signing_key_bytes) + pending: HashMap, Vec)>, + clients: HashMap>, +} + +impl HypercoreClients { + pub fn new() -> Self { + Self::default() + } + + /// Queue a test for deferred hypercore client creation. + pub fn insert_pending( + &mut self, + test_name: String, + hyperswarm_bootstrap: Option, + signing_key_bytes: Vec, + ) { + self.pending + .insert(test_name, (hyperswarm_bootstrap, signing_key_bytes)); + } + + /// Create hypercore clients for all pending tests. + /// Called right before tests run to ensure DHT announcements are fresh. + pub async fn create_clients(&mut self) -> anyhow::Result<()> { + for (test_name, (bootstrap, key_bytes)) in &self.pending { + if !self.clients.contains_key(test_name) { + tracing::info!( + "Creating hypercore client for test '{}' right before test execution", + test_name + ); + let client = + HypercoreTestClient::new(test_name, bootstrap.clone(), key_bytes).await?; + self.clients.insert(test_name.clone(), Arc::new(client)); + } + } + Ok(()) + } + + /// Get a hypercore test client by test name. + pub fn get(&self, test_name: &str) -> Option> { + self.clients.get(test_name).cloned() + } +} + /// Test client for creating and managing hypercore feeds in e2e tests. pub struct HypercoreTestClient { /// The hypercore feed diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index c7748db97..c5d2868a6 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -20,6 +20,7 @@ use wavs_types::{ Workflow, WorkflowId, }; +use crate::e2e::handles::hypercore::HypercoreClients; use crate::e2e::helpers::wait_for_hypercore_streams_to_finalize; use crate::e2e::helpers::{ change_service_for_test, cosmos_wait_for_task_to_land, wait_for_hypercore_mesh_ready, @@ -49,6 +50,7 @@ pub struct Runner { clients: Arc, registry: Arc, component_sources: Arc, + hypercore_clients: HypercoreClients, service_managers: ServiceManagers, cosmos_code_map: CosmosCodeMap, report: TestReport, @@ -75,11 +77,13 @@ fn extract_aggregator_service_handler(submit: &Submit) -> Option) { + pub async fn run_tests(&mut self, mut all_services: HashMap) { let test_groups = self.registry.list_all_grouped(self.configs.grouping); for (group, mut group_tests) in test_groups { - // Create hypercore clients BEFORE deploying services - // This ensures the test client announces to DHT before WAVS starts its hypercore streams. - // When WAVS deploys a service with a HypercoreAppend trigger, it immediately starts - // the hyperswarm discovery. If the test client hasn't announced yet, WAVS won't find it. - if let Err(e) = self.registry.create_hypercore_clients().await { - tracing::error!("Failed to create hypercore clients: {}", e); - } - - // Give the hypercore client time to announce to DHT before services start discovering - // In CI with multiple operators, DHT propagation may take longer - if self - .registry - .get_hypercore_client("evm_hypercore_echo_data") - .is_some() - { - tracing::info!( - "Waiting for hypercore client DHT announcement to propagate (10s)..." - ); - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - } - let services = group_tests .iter() .map(|test| all_services.get(&test.name).cloned().unwrap().service) @@ -141,14 +125,17 @@ impl Runner { for test in group_tests.iter() { if let Some(change_service) = test.change_service.clone() { let service = all_services.get(&test.name).cloned().unwrap().service; + let clients = self.clients.clone(); + let component_sources = self.component_sources.clone(); + let cosmos_code_map = self.cosmos_code_map.clone(); futures.push(async move { let mut service = service; change_service_for_test( &mut service, change_service.clone(), - &self.clients, - &self.component_sources, - self.cosmos_code_map.clone(), + &clients, + &component_sources, + cosmos_code_map, ) .await; (service, change_service) @@ -222,11 +209,19 @@ impl Runner { .await; } + // Create hypercore clients AFTER deploying services so WAVS is already + // doing DHT lookups when the test client announces. This avoids the stale-DHT + // problem where the client announces 10+ seconds before WAVS starts looking. + if let Err(e) = self.hypercore_clients.create_clients().await { + tracing::error!("Failed to create hypercore clients: {}", e); + } + // All services are now deployed and ready for the tests // From here on in we're strictly testing the trigger->execute->aggregate->submit flow tracing::info!("Running group {:?} with {} tests", group, group_tests.len()); let mut futures = FuturesUnordered::new(); + let hypercore_clients = &self.hypercore_clients; for test in group_tests { let clients = self.clients.clone(); @@ -235,8 +230,15 @@ impl Runner { let report = self.report.clone(); let service = all_services.get(&test.name).cloned().unwrap(); futures.push(async move { - self.execute_test(&test, service, clients, component_sources, report) - .await + Self::execute_test( + &test, + service, + clients, + component_sources, + hypercore_clients, + report, + ) + .await }); } @@ -246,11 +248,11 @@ impl Runner { // Execute a single test with timings async fn execute_test( - &self, test: &TestDefinition, service_deployment: ServiceDeployment, clients: Arc, component_sources: Arc, + hypercore_clients: &HypercoreClients, report: TestReport, ) { report.start_test(test.name.clone()); @@ -260,7 +262,7 @@ impl Runner { service_deployment, &clients, &component_sources, - &self.registry, + hypercore_clients, ) .await .context(test.name.clone()) @@ -276,7 +278,7 @@ async fn run_test( service_deployment: ServiceDeployment, clients: &Clients, component_sources: &ComponentSources, - registry: &TestRegistry, + hypercore_clients: &HypercoreClients, ) -> anyhow::Result<()> { // For multi-operator tests, wait for P2P mesh to form before triggering if test.multi_operator && clients.http_clients.len() > 1 { @@ -495,7 +497,7 @@ async fn run_test( tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); - if let Some(hypercore_client) = registry.get_hypercore_client(&test.name) { + if let Some(hypercore_client) = hypercore_clients.get(&test.name) { let client_feed_key = hypercore_client.feed_key(); tracing::info!( "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", @@ -519,11 +521,9 @@ async fn run_test( .context("Failed to wait for hypercore stream to finalize")?; } - // Wait for hypercore mesh to stabilize - require at least 1 WAVS instance to connect - // In multi-operator mode, DHT discovery may not connect all operators reliably, - // but data will still replicate if at least one connection is established + // Wait for hypercore mesh to stabilize - require at least 1 WAVS instance to connect. + // If 0 peers are connected the append will never replicate, so fail fast. { - // Require at least 1 connection, but ideally all operators let min_required_peers = 1; let total_operators = clients.http_clients.len(); tracing::info!( @@ -532,31 +532,20 @@ async fn run_test( total_operators ); - // Make mesh readiness check non-blocking - warn if not ready but proceed anyway - // Use longer timeout in CI where DHT discovery may be slower - match wait_for_hypercore_mesh_ready( + let peer_count = wait_for_hypercore_mesh_ready( &hypercore_client, min_required_peers, - Duration::from_secs(60), + Duration::from_secs(90), ) .await - { - Ok(peer_count) => { - tracing::info!( - "Hypercore mesh ready for append: {} connected peers (min required: {}, total operators: {})", - peer_count, - min_required_peers, - total_operators - ); - } - Err(e) => { - tracing::warn!( - "Hypercore mesh not fully formed before append: {}. \ - Proceeding anyway - replication may still work when peers connect.", - e - ); - } - } + .context("Hypercore mesh not ready: 0 peers connected, append will never replicate")?; + + tracing::info!( + "Hypercore mesh ready for append: {} connected peers (min required: {}, total operators: {})", + peer_count, + min_required_peers, + total_operators + ); } // Verify feed keys match diff --git a/packages/layer-tests/src/e2e/test_registry.rs b/packages/layer-tests/src/e2e/test_registry.rs index 3169cf99f..db4482169 100644 --- a/packages/layer-tests/src/e2e/test_registry.rs +++ b/packages/layer-tests/src/e2e/test_registry.rs @@ -12,6 +12,7 @@ use wavs_types::AtProtoAction; use super::clients::Clients; use super::components::{AggregatorComponent, ComponentName, OperatorComponent}; use super::config::CRON_INTERVAL_DATA; +use super::handles::hypercore::HypercoreClients; use super::matrix::{CosmosService, CrossChainService, EvmService, TestMatrix}; use super::test_definition::{ AggregatorDefinition, CosmosTriggerDefinition, EvmTriggerDefinition, ExpectedOutput, InputData, @@ -38,34 +39,10 @@ pub enum CosmosContractDefinition { Submit(CosmosSubmitDefinition), } -use super::handles::hypercore::HypercoreTestClient; - -/// Factory data to create a hypercore test client -#[derive(Clone)] -pub struct HypercoreClientFactory { - pub test_name: String, - pub hyperswarm_bootstrap: Option, - pub signing_key_bytes: Vec, -} - /// Registry for managing test definitions and their deployed services +#[derive(Default)] pub struct TestRegistry { tests: Vec, - /// Map of test name to hypercore client factory for real hypercore e2e tests - /// Client is created just before test runs to avoid DHT announcement expiration - hypercore_client_factories: DashMap, - /// Map of test name to actually created hypercore test client - hypercore_clients: DashMap>, -} - -impl Default for TestRegistry { - fn default() -> Self { - Self { - tests: Vec::new(), - hypercore_client_factories: DashMap::new(), - hypercore_clients: DashMap::new(), - } - } } impl TestRegistry { @@ -102,52 +79,16 @@ impl TestRegistry { self.tests.iter() } - /// Get a hypercore test client by test name - pub fn get_hypercore_client(&self, test_name: &str) -> Option> { - self.hypercore_clients.get(test_name).map(|v| v.clone()) - } - - /// Store a hypercore client factory for a test (client will be created later) - pub fn insert_hypercore_client_factory( - &self, - test_name: String, - factory: HypercoreClientFactory, - ) { - self.hypercore_client_factories.insert(test_name, factory); - } - - /// Create hypercore clients from factories for all tests that need them - /// Called right before tests run to ensure DHT announcements are fresh - pub async fn create_hypercore_clients(&self) -> anyhow::Result<()> { - for entry in self.hypercore_client_factories.iter() { - let test_name = entry.key(); - let factory = entry.value(); - if !self.hypercore_clients.contains_key(test_name) { - tracing::info!( - "Creating hypercore client for test '{}' right before test execution", - test_name - ); - let client = HypercoreTestClient::new( - &factory.test_name, - factory.hyperswarm_bootstrap.clone(), - &factory.signing_key_bytes, - ) - .await?; - self.hypercore_clients - .insert(test_name.clone(), Arc::new(client)); - } - } - Ok(()) - } - - /// Create a registry based on the test mode + /// Create a registry based on the test mode. + /// Returns the registry and a `HypercoreClients` with any pending entries + /// that were registered during test definition. pub async fn from_test_mode( test_mode: crate::config::TestMode, chain_configs: Arc>, clients: &Clients, cosmos_code_map: &CosmosCodeMap, hyperswarm_bootstrap: Option, - ) -> Self { + ) -> (Self, HypercoreClients) { // Convert TestMode to TestMatrix let matrix: TestMatrix = test_mode.into(); @@ -155,6 +96,7 @@ impl TestRegistry { let chains = ChainKeys::from_config(&chain_configs.read().unwrap()); let mut registry = Self::new(); + let mut hypercore_clients = HypercoreClients::new(); // Process EVM services for service in &matrix.evm { @@ -169,7 +111,11 @@ impl TestRegistry { } EvmService::HypercoreEchoData => { registry - .register_evm_hypercore_echo_data_test(chain, hyperswarm_bootstrap.clone()) + .register_evm_hypercore_echo_data_test( + chain, + hyperswarm_bootstrap.clone(), + &mut hypercore_clients, + ) .await; } EvmService::EchoDataSecondaryChain => { @@ -290,7 +236,7 @@ impl TestRegistry { } } - registry + (registry, hypercore_clients) } // Helper function to create simple aggregator configuration @@ -360,6 +306,7 @@ impl TestRegistry { &mut self, chain: &ChainKey, hyperswarm_bootstrap: Option, + hypercore_clients: &mut HypercoreClients, ) -> &mut Self { // Generate signing key now to get feed_key for the trigger, // but delay creating the full client until right before test runs @@ -378,14 +325,11 @@ impl TestRegistry { feed_key ); - // Store the factory to create the client later - self.insert_hypercore_client_factory( + // Queue deferred client creation (actual client created right before test runs) + hypercore_clients.insert_pending( test_name.to_string(), - HypercoreClientFactory { - test_name: test_name.to_string(), - hyperswarm_bootstrap, - signing_key_bytes: signing_key_bytes.to_vec(), - }, + hyperswarm_bootstrap, + signing_key_bytes.to_vec(), ); self.register( From b0e30fc228b431450e300411914b9e0543ac77a0 Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 13 Feb 2026 13:07:39 -0600 Subject: [PATCH 03/10] Error early --- .../layer-tests/src/e2e/handles/hypercore.rs | 20 +++++++++---------- packages/layer-tests/src/e2e/runner.rs | 7 ++++--- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/packages/layer-tests/src/e2e/handles/hypercore.rs b/packages/layer-tests/src/e2e/handles/hypercore.rs index 2462ab2ff..6f5723a82 100644 --- a/packages/layer-tests/src/e2e/handles/hypercore.rs +++ b/packages/layer-tests/src/e2e/handles/hypercore.rs @@ -46,19 +46,17 @@ impl HypercoreClients { .insert(test_name, (hyperswarm_bootstrap, signing_key_bytes)); } - /// Create hypercore clients for all pending tests. + /// Create hypercore clients for all pending tests, draining the pending queue. /// Called right before tests run to ensure DHT announcements are fresh. + /// No-op if the queue has already been drained. pub async fn create_clients(&mut self) -> anyhow::Result<()> { - for (test_name, (bootstrap, key_bytes)) in &self.pending { - if !self.clients.contains_key(test_name) { - tracing::info!( - "Creating hypercore client for test '{}' right before test execution", - test_name - ); - let client = - HypercoreTestClient::new(test_name, bootstrap.clone(), key_bytes).await?; - self.clients.insert(test_name.clone(), Arc::new(client)); - } + for (test_name, (bootstrap, key_bytes)) in self.pending.drain() { + tracing::info!( + "Creating hypercore client for test '{}' right before test execution", + test_name + ); + let client = HypercoreTestClient::new(&test_name, bootstrap, &key_bytes).await?; + self.clients.insert(test_name, Arc::new(client)); } Ok(()) } diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index c5d2868a6..43521a3f6 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -212,9 +212,10 @@ impl Runner { // Create hypercore clients AFTER deploying services so WAVS is already // doing DHT lookups when the test client announces. This avoids the stale-DHT // problem where the client announces 10+ seconds before WAVS starts looking. - if let Err(e) = self.hypercore_clients.create_clients().await { - tracing::error!("Failed to create hypercore clients: {}", e); - } + self.hypercore_clients + .create_clients() + .await + .expect("Failed to create hypercore clients"); // All services are now deployed and ready for the tests // From here on in we're strictly testing the trigger->execute->aggregate->submit flow From b7fc3bc4269f069c8b726ef5e4012066bfef4907 Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 20 Feb 2026 12:38:25 -0600 Subject: [PATCH 04/10] Fix hypercore e2e test deadlock in trigger stream finalization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wait_for_evm_trigger_streams_to_finalize() function called finalized() which required hypercore feeds to be Connected. This ran during update_services() before the test client was created, so the hypercore feed had no peer to connect to and the check could never pass — causing a 30s timeout panic. Split finalized() into chains_finalized() and hypercore_finalized() so the EVM stream wait only checks chain readiness. Hypercore readiness is already checked separately later via wait_for_hypercore_streams_to_finalize(). Co-Authored-By: Claude Opus 4.6 --- packages/layer-tests/src/e2e/helpers.rs | 1544 ++++++++--------- .../layer-tests/src/e2e/service_managers.rs | 1041 +++++------ packages/types/src/http.rs | 302 ++-- 3 files changed, 1449 insertions(+), 1438 deletions(-) diff --git a/packages/layer-tests/src/e2e/helpers.rs b/packages/layer-tests/src/e2e/helpers.rs index eb0636c8f..991e4ffaf 100644 --- a/packages/layer-tests/src/e2e/helpers.rs +++ b/packages/layer-tests/src/e2e/helpers.rs @@ -1,772 +1,772 @@ -use alloy_primitives::U256; -use alloy_provider::{ext::AnvilApi, Provider}; -use alloy_sol_types::SolEvent; -use anyhow::{anyhow, Context, Result}; -use deadpool::managed::Object; -use layer_climb::pool::SigningClientPoolManager; -use layer_climb::prelude::CosmosAddr; -use std::{collections::BTreeMap, num::NonZero, sync::Arc, time::Duration}; -use utils::evm_client::AnyNonceManager; -use utils::{config::WAVS_ENV_PREFIX, evm_client::EvmSigningClient, filesystem::workspace_path}; -use uuid::Uuid; -use wavs_cli::clients::HttpClient; - -use wavs_types::{ - AllowedHostPermission, ByteArray, ChainKey, Component, DevHypercoreStreamState, - DevTriggerStreamSubscriptionKind, Permissions, Service, ServiceManager, ServiceStatus, - SignatureKind, Submit, Trigger, Workflow, -}; - -use crate::deployment::{ServiceDeployment, WorkflowDeployment}; - -use crate::e2e::test_definition::CosmosSubmitDefinition; -use crate::e2e::test_registry::CosmosContractDefinition; -use crate::example_cosmos_client::SimpleCosmosSubmitClient; -use crate::{ - e2e::{ - clients::Clients, - components::ComponentSources, - config::BLOCK_INTERVAL, - test_definition::{ - AggregatorDefinition, ChangeServiceDefinition, ComponentDefinition, SubmitDefinition, - TestDefinition, TriggerDefinition, - }, - }, - example_cosmos_client::SimpleCosmosTriggerClient, - example_evm_client::{ - example_submit::ISimpleSubmit::SignedData, example_trigger::SimpleTrigger, LogSpamClient, - SimpleEvmSubmitClient, TriggerId, - }, -}; - -use super::{ - test_definition::{CosmosTriggerDefinition, EvmTriggerDefinition, WorkflowDefinition}, - test_registry::CosmosCodeMap, -}; - -/// Helper function to deploy a service for a test -pub async fn create_service_for_test( - test: &TestDefinition, - clients: &Clients, - component_sources: &ComponentSources, - service_manager: ServiceManager, - cosmos_code_map: CosmosCodeMap, -) -> ServiceDeployment { - tracing::info!("Deploying service for test: {}", test.name); - tracing::info!("Service manager: {:?}", service_manager); - tracing::info!( - "[{}] Deploying service manager on chain {}", - test.name, - service_manager.chain() - ); - - // No need to load the actual service, it was a placeholder - let mut service = Service { - name: test.name.clone(), - workflows: BTreeMap::new(), - status: ServiceStatus::Active, - manager: service_manager, - }; - - let mut submission_handlers = BTreeMap::new(); - - for (workflow_id, workflow_definition) in test.workflows.iter() { - let deployment_result = deploy_workflow( - &test.name, - workflow_definition, - service.manager.clone(), - clients, - component_sources, - cosmos_code_map.clone(), - ) - .await; - - service - .workflows - .insert(workflow_id.clone(), deployment_result.workflow); - submission_handlers.insert(workflow_id.clone(), deployment_result.submission_handler); - } - - ServiceDeployment { - service, - submission_handlers, - } -} - -fn deploy_component( - component_sources: &ComponentSources, - component_definition: &ComponentDefinition, - config_vars: BTreeMap, - env_vars: BTreeMap, -) -> Component { - // Create components from test definition - let component_source = component_sources - .lookup - .get(&component_definition.name) - .unwrap() - .clone(); - - let mut component = Component::new(component_source); - component.permissions = Permissions { - allowed_http_hosts: AllowedHostPermission::All, - file_system: true, - raw_sockets: true, - dns_resolution: true, - }; - component.config = config_vars; - // Set env_keys to the actual prefixed env var names that will be read by the component - component.env_keys = env_vars - .keys() - .map(|k| format!("{}_{}", WAVS_ENV_PREFIX, k)) - .collect(); - - for (k, v) in env_vars.iter() { - // NOTE: we should avoid collisions here - std::env::set_var(format!("{}_{}", WAVS_ENV_PREFIX, k), v); - } - - component -} - -async fn deploy_workflow( - test_name: &str, - workflow_definition: &WorkflowDefinition, - service_manager: ServiceManager, - clients: &Clients, - component_sources: &ComponentSources, - cosmos_code_map: CosmosCodeMap, -) -> WorkflowDeployment { - let component = deploy_component( - component_sources, - &workflow_definition.component, - Default::default(), - Default::default(), - ); - - tracing::info!("[{}] Creating submit from config", test_name); - - let submission_contract = - deploy_submit_contract(clients, cosmos_code_map.clone(), service_manager) - .await - .unwrap(); - - let submit = create_submit_from_config( - &workflow_definition.submit, - &submission_contract, - Some(component_sources), - ) - .await - .unwrap(); - - tracing::info!("[{}] Creating trigger from config", test_name); - // Create the trigger based on test configuration - let trigger = create_trigger_from_config( - workflow_definition.trigger.clone(), - clients, - cosmos_code_map.clone(), - Some(workflow_definition), - ) - .await; - - // Create service workflows - WorkflowDeployment { - workflow: Workflow { - trigger: trigger.clone(), // Clone for possible use in multi-trigger service - component, - submit: submit.clone(), - }, - submission_handler: submission_contract, - } -} - -/// Create a trigger based on test configuration -pub async fn create_trigger_from_config( - trigger_definition: TriggerDefinition, - clients: &Clients, - cosmos_code_map: CosmosCodeMap, - _workflow_definition: Option<&WorkflowDefinition>, -) -> Trigger { - match trigger_definition { - TriggerDefinition::NewEvmContract(evm_trigger_definition) => match evm_trigger_definition { - EvmTriggerDefinition::SimpleContractEvent { chain } => { - let client = clients.get_evm_client(&chain); - - // Deploy a new EVM trigger contract - tracing::info!("Deploying EVM trigger contract on chain {}", chain); - let contract = SimpleTrigger::deploy(client.provider.clone()) - .await - .unwrap(); - let address = *contract.address(); - - // Get the event hash - let event_hash = - *crate::example_evm_client::example_trigger::NewTrigger::SIGNATURE_HASH; - - Trigger::EvmContractEvent { - chain: chain.clone(), - address, - event_hash: ByteArray::new(event_hash), - } - } - }, - TriggerDefinition::NewCosmosContract(cosmos_trigger_definition) => { - match cosmos_trigger_definition.clone() { - CosmosTriggerDefinition::SimpleContractEvent { ref chain } => { - let client = clients.get_cosmos_client(chain).await; - - // Get the code ID with better error handling - tracing::info!("Getting cosmos code ID for chain {}", chain); - let code_id = get_cosmos_code_id( - clients, - &CosmosContractDefinition::Trigger(cosmos_trigger_definition), - cosmos_code_map, - ) - .await; - - tracing::info!("Using cosmos code ID: {} for chain {}", code_id, chain); - - // Deploy a new Cosmos trigger contract with better error handling - let contract_name = format!("simple_trigger_{}", Uuid::now_v7()); - tracing::info!( - "Instantiating new contract '{}' with code ID {} on chain {}", - contract_name, - code_id, - chain - ); - - let contract = - SimpleCosmosTriggerClient::new_code_id(client, code_id, &contract_name) - .await - .unwrap(); - - tracing::info!( - "Successfully deployed cosmos contract at address: {}", - contract.contract_address - ); - - Trigger::CosmosContractEvent { - chain: chain.clone(), - address: contract.contract_address.try_into().unwrap(), - event_type: cw_wavs_trigger_api::simple::PushMessageEvent::EVENT_TYPE - .to_string(), - } - } - } - } - TriggerDefinition::BlockInterval { chain, start_stop } => match start_stop { - false => Trigger::BlockInterval { - chain, - n_blocks: BLOCK_INTERVAL, - start_block: None, - end_block: None, - }, - true => { - let current_block = if clients.evm_clients.contains_key(&chain) { - let client = clients.get_evm_client(&chain); - client.provider.get_block_number().await.unwrap() - } else if clients.cosmos_client_pools.contains_key(&chain) { - let client = clients.get_cosmos_client(&chain).await; - client.querier.block_height().await.unwrap() - } else { - panic!("Chain is not configured: {}", chain) - }; - - let current_block = NonZero::new(current_block).unwrap(); - - Trigger::BlockInterval { - chain, - n_blocks: BLOCK_INTERVAL, - start_block: Some(current_block), - end_block: Some(current_block), - } - } - }, - TriggerDefinition::Existing(trigger) => trigger.clone(), - } -} - -/// Create a submit based on test configuration -pub async fn create_submit_from_config( - submit_config: &SubmitDefinition, - submission_contract: &layer_climb::prelude::Address, - component_sources: Option<&ComponentSources>, -) -> Result { - match submit_config { - SubmitDefinition::Aggregator(aggregator) => match aggregator { - AggregatorDefinition::ComponentBasedAggregator { - component: component_def, - .. - } => { - let sources = component_sources.ok_or_else(|| { - anyhow!("ComponentBasedAggregator requires component_sources") - })?; - - let mut config_vars = BTreeMap::new(); - let mut env_vars = BTreeMap::new(); - - for (hardcoded_key, hardcoded_value) in &component_def.configs_to_add.hardcoded { - config_vars.insert(hardcoded_key.clone(), hardcoded_value.clone()); - } - - for (env_key, env_value) in &component_def.env_vars_to_add { - env_vars.insert(env_key.clone(), env_value.clone()); - } - - if component_def.configs_to_add.service_handler { - config_vars.insert( - "service_handler".to_string(), - submission_contract.to_string(), - ); - } - - let component = deploy_component(sources, component_def, config_vars, env_vars); - - Ok(Submit::Aggregator { - component: Box::new(component), - signature_kind: SignatureKind::evm_default(), - }) - } - }, - } -} - -/// Deploy submit contract and return its address -pub async fn deploy_submit_contract( - clients: &Clients, - cosmos_code_map: CosmosCodeMap, - service_manager: ServiceManager, -) -> Result { - match service_manager { - ServiceManager::Cosmos { chain, address } => { - let code_id = get_cosmos_code_id( - clients, - &CosmosContractDefinition::Submit(CosmosSubmitDefinition::MockServiceHandler { - chain: chain.clone(), - }), - cosmos_code_map, - ) - .await; - - let client = clients.get_cosmos_client(&chain).await; - let contract_client = - crate::example_cosmos_client::SimpleCosmosSubmitClient::new_code_id( - client, - code_id, - &address, - "Mock service handler", - ) - .await?; - - Ok(contract_client.contract_address) - } - ServiceManager::Evm { chain, address } => { - let evm_client = clients.get_evm_client(&chain); - - tracing::info!( - "Deploying submit contract on chain {} with service manager: {}", - chain, - address - ); - - let result = crate::example_evm_client::example_submit::SimpleSubmit::deploy( - evm_client.provider.clone(), - address, - ) - .await - .context("Failed to deploy submit contract")?; - - let address = *result.address(); - tracing::info!("Submit contract deployed at address: {}", address); - - Ok(address.into()) - } - } -} - -/// Deploy LogSpam contract and return its address -pub async fn deploy_log_spam_contract( - clients: &Clients, - chain: &ChainKey, -) -> Result { - let evm_client = clients.get_evm_client(chain); - - tracing::info!("Deploying LogSpam contract on chain {}", chain); - - let address = LogSpamClient::deploy(evm_client.provider.clone()) - .await - .context("Failed to deploy LogSpam contract")?; - - tracing::info!("LogSpam contract deployed at address: {}", address); - - Ok(address) -} - -/// Deploy submit contract and create a Submit from it -pub async fn get_cosmos_code_id( - clients: &Clients, - cosmos_contract_definition: &CosmosContractDefinition, - cosmos_code_map: CosmosCodeMap, -) -> u64 { - // Get or insert the entry - let entry = cosmos_code_map - .entry(cosmos_contract_definition.clone()) - .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(None))) - .clone(); - - // try to read (non-blocking for other readers) - { - let read_guard = entry.read().await; - if let Some(code_id) = *read_guard { - return code_id; - } - } - - // cache miss, acquire write lock for upload - let mut write_guard = entry.write().await; - - // check cache after acquiring write lock, if another thread already uploaded - if let Some(code_id) = *write_guard { - return code_id; - } - - // Upload since not cached - let (chain, cosmos_bytecode) = match cosmos_contract_definition { - CosmosContractDefinition::Trigger(CosmosTriggerDefinition::SimpleContractEvent { - chain, - }) => { - let wasm_path = workspace_path() - .join("examples") - .join("build") - .join("contracts") - .join("cw_wavs_trigger_simple.wasm"); - - if !wasm_path.exists() { - panic!( - "Cosmos contract WASM file not found at: {}", - wasm_path.display() - ); - } - - (chain, tokio::fs::read(&wasm_path).await.unwrap()) - } - CosmosContractDefinition::Submit(CosmosSubmitDefinition::MockServiceHandler { chain }) => { - let wasm_path = workspace_path() - .join("examples") - .join("build") - .join("contracts") - .join("cw_wavs_mock_service_handler.wasm"); - - if !wasm_path.exists() { - panic!( - "Cosmos contract WASM file not found at: {}", - wasm_path.display() - ); - } - - (chain, tokio::fs::read(&wasm_path).await.unwrap()) - } - }; - - tracing::info!( - "Uploading cosmos wasm byte code ({} bytes) to chain {}", - cosmos_bytecode.len(), - chain - ); - - let client = clients.get_cosmos_client(chain).await; - - let (code_id, _) = client - .contract_upload_file(cosmos_bytecode, None) - .await - .unwrap(); - - tracing::info!( - "Successfully uploaded WASM bytecode to chain {}, code_id: {}", - chain, - code_id - ); - - // Cache result and return - *write_guard = Some(code_id); - code_id -} - -/// Simulate a re-org by reverting to a previous block and mining new blocks -pub async fn simulate_anvil_reorg( - evm_client: &EvmSigningClient, - reorg_snapshot: U256, -) -> Result<()> { - // Revert to the specified block using Anvil's revert RPC - evm_client.provider.anvil_revert(reorg_snapshot).await?; - - // Update nonce - if let AnyNonceManager::Fast(fast_nonce_manager) = &evm_client.nonce_manager { - fast_nonce_manager - .set_current_nonce(&evm_client.provider) - .await - .unwrap(); - } - - // Mine new blocks to simulate chain reorganization - evm_client.provider.evm_mine(None).await?; - Ok(()) -} - -pub async fn evm_wait_for_task_to_land( - evm_submit_client: EvmSigningClient, - address: alloy_primitives::Address, - trigger_id: TriggerId, - submit_start_block: u64, - timeout: Duration, -) -> Result { - let submit_client = SimpleEvmSubmitClient::new(evm_submit_client, address); - - tokio::time::timeout(timeout, async move { - loop { - let current_block = submit_client - .evm_client - .provider - .get_block_number() - .await - .map_err(|e| anyhow!("Failed to get block number: {e}"))?; - - if current_block <= submit_start_block { - submit_client.evm_client.provider.evm_mine(None).await?; - } - - if submit_client.trigger_validated(trigger_id).await { - return submit_client - .signed_data(trigger_id) - .await - .map_err(|e| anyhow!("Failed to get signed data: {e}")); - } - - tracing::debug!("Waiting for task response on trigger {}", trigger_id); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - }) - .await - .map_err(|_| anyhow::anyhow!("Timeout when waiting for task to land"))? -} - -pub async fn cosmos_wait_for_task_to_land( - cosmos_submit_client: Object, - address: CosmosAddr, - trigger_id: TriggerId, - timeout: Duration, -) -> Result> { - let submit_client = SimpleCosmosSubmitClient::new(cosmos_submit_client, address.into()); - - let trigger_id = trigger_id.u64(); - tokio::time::timeout(timeout, async move { - loop { - if submit_client - .trigger_validated(trigger_id) - .await - .unwrap_or(false) - { - return submit_client - .trigger_message(trigger_id) - .await - .map_err(|e| anyhow!("Failed to get signed data: {e}")); - } - - tracing::debug!("Waiting for task response on trigger {}", trigger_id); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } - }) - .await - .map_err(|_| anyhow::anyhow!("Timeout when waiting for task to land"))? -} - -/// Helper function to deploy a service for a test -pub async fn change_service_for_test( - service: &mut Service, - change_service: ChangeServiceDefinition, - clients: &Clients, - component_sources: &ComponentSources, - cosmos_code_map: CosmosCodeMap, -) { - match change_service { - ChangeServiceDefinition::Component { - workflow_id, - component: component_definition, - } => { - let component = deploy_component( - component_sources, - &component_definition, - Default::default(), - Default::default(), - ); - let workflow = service - .workflows - .get_mut(&workflow_id) - .expect("Workflow not found in service"); - - workflow.component = component; - } - ChangeServiceDefinition::AddWorkflow { - workflow_id, - workflow, - } => { - let deployed_workflow = deploy_workflow( - &workflow_id, - &workflow, - service.manager.clone(), - clients, - component_sources, - cosmos_code_map, - ) - .await; - - service - .workflows - .insert(workflow_id.clone(), deployed_workflow.workflow); - } - } -} - -pub async fn wait_for_evm_trigger_streams_to_finalize( - client: &HttpClient, - service_manager: Option, -) { - tokio::time::timeout(Duration::from_secs(30), async { - loop { - tracing::info!("Getting trigger stream info..."); - let info = client.get_trigger_streams_info().await.unwrap(); - - if info.finalized() { - if let Some(service_manager) = &service_manager { - match service_manager { - ServiceManager::Evm { chain, address } => { - let address = ByteArray::new(address.into_array()); - if info.chains.iter().any(|(key, value)| { - key == chain - && value.active_subscriptions.values().any(|kind| match kind { - DevTriggerStreamSubscriptionKind::Logs { - addresses, - .. - } => addresses.contains(&address), - _ => false, - }) - }) { - break; - } - } - ServiceManager::Cosmos { .. } => { - unreachable!("This is only meant for EVM"); - } - } - } else if info.any_active_subscriptions() { - break; - } - } else { - tracing::warn!("Still waiting for trigger streams to finalize"); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - .await - .unwrap(); -} - -pub async fn wait_for_hypercore_streams_to_finalize( - client: &HttpClient, - feed_key: &str, - timeout: Option, -) -> anyhow::Result<()> { - let timeout = timeout.unwrap_or(Duration::from_secs(30)); - let start = std::time::Instant::now(); - let mut poll_interval = Duration::from_millis(100); - - tokio::time::timeout(timeout, async { - loop { - // Retry HTTP request with backoff on failure - match client.get_trigger_streams_info().await { - Ok(info) => { - match info.hypercore.get(feed_key) { - Some(DevHypercoreStreamState::Connected) => { - tracing::info!("Hypercore stream connected for feed_key {}", feed_key); - return Ok(()); - } - Some(DevHypercoreStreamState::Connecting) => { - tracing::info!("Hypercore stream connecting for feed_key {}", feed_key); - } - Some(DevHypercoreStreamState::Waiting) => { - tracing::info!("Hypercore stream waiting for feed_key {}", feed_key); - } - None => { - tracing::info!( - "Hypercore stream not registered yet for feed_key {}", - feed_key - ); - } - } - // Reset poll interval on successful response - poll_interval = Duration::from_millis(100); - } - Err(e) => { - tracing::warn!("HTTP error getting trigger streams info: {}", e); - // Exponential backoff for HTTP errors - poll_interval = (poll_interval * 2).min(Duration::from_secs(1)); - } - } - - tokio::time::sleep(poll_interval).await; - - // Log progress every 5 seconds - if start.elapsed().as_secs().is_multiple_of(5) { - tracing::info!( - "Still waiting for hypercore stream (elapsed: {}s)", - start.elapsed().as_secs() - ); - } - } - }) - .await - .map_err(|_| { - anyhow::anyhow!( - "Timed out waiting for hypercore stream to connect (feed_key: {})", - feed_key - ) - })? -} - -/// Wait for hypercore mesh to form by checking the test client's peer connection count directly. -/// -/// This is used in multi-operator tests to ensure all operators have discovered each other -/// via hyperswarm before proceeding with test execution. -pub async fn wait_for_hypercore_mesh_ready( - hypercore_client: &std::sync::Arc, - expected_peers: usize, - timeout: Duration, -) -> anyhow::Result { - let start = std::time::Instant::now(); - let mut poll_interval = Duration::from_millis(100); - - loop { - let peer_count = hypercore_client.connected_peer_count(); - - if peer_count >= expected_peers { - tracing::info!( - "Hypercore mesh ready: {} connected peers (expected {})", - peer_count, - expected_peers - ); - return Ok(peer_count); - } - - if start.elapsed() > timeout { - anyhow::bail!( - "Timeout waiting for hypercore mesh: {} peers connected, expected {}", - peer_count, - expected_peers - ); - } - - // Exponential backoff with jitter for mesh formation - let wait_time = poll_interval; - poll_interval = std::cmp::min(Duration::from_secs(2), poll_interval * 2); - tokio::time::sleep(wait_time).await; - } -} +use alloy_primitives::U256; +use alloy_provider::{ext::AnvilApi, Provider}; +use alloy_sol_types::SolEvent; +use anyhow::{anyhow, Context, Result}; +use deadpool::managed::Object; +use layer_climb::pool::SigningClientPoolManager; +use layer_climb::prelude::CosmosAddr; +use std::{collections::BTreeMap, num::NonZero, sync::Arc, time::Duration}; +use utils::evm_client::AnyNonceManager; +use utils::{config::WAVS_ENV_PREFIX, evm_client::EvmSigningClient, filesystem::workspace_path}; +use uuid::Uuid; +use wavs_cli::clients::HttpClient; + +use wavs_types::{ + AllowedHostPermission, ByteArray, ChainKey, Component, DevHypercoreStreamState, + DevTriggerStreamSubscriptionKind, Permissions, Service, ServiceManager, ServiceStatus, + SignatureKind, Submit, Trigger, Workflow, +}; + +use crate::deployment::{ServiceDeployment, WorkflowDeployment}; + +use crate::e2e::test_definition::CosmosSubmitDefinition; +use crate::e2e::test_registry::CosmosContractDefinition; +use crate::example_cosmos_client::SimpleCosmosSubmitClient; +use crate::{ + e2e::{ + clients::Clients, + components::ComponentSources, + config::BLOCK_INTERVAL, + test_definition::{ + AggregatorDefinition, ChangeServiceDefinition, ComponentDefinition, SubmitDefinition, + TestDefinition, TriggerDefinition, + }, + }, + example_cosmos_client::SimpleCosmosTriggerClient, + example_evm_client::{ + example_submit::ISimpleSubmit::SignedData, example_trigger::SimpleTrigger, LogSpamClient, + SimpleEvmSubmitClient, TriggerId, + }, +}; + +use super::{ + test_definition::{CosmosTriggerDefinition, EvmTriggerDefinition, WorkflowDefinition}, + test_registry::CosmosCodeMap, +}; + +/// Helper function to deploy a service for a test +pub async fn create_service_for_test( + test: &TestDefinition, + clients: &Clients, + component_sources: &ComponentSources, + service_manager: ServiceManager, + cosmos_code_map: CosmosCodeMap, +) -> ServiceDeployment { + tracing::info!("Deploying service for test: {}", test.name); + tracing::info!("Service manager: {:?}", service_manager); + tracing::info!( + "[{}] Deploying service manager on chain {}", + test.name, + service_manager.chain() + ); + + // No need to load the actual service, it was a placeholder + let mut service = Service { + name: test.name.clone(), + workflows: BTreeMap::new(), + status: ServiceStatus::Active, + manager: service_manager, + }; + + let mut submission_handlers = BTreeMap::new(); + + for (workflow_id, workflow_definition) in test.workflows.iter() { + let deployment_result = deploy_workflow( + &test.name, + workflow_definition, + service.manager.clone(), + clients, + component_sources, + cosmos_code_map.clone(), + ) + .await; + + service + .workflows + .insert(workflow_id.clone(), deployment_result.workflow); + submission_handlers.insert(workflow_id.clone(), deployment_result.submission_handler); + } + + ServiceDeployment { + service, + submission_handlers, + } +} + +fn deploy_component( + component_sources: &ComponentSources, + component_definition: &ComponentDefinition, + config_vars: BTreeMap, + env_vars: BTreeMap, +) -> Component { + // Create components from test definition + let component_source = component_sources + .lookup + .get(&component_definition.name) + .unwrap() + .clone(); + + let mut component = Component::new(component_source); + component.permissions = Permissions { + allowed_http_hosts: AllowedHostPermission::All, + file_system: true, + raw_sockets: true, + dns_resolution: true, + }; + component.config = config_vars; + // Set env_keys to the actual prefixed env var names that will be read by the component + component.env_keys = env_vars + .keys() + .map(|k| format!("{}_{}", WAVS_ENV_PREFIX, k)) + .collect(); + + for (k, v) in env_vars.iter() { + // NOTE: we should avoid collisions here + std::env::set_var(format!("{}_{}", WAVS_ENV_PREFIX, k), v); + } + + component +} + +async fn deploy_workflow( + test_name: &str, + workflow_definition: &WorkflowDefinition, + service_manager: ServiceManager, + clients: &Clients, + component_sources: &ComponentSources, + cosmos_code_map: CosmosCodeMap, +) -> WorkflowDeployment { + let component = deploy_component( + component_sources, + &workflow_definition.component, + Default::default(), + Default::default(), + ); + + tracing::info!("[{}] Creating submit from config", test_name); + + let submission_contract = + deploy_submit_contract(clients, cosmos_code_map.clone(), service_manager) + .await + .unwrap(); + + let submit = create_submit_from_config( + &workflow_definition.submit, + &submission_contract, + Some(component_sources), + ) + .await + .unwrap(); + + tracing::info!("[{}] Creating trigger from config", test_name); + // Create the trigger based on test configuration + let trigger = create_trigger_from_config( + workflow_definition.trigger.clone(), + clients, + cosmos_code_map.clone(), + Some(workflow_definition), + ) + .await; + + // Create service workflows + WorkflowDeployment { + workflow: Workflow { + trigger: trigger.clone(), // Clone for possible use in multi-trigger service + component, + submit: submit.clone(), + }, + submission_handler: submission_contract, + } +} + +/// Create a trigger based on test configuration +pub async fn create_trigger_from_config( + trigger_definition: TriggerDefinition, + clients: &Clients, + cosmos_code_map: CosmosCodeMap, + _workflow_definition: Option<&WorkflowDefinition>, +) -> Trigger { + match trigger_definition { + TriggerDefinition::NewEvmContract(evm_trigger_definition) => match evm_trigger_definition { + EvmTriggerDefinition::SimpleContractEvent { chain } => { + let client = clients.get_evm_client(&chain); + + // Deploy a new EVM trigger contract + tracing::info!("Deploying EVM trigger contract on chain {}", chain); + let contract = SimpleTrigger::deploy(client.provider.clone()) + .await + .unwrap(); + let address = *contract.address(); + + // Get the event hash + let event_hash = + *crate::example_evm_client::example_trigger::NewTrigger::SIGNATURE_HASH; + + Trigger::EvmContractEvent { + chain: chain.clone(), + address, + event_hash: ByteArray::new(event_hash), + } + } + }, + TriggerDefinition::NewCosmosContract(cosmos_trigger_definition) => { + match cosmos_trigger_definition.clone() { + CosmosTriggerDefinition::SimpleContractEvent { ref chain } => { + let client = clients.get_cosmos_client(chain).await; + + // Get the code ID with better error handling + tracing::info!("Getting cosmos code ID for chain {}", chain); + let code_id = get_cosmos_code_id( + clients, + &CosmosContractDefinition::Trigger(cosmos_trigger_definition), + cosmos_code_map, + ) + .await; + + tracing::info!("Using cosmos code ID: {} for chain {}", code_id, chain); + + // Deploy a new Cosmos trigger contract with better error handling + let contract_name = format!("simple_trigger_{}", Uuid::now_v7()); + tracing::info!( + "Instantiating new contract '{}' with code ID {} on chain {}", + contract_name, + code_id, + chain + ); + + let contract = + SimpleCosmosTriggerClient::new_code_id(client, code_id, &contract_name) + .await + .unwrap(); + + tracing::info!( + "Successfully deployed cosmos contract at address: {}", + contract.contract_address + ); + + Trigger::CosmosContractEvent { + chain: chain.clone(), + address: contract.contract_address.try_into().unwrap(), + event_type: cw_wavs_trigger_api::simple::PushMessageEvent::EVENT_TYPE + .to_string(), + } + } + } + } + TriggerDefinition::BlockInterval { chain, start_stop } => match start_stop { + false => Trigger::BlockInterval { + chain, + n_blocks: BLOCK_INTERVAL, + start_block: None, + end_block: None, + }, + true => { + let current_block = if clients.evm_clients.contains_key(&chain) { + let client = clients.get_evm_client(&chain); + client.provider.get_block_number().await.unwrap() + } else if clients.cosmos_client_pools.contains_key(&chain) { + let client = clients.get_cosmos_client(&chain).await; + client.querier.block_height().await.unwrap() + } else { + panic!("Chain is not configured: {}", chain) + }; + + let current_block = NonZero::new(current_block).unwrap(); + + Trigger::BlockInterval { + chain, + n_blocks: BLOCK_INTERVAL, + start_block: Some(current_block), + end_block: Some(current_block), + } + } + }, + TriggerDefinition::Existing(trigger) => trigger.clone(), + } +} + +/// Create a submit based on test configuration +pub async fn create_submit_from_config( + submit_config: &SubmitDefinition, + submission_contract: &layer_climb::prelude::Address, + component_sources: Option<&ComponentSources>, +) -> Result { + match submit_config { + SubmitDefinition::Aggregator(aggregator) => match aggregator { + AggregatorDefinition::ComponentBasedAggregator { + component: component_def, + .. + } => { + let sources = component_sources.ok_or_else(|| { + anyhow!("ComponentBasedAggregator requires component_sources") + })?; + + let mut config_vars = BTreeMap::new(); + let mut env_vars = BTreeMap::new(); + + for (hardcoded_key, hardcoded_value) in &component_def.configs_to_add.hardcoded { + config_vars.insert(hardcoded_key.clone(), hardcoded_value.clone()); + } + + for (env_key, env_value) in &component_def.env_vars_to_add { + env_vars.insert(env_key.clone(), env_value.clone()); + } + + if component_def.configs_to_add.service_handler { + config_vars.insert( + "service_handler".to_string(), + submission_contract.to_string(), + ); + } + + let component = deploy_component(sources, component_def, config_vars, env_vars); + + Ok(Submit::Aggregator { + component: Box::new(component), + signature_kind: SignatureKind::evm_default(), + }) + } + }, + } +} + +/// Deploy submit contract and return its address +pub async fn deploy_submit_contract( + clients: &Clients, + cosmos_code_map: CosmosCodeMap, + service_manager: ServiceManager, +) -> Result { + match service_manager { + ServiceManager::Cosmos { chain, address } => { + let code_id = get_cosmos_code_id( + clients, + &CosmosContractDefinition::Submit(CosmosSubmitDefinition::MockServiceHandler { + chain: chain.clone(), + }), + cosmos_code_map, + ) + .await; + + let client = clients.get_cosmos_client(&chain).await; + let contract_client = + crate::example_cosmos_client::SimpleCosmosSubmitClient::new_code_id( + client, + code_id, + &address, + "Mock service handler", + ) + .await?; + + Ok(contract_client.contract_address) + } + ServiceManager::Evm { chain, address } => { + let evm_client = clients.get_evm_client(&chain); + + tracing::info!( + "Deploying submit contract on chain {} with service manager: {}", + chain, + address + ); + + let result = crate::example_evm_client::example_submit::SimpleSubmit::deploy( + evm_client.provider.clone(), + address, + ) + .await + .context("Failed to deploy submit contract")?; + + let address = *result.address(); + tracing::info!("Submit contract deployed at address: {}", address); + + Ok(address.into()) + } + } +} + +/// Deploy LogSpam contract and return its address +pub async fn deploy_log_spam_contract( + clients: &Clients, + chain: &ChainKey, +) -> Result { + let evm_client = clients.get_evm_client(chain); + + tracing::info!("Deploying LogSpam contract on chain {}", chain); + + let address = LogSpamClient::deploy(evm_client.provider.clone()) + .await + .context("Failed to deploy LogSpam contract")?; + + tracing::info!("LogSpam contract deployed at address: {}", address); + + Ok(address) +} + +/// Deploy submit contract and create a Submit from it +pub async fn get_cosmos_code_id( + clients: &Clients, + cosmos_contract_definition: &CosmosContractDefinition, + cosmos_code_map: CosmosCodeMap, +) -> u64 { + // Get or insert the entry + let entry = cosmos_code_map + .entry(cosmos_contract_definition.clone()) + .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(None))) + .clone(); + + // try to read (non-blocking for other readers) + { + let read_guard = entry.read().await; + if let Some(code_id) = *read_guard { + return code_id; + } + } + + // cache miss, acquire write lock for upload + let mut write_guard = entry.write().await; + + // check cache after acquiring write lock, if another thread already uploaded + if let Some(code_id) = *write_guard { + return code_id; + } + + // Upload since not cached + let (chain, cosmos_bytecode) = match cosmos_contract_definition { + CosmosContractDefinition::Trigger(CosmosTriggerDefinition::SimpleContractEvent { + chain, + }) => { + let wasm_path = workspace_path() + .join("examples") + .join("build") + .join("contracts") + .join("cw_wavs_trigger_simple.wasm"); + + if !wasm_path.exists() { + panic!( + "Cosmos contract WASM file not found at: {}", + wasm_path.display() + ); + } + + (chain, tokio::fs::read(&wasm_path).await.unwrap()) + } + CosmosContractDefinition::Submit(CosmosSubmitDefinition::MockServiceHandler { chain }) => { + let wasm_path = workspace_path() + .join("examples") + .join("build") + .join("contracts") + .join("cw_wavs_mock_service_handler.wasm"); + + if !wasm_path.exists() { + panic!( + "Cosmos contract WASM file not found at: {}", + wasm_path.display() + ); + } + + (chain, tokio::fs::read(&wasm_path).await.unwrap()) + } + }; + + tracing::info!( + "Uploading cosmos wasm byte code ({} bytes) to chain {}", + cosmos_bytecode.len(), + chain + ); + + let client = clients.get_cosmos_client(chain).await; + + let (code_id, _) = client + .contract_upload_file(cosmos_bytecode, None) + .await + .unwrap(); + + tracing::info!( + "Successfully uploaded WASM bytecode to chain {}, code_id: {}", + chain, + code_id + ); + + // Cache result and return + *write_guard = Some(code_id); + code_id +} + +/// Simulate a re-org by reverting to a previous block and mining new blocks +pub async fn simulate_anvil_reorg( + evm_client: &EvmSigningClient, + reorg_snapshot: U256, +) -> Result<()> { + // Revert to the specified block using Anvil's revert RPC + evm_client.provider.anvil_revert(reorg_snapshot).await?; + + // Update nonce + if let AnyNonceManager::Fast(fast_nonce_manager) = &evm_client.nonce_manager { + fast_nonce_manager + .set_current_nonce(&evm_client.provider) + .await + .unwrap(); + } + + // Mine new blocks to simulate chain reorganization + evm_client.provider.evm_mine(None).await?; + Ok(()) +} + +pub async fn evm_wait_for_task_to_land( + evm_submit_client: EvmSigningClient, + address: alloy_primitives::Address, + trigger_id: TriggerId, + submit_start_block: u64, + timeout: Duration, +) -> Result { + let submit_client = SimpleEvmSubmitClient::new(evm_submit_client, address); + + tokio::time::timeout(timeout, async move { + loop { + let current_block = submit_client + .evm_client + .provider + .get_block_number() + .await + .map_err(|e| anyhow!("Failed to get block number: {e}"))?; + + if current_block <= submit_start_block { + submit_client.evm_client.provider.evm_mine(None).await?; + } + + if submit_client.trigger_validated(trigger_id).await { + return submit_client + .signed_data(trigger_id) + .await + .map_err(|e| anyhow!("Failed to get signed data: {e}")); + } + + tracing::debug!("Waiting for task response on trigger {}", trigger_id); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .await + .map_err(|_| anyhow::anyhow!("Timeout when waiting for task to land"))? +} + +pub async fn cosmos_wait_for_task_to_land( + cosmos_submit_client: Object, + address: CosmosAddr, + trigger_id: TriggerId, + timeout: Duration, +) -> Result> { + let submit_client = SimpleCosmosSubmitClient::new(cosmos_submit_client, address.into()); + + let trigger_id = trigger_id.u64(); + tokio::time::timeout(timeout, async move { + loop { + if submit_client + .trigger_validated(trigger_id) + .await + .unwrap_or(false) + { + return submit_client + .trigger_message(trigger_id) + .await + .map_err(|e| anyhow!("Failed to get signed data: {e}")); + } + + tracing::debug!("Waiting for task response on trigger {}", trigger_id); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .await + .map_err(|_| anyhow::anyhow!("Timeout when waiting for task to land"))? +} + +/// Helper function to deploy a service for a test +pub async fn change_service_for_test( + service: &mut Service, + change_service: ChangeServiceDefinition, + clients: &Clients, + component_sources: &ComponentSources, + cosmos_code_map: CosmosCodeMap, +) { + match change_service { + ChangeServiceDefinition::Component { + workflow_id, + component: component_definition, + } => { + let component = deploy_component( + component_sources, + &component_definition, + Default::default(), + Default::default(), + ); + let workflow = service + .workflows + .get_mut(&workflow_id) + .expect("Workflow not found in service"); + + workflow.component = component; + } + ChangeServiceDefinition::AddWorkflow { + workflow_id, + workflow, + } => { + let deployed_workflow = deploy_workflow( + &workflow_id, + &workflow, + service.manager.clone(), + clients, + component_sources, + cosmos_code_map, + ) + .await; + + service + .workflows + .insert(workflow_id.clone(), deployed_workflow.workflow); + } + } +} + +pub async fn wait_for_evm_trigger_streams_to_finalize( + client: &HttpClient, + service_manager: Option, +) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + tracing::info!("Getting trigger stream info..."); + let info = client.get_trigger_streams_info().await.unwrap(); + + if info.chains_finalized() { + if let Some(service_manager) = &service_manager { + match service_manager { + ServiceManager::Evm { chain, address } => { + let address = ByteArray::new(address.into_array()); + if info.chains.iter().any(|(key, value)| { + key == chain + && value.active_subscriptions.values().any(|kind| match kind { + DevTriggerStreamSubscriptionKind::Logs { + addresses, + .. + } => addresses.contains(&address), + _ => false, + }) + }) { + break; + } + } + ServiceManager::Cosmos { .. } => { + unreachable!("This is only meant for EVM"); + } + } + } else if info.any_active_subscriptions() { + break; + } + } else { + tracing::warn!("Still waiting for trigger streams to finalize"); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .unwrap(); +} + +pub async fn wait_for_hypercore_streams_to_finalize( + client: &HttpClient, + feed_key: &str, + timeout: Option, +) -> anyhow::Result<()> { + let timeout = timeout.unwrap_or(Duration::from_secs(30)); + let start = std::time::Instant::now(); + let mut poll_interval = Duration::from_millis(100); + + tokio::time::timeout(timeout, async { + loop { + // Retry HTTP request with backoff on failure + match client.get_trigger_streams_info().await { + Ok(info) => { + match info.hypercore.get(feed_key) { + Some(DevHypercoreStreamState::Connected) => { + tracing::info!("Hypercore stream connected for feed_key {}", feed_key); + return Ok(()); + } + Some(DevHypercoreStreamState::Connecting) => { + tracing::info!("Hypercore stream connecting for feed_key {}", feed_key); + } + Some(DevHypercoreStreamState::Waiting) => { + tracing::info!("Hypercore stream waiting for feed_key {}", feed_key); + } + None => { + tracing::info!( + "Hypercore stream not registered yet for feed_key {}", + feed_key + ); + } + } + // Reset poll interval on successful response + poll_interval = Duration::from_millis(100); + } + Err(e) => { + tracing::warn!("HTTP error getting trigger streams info: {}", e); + // Exponential backoff for HTTP errors + poll_interval = (poll_interval * 2).min(Duration::from_secs(1)); + } + } + + tokio::time::sleep(poll_interval).await; + + // Log progress every 5 seconds + if start.elapsed().as_secs().is_multiple_of(5) { + tracing::info!( + "Still waiting for hypercore stream (elapsed: {}s)", + start.elapsed().as_secs() + ); + } + } + }) + .await + .map_err(|_| { + anyhow::anyhow!( + "Timed out waiting for hypercore stream to connect (feed_key: {})", + feed_key + ) + })? +} + +/// Wait for hypercore mesh to form by checking the test client's peer connection count directly. +/// +/// This is used in multi-operator tests to ensure all operators have discovered each other +/// via hyperswarm before proceeding with test execution. +pub async fn wait_for_hypercore_mesh_ready( + hypercore_client: &std::sync::Arc, + expected_peers: usize, + timeout: Duration, +) -> anyhow::Result { + let start = std::time::Instant::now(); + let mut poll_interval = Duration::from_millis(100); + + loop { + let peer_count = hypercore_client.connected_peer_count(); + + if peer_count >= expected_peers { + tracing::info!( + "Hypercore mesh ready: {} connected peers (expected {})", + peer_count, + expected_peers + ); + return Ok(peer_count); + } + + if start.elapsed() > timeout { + anyhow::bail!( + "Timeout waiting for hypercore mesh: {} peers connected, expected {}", + peer_count, + expected_peers + ); + } + + // Exponential backoff with jitter for mesh formation + let wait_time = poll_interval; + poll_interval = std::cmp::min(Duration::from_secs(2), poll_interval * 2); + tokio::time::sleep(wait_time).await; + } +} diff --git a/packages/layer-tests/src/e2e/service_managers.rs b/packages/layer-tests/src/e2e/service_managers.rs index c55eed46c..ce1c6670c 100644 --- a/packages/layer-tests/src/e2e/service_managers.rs +++ b/packages/layer-tests/src/e2e/service_managers.rs @@ -1,520 +1,521 @@ -use std::{collections::HashMap, sync::Arc}; - -use futures::{stream::FuturesUnordered, StreamExt}; -use utils::test_utils::{ - middleware::{ - cosmos::CosmosServiceManager, - evm::{EvmMiddleware, MiddlewareServiceManagerConfig}, - operator::AvsOperator, - }, - mock_service_manager::MockEvmServiceManager, -}; -use wavs_cli::command::deploy_service::DeployService; -use wavs_types::{ - ChainKey, ChainKeyNamespace, Service, ServiceManager, ServiceStatus, SignerResponse, -}; - -use crate::{ - deployment::ServiceDeployment, - e2e::{handles::CosmosMiddlewares, helpers::wait_for_evm_trigger_streams_to_finalize}, -}; - -use crate::e2e::{ - clients::Clients, - components::ComponentSources, - config::Configs, - helpers::create_service_for_test, - test_registry::{CosmosCodeMap, TestRegistry}, -}; - -#[derive(Clone)] -pub struct ServiceManagers { - configs: Arc, - lookup: Arc>, -} - -pub enum AnyServiceManagerInstance { - Evm { - chain: ChainKey, - manager: MockEvmServiceManager, - }, - Cosmos { - chain: ChainKey, - manager: CosmosServiceManager, - }, -} - -impl ServiceManagers { - pub fn new(configs: Configs) -> Self { - Self { - lookup: Arc::new(HashMap::new()), - configs: Arc::new(configs), - } - } -} - -impl ServiceManagers { - pub async fn bootstrap( - &mut self, - registry: &TestRegistry, - clients: &Clients, - evm_middleware: Option, - cosmos_middlewares: CosmosMiddlewares, - ) { - tracing::warn!("WAVS Concurrency: {}", self.configs.wavs_concurrency); - tracing::warn!( - "Middleware Concurrency: {}", - self.configs.middleware_concurrency - ); - tracing::warn!("Bootstrapping service managers..."); - self.deploy_service_managers(registry, clients, evm_middleware, cosmos_middlewares) - .await; - tracing::warn!("Bootstrapping initial service uris..."); - self.set_initial_service_uris(registry, clients).await; - tracing::warn!("Bootstrapping initial services..."); - self.deploy_initial_wavs_services(registry, clients).await; - tracing::warn!("Bootstrapping register operators..."); - self.register_operators(registry, clients).await; - } - - pub fn get_service_manager(&self, test_name: &str) -> ServiceManager { - match self.lookup.get(test_name).unwrap() { - AnyServiceManagerInstance::Evm { chain, manager } => ServiceManager::Evm { - chain: chain.clone(), - address: manager.address(), - }, - AnyServiceManagerInstance::Cosmos { chain, manager } => ServiceManager::Cosmos { - chain: chain.clone(), - address: manager.address.clone(), - }, - } - } - - pub async fn deploy_service_managers( - &mut self, - registry: &TestRegistry, - clients: &Clients, - evm_middleware: Option, - cosmos_middlewares: CosmosMiddlewares, - ) { - let mut lookup = HashMap::new(); - - let mut futures = Vec::new(); - - for test in registry.list_all() { - let chain = test - .service_manager_chain - .clone() - .unwrap_or_else(|| panic!("missing service manager chain for test {}", test.name)); - futures.push({ - let evm_middleware = evm_middleware.clone(); - let cosmos_middlewares = cosmos_middlewares.clone(); - async move { - match chain.namespace.as_str() { - ChainKeyNamespace::EVM => { - let wallet_client = clients.get_evm_client(&chain); - let test_name = test.name.clone(); - let middleware = evm_middleware.clone().unwrap(); - tracing::info!("Deploying service manager for test {}", test_name); - let manager = MockEvmServiceManager::new(middleware, wallet_client) - .await - .unwrap(); - tracing::info!( - "EVM Service manager for test {} is {}", - test_name, - manager.address() - ); - (test_name, AnyServiceManagerInstance::Evm { manager, chain }) - } - ChainKeyNamespace::COSMOS => { - let middleware = cosmos_middlewares.get(&chain).unwrap(); - let manager = middleware.deploy_service_manager().await.unwrap(); - tracing::info!( - "Cosmos Service manager for test {} is {}", - test.name, - manager.address - ); - ( - test.name.clone(), - AnyServiceManagerInstance::Cosmos { manager, chain }, - ) - } - other => panic!("Unsupported chain namespace: {}", other), - } - } - }); - } - - tracing::info!("Deploying {} service managers", futures.len()); - - if self.configs.middleware_concurrency { - let mut futures_unordered = FuturesUnordered::from_iter(futures); - while let Some((test_name, value)) = futures_unordered.next().await { - if lookup.insert(test_name.clone(), value).is_some() { - panic!("Service manager for test {} already exists", test_name); - } - } - } else { - for future in futures { - let (test_name, value) = future.await; - if lookup.insert(test_name.clone(), value).is_some() { - panic!("Service manager for test {} already exists", test_name); - } - } - } - - self.lookup = Arc::new(lookup); - } - - pub async fn set_initial_service_uris(&self, registry: &TestRegistry, clients: &Clients) { - let mut futures = Vec::new(); - - for test in registry.list_all() { - let service_manager = self.get_service_manager(&test.name); - - let service = Service { - name: test.name.to_string(), - workflows: Default::default(), - status: ServiceStatus::Paused, - manager: service_manager, - }; - - // Save the service on WAVS endpoint (just a local test thing, real-world would be IPFS or similar) - let service_url = DeployService::save_service(&clients.cli_ctx, &service) - .await - .unwrap(); - - let service_manager_instance = self.lookup.get(&test.name).unwrap(); - - futures.push(async move { - match service_manager_instance { - AnyServiceManagerInstance::Evm { manager, .. } => { - manager.set_service_uri(service_url).await.unwrap(); - } - AnyServiceManagerInstance::Cosmos { manager, .. } => { - manager.set_service_uri(&service_url).await.unwrap(); - } - } - }); - } - - if self.configs.middleware_concurrency { - futures::future::join_all(futures).await; - } else { - for future in futures { - future.await; - } - } - } - - pub async fn deploy_initial_wavs_services( - &mut self, - registry: &TestRegistry, - clients: &Clients, - ) { - let mut futures = Vec::new(); - - for test in registry.list_all() { - let service_manager = self.get_service_manager(&test.name); - let http_clients = clients.http_clients.clone(); - - futures.push(async move { - tracing::info!("Deploying service {} on all WAVS instances", test.name); - - // Deploy the service on ALL WAVS instances - for (idx, http_client) in http_clients.iter().enumerate() { - tracing::info!("Deploying service {} on WAVS instance {}", test.name, idx); - http_client - .create_service(service_manager.clone(), None) - .await - .unwrap(); - } - }); - } - - if self.configs.wavs_concurrency { - let mut futures_unordered = FuturesUnordered::from_iter(futures); - while (futures_unordered.next().await).is_some() {} - } else { - for future in futures { - future.await; - } - } - } - - pub async fn register_operators(&self, registry: &TestRegistry, clients: &Clients) { - use crate::e2e::config::MULTI_OPERATOR_COUNT; - - let mut futures = Vec::new(); - - for (test_index, test) in registry.list_all().enumerate() { - let service_manager = self.get_service_manager(&test.name); - - // Register operators for all running WAVS instances since any of them - // might execute aggregation and submit. Cap at the number of available - // instances (may be less than MULTI_OPERATOR_COUNT for isolated tests). - let num_operators = std::cmp::min(MULTI_OPERATOR_COUNT, clients.http_clients.len()); - - // Collect all operators for this test - let mut avs_operators = Vec::with_capacity(num_operators); - - for operator_offset in 0..num_operators { - // Reuse existing HTTP client for this WAVS instance - let http_client = &clients.http_clients[operator_offset]; - - let SignerResponse::Secp256k1 { - evm_address: avs_signer_address, - hd_index: wavs_signer_hd_index, - } = http_client - .get_service_signer(service_manager.clone()) - .await - .unwrap(); - - // unique HD index per test and operator to avoid nonce collisions - let operator_hd_index = - (test_index * MULTI_OPERATOR_COUNT + operator_offset) as u32; - let operator_mnemonic = &self.configs.mnemonics.operators[operator_offset]; - let operator_signer = utils::evm_client::signing::make_signer( - operator_mnemonic, - Some(operator_hd_index), - ) - .unwrap(); - let operator_address = operator_signer.address(); - let operator_private_key = const_hex::encode(operator_signer.to_bytes()); - - // Get the signing key that this WAVS instance will use - let signing_signer = utils::evm_client::signing::make_signer( - operator_mnemonic, - Some(wavs_signer_hd_index), - ) - .unwrap(); - let signing_address = signing_signer.address(); - let signing_private_key = const_hex::encode(signing_signer.to_bytes()); - - assert_eq!( - signing_address.to_string().to_lowercase(), - avs_signer_address.to_lowercase(), - "Derived signing address doesn't match WAVS signer address for operator {}", - operator_offset - ); - - let avs_operator = AvsOperator::with_keys( - operator_address, - signing_address, - operator_private_key, - signing_private_key, - ); - - avs_operators.push(avs_operator); - } - - // Calculate required signatures for quorum - // Multi-operator: 2/3 quorum (requires multiple signatures) - // Single-operator: quorum of 1 (any single operator can submit) - let required_to_pass = if test.multi_operator { - ((num_operators as u64) * 2).div_ceil(3) - } else { - 1 - }; - - let service_manager_instance = self.lookup.get(&test.name).unwrap(); - futures.push(async move { - match service_manager_instance { - AnyServiceManagerInstance::Evm { manager, .. } => { - let config = - MiddlewareServiceManagerConfig::new(&avs_operators, required_to_pass); - manager.configure(&config).await.unwrap(); - // Validate that operators are properly registered before proceeding - manager - .validate_operator_registration(&config) - .await - .unwrap(); - } - AnyServiceManagerInstance::Cosmos { manager, .. } => { - // For Cosmos, register each operator individually - for operator in avs_operators { - manager.register_operator(operator.clone()).await.unwrap(); - } - } - } - }); - } - - if self.configs.middleware_concurrency { - let mut futures_unordered = FuturesUnordered::from_iter(futures); - while (futures_unordered.next().await).is_some() {} - } else { - for future in futures { - future.await; - } - } - } - - pub async fn create_real_wavs_services( - &mut self, - registry: &TestRegistry, - clients: &Clients, - component_sources: &ComponentSources, - cosmos_code_map: CosmosCodeMap, - ) -> HashMap { - let mut futures = Vec::new(); - - for test in registry.list_all() { - let service_manager = self.get_service_manager(&test.name); - - futures.push(create_service_for_test( - test, - clients, - component_sources, - service_manager, - cosmos_code_map.clone(), - )); - } - - let mut services = HashMap::new(); - - if self.configs.wavs_concurrency { - let mut futures_unordered = FuturesUnordered::from_iter(futures); - while let Some(deployment_result) = futures_unordered.next().await { - services.insert(deployment_result.service.name.clone(), deployment_result); - } - } else { - for future in futures { - let deployment_result = future.await; - services.insert(deployment_result.service.name.clone(), deployment_result); - } - } - - services - } - - pub async fn update_services(&self, clients: &Clients, services: Vec) { - let mut futures = Vec::new(); - - for service in services { - // Save the service to the primary instance and get the URL for on-chain - let service_url = DeployService::save_service(&clients.cli_ctx, &service) - .await - .unwrap(); - - let service_manager_instance = self.lookup.get(&service.name).unwrap(); - let http_clients = clients.http_clients.clone(); - futures.push(async move { - match service_manager_instance { - AnyServiceManagerInstance::Evm { manager, .. } => { - // wait for the trigger streams to be ready on all instances before we update the service uri - for (idx, http_client) in http_clients.iter().enumerate() { - tracing::info!( - "Waiting for trigger streams on instance {} for service {}", - idx, - service.name - ); - wait_for_evm_trigger_streams_to_finalize( - http_client, - Some(service.manager.clone()), - ) - .await; - } - manager.set_service_uri(service_url).await.unwrap(); - } - AnyServiceManagerInstance::Cosmos { manager, .. } => { - manager.set_service_uri(&service_url).await.unwrap(); - } - } - - // Directly add the service to ALL instances using the dev endpoint - // This bypasses on-chain event detection which may not work reliably - // across multiple WAVS instances in the test environment - for (idx, http_client) in http_clients.iter().enumerate() { - tracing::info!( - "Directly adding service to instance {} for service {}", - idx, - service.name - ); - // Ignore "already registered" errors - the instance may have - // already received the service via on-chain event detection - match http_client.dev_add_service_direct(&service).await { - Ok(_) => { - tracing::info!( - "Service directly added to instance {} for service {}", - idx, - service.name - ); - } - Err(e) if e.to_string().contains("already registered") => { - tracing::info!( - "Service already registered on instance {} for service {} (via on-chain detection)", - idx, - service.name - ); - } - Err(e) => { - panic!( - "Failed to add service to instance {} for service {}: {}", - idx, service.name, e - ); - } - } - } - - // Wait for service update on all WAVS instances (should be instant now) - for (idx, http_client) in http_clients.iter().enumerate() { - tracing::info!( - "Waiting for service update on instance {} for service {}", - idx, - service.name - ); - http_client - .wait_for_service_update(&service, None) - .await - .unwrap(); - tracing::info!( - "Service update complete on instance {} for service {}", - idx, - service.name - ); - } - - // Debug: Log trigger streams status - let http_client = clients - .http_clients - .first() - .expect("Expected at least one WAVS HTTP client"); - match http_client.get_trigger_streams_info().await { - Ok(streams) => { - tracing::info!( - "Trigger streams finalized={}, chains={:?}, hypercore_feeds={:?}", - streams.finalized(), - streams.chains, - streams.hypercore - ); - } - Err(e) => { - tracing::warn!("Failed to get trigger streams info: {:?}", e); - } - } - - // doesn't hurt to wait again for rpcs at least in case trigger contract changed - if let AnyServiceManagerInstance::Evm { .. } = service_manager_instance { - for (idx, http_client) in http_clients.iter().enumerate() { - tracing::info!( - "Final trigger stream wait on instance {} for service {}", - idx, - service.name - ); - wait_for_evm_trigger_streams_to_finalize(http_client, None).await; - } - } - }); - } - - if self.configs.middleware_concurrency { - let mut futures_unordered = FuturesUnordered::from_iter(futures); - while (futures_unordered.next().await).is_some() {} - } else { - for future in futures { - future.await; - } - } - } -} +use std::{collections::HashMap, sync::Arc}; + +use futures::{stream::FuturesUnordered, StreamExt}; +use utils::test_utils::{ + middleware::{ + cosmos::CosmosServiceManager, + evm::{EvmMiddleware, MiddlewareServiceManagerConfig}, + operator::AvsOperator, + }, + mock_service_manager::MockEvmServiceManager, +}; +use wavs_cli::command::deploy_service::DeployService; +use wavs_types::{ + ChainKey, ChainKeyNamespace, Service, ServiceManager, ServiceStatus, SignerResponse, +}; + +use crate::{ + deployment::ServiceDeployment, + e2e::{handles::CosmosMiddlewares, helpers::wait_for_evm_trigger_streams_to_finalize}, +}; + +use crate::e2e::{ + clients::Clients, + components::ComponentSources, + config::Configs, + helpers::create_service_for_test, + test_registry::{CosmosCodeMap, TestRegistry}, +}; + +#[derive(Clone)] +pub struct ServiceManagers { + configs: Arc, + lookup: Arc>, +} + +pub enum AnyServiceManagerInstance { + Evm { + chain: ChainKey, + manager: MockEvmServiceManager, + }, + Cosmos { + chain: ChainKey, + manager: CosmosServiceManager, + }, +} + +impl ServiceManagers { + pub fn new(configs: Configs) -> Self { + Self { + lookup: Arc::new(HashMap::new()), + configs: Arc::new(configs), + } + } +} + +impl ServiceManagers { + pub async fn bootstrap( + &mut self, + registry: &TestRegistry, + clients: &Clients, + evm_middleware: Option, + cosmos_middlewares: CosmosMiddlewares, + ) { + tracing::warn!("WAVS Concurrency: {}", self.configs.wavs_concurrency); + tracing::warn!( + "Middleware Concurrency: {}", + self.configs.middleware_concurrency + ); + tracing::warn!("Bootstrapping service managers..."); + self.deploy_service_managers(registry, clients, evm_middleware, cosmos_middlewares) + .await; + tracing::warn!("Bootstrapping initial service uris..."); + self.set_initial_service_uris(registry, clients).await; + tracing::warn!("Bootstrapping initial services..."); + self.deploy_initial_wavs_services(registry, clients).await; + tracing::warn!("Bootstrapping register operators..."); + self.register_operators(registry, clients).await; + } + + pub fn get_service_manager(&self, test_name: &str) -> ServiceManager { + match self.lookup.get(test_name).unwrap() { + AnyServiceManagerInstance::Evm { chain, manager } => ServiceManager::Evm { + chain: chain.clone(), + address: manager.address(), + }, + AnyServiceManagerInstance::Cosmos { chain, manager } => ServiceManager::Cosmos { + chain: chain.clone(), + address: manager.address.clone(), + }, + } + } + + pub async fn deploy_service_managers( + &mut self, + registry: &TestRegistry, + clients: &Clients, + evm_middleware: Option, + cosmos_middlewares: CosmosMiddlewares, + ) { + let mut lookup = HashMap::new(); + + let mut futures = Vec::new(); + + for test in registry.list_all() { + let chain = test + .service_manager_chain + .clone() + .unwrap_or_else(|| panic!("missing service manager chain for test {}", test.name)); + futures.push({ + let evm_middleware = evm_middleware.clone(); + let cosmos_middlewares = cosmos_middlewares.clone(); + async move { + match chain.namespace.as_str() { + ChainKeyNamespace::EVM => { + let wallet_client = clients.get_evm_client(&chain); + let test_name = test.name.clone(); + let middleware = evm_middleware.clone().unwrap(); + tracing::info!("Deploying service manager for test {}", test_name); + let manager = MockEvmServiceManager::new(middleware, wallet_client) + .await + .unwrap(); + tracing::info!( + "EVM Service manager for test {} is {}", + test_name, + manager.address() + ); + (test_name, AnyServiceManagerInstance::Evm { manager, chain }) + } + ChainKeyNamespace::COSMOS => { + let middleware = cosmos_middlewares.get(&chain).unwrap(); + let manager = middleware.deploy_service_manager().await.unwrap(); + tracing::info!( + "Cosmos Service manager for test {} is {}", + test.name, + manager.address + ); + ( + test.name.clone(), + AnyServiceManagerInstance::Cosmos { manager, chain }, + ) + } + other => panic!("Unsupported chain namespace: {}", other), + } + } + }); + } + + tracing::info!("Deploying {} service managers", futures.len()); + + if self.configs.middleware_concurrency { + let mut futures_unordered = FuturesUnordered::from_iter(futures); + while let Some((test_name, value)) = futures_unordered.next().await { + if lookup.insert(test_name.clone(), value).is_some() { + panic!("Service manager for test {} already exists", test_name); + } + } + } else { + for future in futures { + let (test_name, value) = future.await; + if lookup.insert(test_name.clone(), value).is_some() { + panic!("Service manager for test {} already exists", test_name); + } + } + } + + self.lookup = Arc::new(lookup); + } + + pub async fn set_initial_service_uris(&self, registry: &TestRegistry, clients: &Clients) { + let mut futures = Vec::new(); + + for test in registry.list_all() { + let service_manager = self.get_service_manager(&test.name); + + let service = Service { + name: test.name.to_string(), + workflows: Default::default(), + status: ServiceStatus::Paused, + manager: service_manager, + }; + + // Save the service on WAVS endpoint (just a local test thing, real-world would be IPFS or similar) + let service_url = DeployService::save_service(&clients.cli_ctx, &service) + .await + .unwrap(); + + let service_manager_instance = self.lookup.get(&test.name).unwrap(); + + futures.push(async move { + match service_manager_instance { + AnyServiceManagerInstance::Evm { manager, .. } => { + manager.set_service_uri(service_url).await.unwrap(); + } + AnyServiceManagerInstance::Cosmos { manager, .. } => { + manager.set_service_uri(&service_url).await.unwrap(); + } + } + }); + } + + if self.configs.middleware_concurrency { + futures::future::join_all(futures).await; + } else { + for future in futures { + future.await; + } + } + } + + pub async fn deploy_initial_wavs_services( + &mut self, + registry: &TestRegistry, + clients: &Clients, + ) { + let mut futures = Vec::new(); + + for test in registry.list_all() { + let service_manager = self.get_service_manager(&test.name); + let http_clients = clients.http_clients.clone(); + + futures.push(async move { + tracing::info!("Deploying service {} on all WAVS instances", test.name); + + // Deploy the service on ALL WAVS instances + for (idx, http_client) in http_clients.iter().enumerate() { + tracing::info!("Deploying service {} on WAVS instance {}", test.name, idx); + http_client + .create_service(service_manager.clone(), None) + .await + .unwrap(); + } + }); + } + + if self.configs.wavs_concurrency { + let mut futures_unordered = FuturesUnordered::from_iter(futures); + while (futures_unordered.next().await).is_some() {} + } else { + for future in futures { + future.await; + } + } + } + + pub async fn register_operators(&self, registry: &TestRegistry, clients: &Clients) { + use crate::e2e::config::MULTI_OPERATOR_COUNT; + + let mut futures = Vec::new(); + + for (test_index, test) in registry.list_all().enumerate() { + let service_manager = self.get_service_manager(&test.name); + + // Register operators for all running WAVS instances since any of them + // might execute aggregation and submit. Cap at the number of available + // instances (may be less than MULTI_OPERATOR_COUNT for isolated tests). + let num_operators = std::cmp::min(MULTI_OPERATOR_COUNT, clients.http_clients.len()); + + // Collect all operators for this test + let mut avs_operators = Vec::with_capacity(num_operators); + + for operator_offset in 0..num_operators { + // Reuse existing HTTP client for this WAVS instance + let http_client = &clients.http_clients[operator_offset]; + + let SignerResponse::Secp256k1 { + evm_address: avs_signer_address, + hd_index: wavs_signer_hd_index, + } = http_client + .get_service_signer(service_manager.clone()) + .await + .unwrap(); + + // unique HD index per test and operator to avoid nonce collisions + let operator_hd_index = + (test_index * MULTI_OPERATOR_COUNT + operator_offset) as u32; + let operator_mnemonic = &self.configs.mnemonics.operators[operator_offset]; + let operator_signer = utils::evm_client::signing::make_signer( + operator_mnemonic, + Some(operator_hd_index), + ) + .unwrap(); + let operator_address = operator_signer.address(); + let operator_private_key = const_hex::encode(operator_signer.to_bytes()); + + // Get the signing key that this WAVS instance will use + let signing_signer = utils::evm_client::signing::make_signer( + operator_mnemonic, + Some(wavs_signer_hd_index), + ) + .unwrap(); + let signing_address = signing_signer.address(); + let signing_private_key = const_hex::encode(signing_signer.to_bytes()); + + assert_eq!( + signing_address.to_string().to_lowercase(), + avs_signer_address.to_lowercase(), + "Derived signing address doesn't match WAVS signer address for operator {}", + operator_offset + ); + + let avs_operator = AvsOperator::with_keys( + operator_address, + signing_address, + operator_private_key, + signing_private_key, + ); + + avs_operators.push(avs_operator); + } + + // Calculate required signatures for quorum + // Multi-operator: 2/3 quorum (requires multiple signatures) + // Single-operator: quorum of 1 (any single operator can submit) + let required_to_pass = if test.multi_operator { + ((num_operators as u64) * 2).div_ceil(3) + } else { + 1 + }; + + let service_manager_instance = self.lookup.get(&test.name).unwrap(); + futures.push(async move { + match service_manager_instance { + AnyServiceManagerInstance::Evm { manager, .. } => { + let config = + MiddlewareServiceManagerConfig::new(&avs_operators, required_to_pass); + manager.configure(&config).await.unwrap(); + // Validate that operators are properly registered before proceeding + manager + .validate_operator_registration(&config) + .await + .unwrap(); + } + AnyServiceManagerInstance::Cosmos { manager, .. } => { + // For Cosmos, register each operator individually + for operator in avs_operators { + manager.register_operator(operator.clone()).await.unwrap(); + } + } + } + }); + } + + if self.configs.middleware_concurrency { + let mut futures_unordered = FuturesUnordered::from_iter(futures); + while (futures_unordered.next().await).is_some() {} + } else { + for future in futures { + future.await; + } + } + } + + pub async fn create_real_wavs_services( + &mut self, + registry: &TestRegistry, + clients: &Clients, + component_sources: &ComponentSources, + cosmos_code_map: CosmosCodeMap, + ) -> HashMap { + let mut futures = Vec::new(); + + for test in registry.list_all() { + let service_manager = self.get_service_manager(&test.name); + + futures.push(create_service_for_test( + test, + clients, + component_sources, + service_manager, + cosmos_code_map.clone(), + )); + } + + let mut services = HashMap::new(); + + if self.configs.wavs_concurrency { + let mut futures_unordered = FuturesUnordered::from_iter(futures); + while let Some(deployment_result) = futures_unordered.next().await { + services.insert(deployment_result.service.name.clone(), deployment_result); + } + } else { + for future in futures { + let deployment_result = future.await; + services.insert(deployment_result.service.name.clone(), deployment_result); + } + } + + services + } + + pub async fn update_services(&self, clients: &Clients, services: Vec) { + let mut futures = Vec::new(); + + for service in services { + // Save the service to the primary instance and get the URL for on-chain + let service_url = DeployService::save_service(&clients.cli_ctx, &service) + .await + .unwrap(); + + let service_manager_instance = self.lookup.get(&service.name).unwrap(); + let http_clients = clients.http_clients.clone(); + futures.push(async move { + match service_manager_instance { + AnyServiceManagerInstance::Evm { manager, .. } => { + // wait for the trigger streams to be ready on all instances before we update the service uri + for (idx, http_client) in http_clients.iter().enumerate() { + tracing::info!( + "Waiting for trigger streams on instance {} for service {}", + idx, + service.name + ); + wait_for_evm_trigger_streams_to_finalize( + http_client, + Some(service.manager.clone()), + ) + .await; + } + manager.set_service_uri(service_url).await.unwrap(); + } + AnyServiceManagerInstance::Cosmos { manager, .. } => { + manager.set_service_uri(&service_url).await.unwrap(); + } + } + + // Directly add the service to ALL instances using the dev endpoint + // This bypasses on-chain event detection which may not work reliably + // across multiple WAVS instances in the test environment + for (idx, http_client) in http_clients.iter().enumerate() { + tracing::info!( + "Directly adding service to instance {} for service {}", + idx, + service.name + ); + // Ignore "already registered" errors - the instance may have + // already received the service via on-chain event detection + match http_client.dev_add_service_direct(&service).await { + Ok(_) => { + tracing::info!( + "Service directly added to instance {} for service {}", + idx, + service.name + ); + } + Err(e) if e.to_string().contains("already registered") => { + tracing::info!( + "Service already registered on instance {} for service {} (via on-chain detection)", + idx, + service.name + ); + } + Err(e) => { + panic!( + "Failed to add service to instance {} for service {}: {}", + idx, service.name, e + ); + } + } + } + + // Wait for service update on all WAVS instances (should be instant now) + for (idx, http_client) in http_clients.iter().enumerate() { + tracing::info!( + "Waiting for service update on instance {} for service {}", + idx, + service.name + ); + http_client + .wait_for_service_update(&service, None) + .await + .unwrap(); + tracing::info!( + "Service update complete on instance {} for service {}", + idx, + service.name + ); + } + + // Debug: Log trigger streams status + let http_client = clients + .http_clients + .first() + .expect("Expected at least one WAVS HTTP client"); + match http_client.get_trigger_streams_info().await { + Ok(streams) => { + tracing::info!( + "Trigger streams chains_finalized={}, hypercore_finalized={}, chains={:?}, hypercore_feeds={:?}", + streams.chains_finalized(), + streams.hypercore_finalized(), + streams.chains, + streams.hypercore + ); + } + Err(e) => { + tracing::warn!("Failed to get trigger streams info: {:?}", e); + } + } + + // doesn't hurt to wait again for rpcs at least in case trigger contract changed + if let AnyServiceManagerInstance::Evm { .. } = service_manager_instance { + for (idx, http_client) in http_clients.iter().enumerate() { + tracing::info!( + "Final trigger stream wait on instance {} for service {}", + idx, + service.name + ); + wait_for_evm_trigger_streams_to_finalize(http_client, None).await; + } + } + }); + } + + if self.configs.middleware_concurrency { + let mut futures_unordered = FuturesUnordered::from_iter(futures); + while (futures_unordered.next().await).is_some() {} + } else { + for future in futures { + future.await; + } + } + } +} diff --git a/packages/types/src/http.rs b/packages/types/src/http.rs index 80747a94d..3814a5e62 100644 --- a/packages/types/src/http.rs +++ b/packages/types/src/http.rs @@ -1,146 +1,156 @@ -use std::collections::HashMap; - -use super::Service; -use crate::{ - AnyChainConfig, ByteArray, ChainKey, ComponentDigest, ServiceDigest, ServiceId, ServiceManager, - Trigger, TriggerData, WorkflowId, -}; -use serde::{Deserialize, Serialize}; -use utoipa::ToSchema; - -#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] -#[serde(rename_all = "snake_case")] -pub enum SignerResponse { - Secp256k1 { - /// The derivation index used to create this key from the mnemonic - hd_index: u32, - /// The evm-style address ("0x" prefixed hex string) derived from the key - evm_address: String, - }, -} - -#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] -pub struct AddServiceRequest { - pub service_manager: ServiceManager, -} - -#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] -pub struct GetSignerRequest { - pub service_manager: ServiceManager, -} - -#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] -pub struct AddChainRequest { - pub chain: ChainKey, - pub config: AnyChainConfig, -} - -#[derive(Serialize, Deserialize, ToSchema)] -pub struct DeleteServicesRequest { - pub service_managers: Vec, -} - -#[derive(Serialize, Deserialize, ToSchema)] -pub struct ListServicesResponse { - pub services: Vec, - pub service_ids: Vec, - pub component_digests: Vec, -} - -#[derive(Serialize, Deserialize, ToSchema)] -pub struct UploadComponentResponse { - pub digest: ComponentDigest, -} - -#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] -pub struct SaveServiceResponse { - pub hash: ServiceDigest, -} - -#[derive(Serialize, Deserialize, Debug, ToSchema)] -pub struct SimulatedTriggerRequest { - pub service_id: ServiceId, - pub workflow_id: WorkflowId, - pub trigger: Trigger, - #[schema(value_type = Object)] - pub data: TriggerData, - #[serde(default = "default_simulated_trigger_count")] - pub count: usize, - pub wait_for_completion: bool, -} - -fn default_simulated_trigger_count() -> usize { - 1 -} - -#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] -pub struct DevTriggerStreamsInfo { - pub chains: HashMap, - #[serde(default)] - pub hypercore: HashMap, -} - -impl DevTriggerStreamsInfo { - pub fn finalized(&self) -> bool { - self.chains.values().all(|info| { - !info.any_active_rpcs_in_flight && info.is_connected && info.current_endpoint.is_some() - }) && self - .hypercore - .values() - .all(|info| matches!(info, DevHypercoreStreamState::Connected)) - } - - pub fn any_active_subscriptions(&self) -> bool { - self.chains - .values() - .any(|info| !info.active_subscriptions.is_empty()) - } -} - -#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema, Clone, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum DevHypercoreStreamState { - Waiting, - Connecting, - Connected, -} - -#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] -pub struct DevTriggerStreamInfo { - pub current_endpoint: Option, - pub is_connected: bool, - pub any_active_rpcs_in_flight: bool, - pub active_subscriptions: HashMap, -} - -#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] -pub enum DevTriggerStreamSubscriptionKind { - NewHeads, - Logs { - addresses: Vec>, - topics: Vec>, - }, - NewPendingTransactions, -} - -/// P2P network status for monitoring and readiness checks -#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)] -pub struct P2pStatus { - /// Whether P2P networking is enabled - pub enabled: bool, - /// Local peer ID - pub local_peer_id: Option, - /// Listen addresses (multiaddrs with peer ID appended, e.g. "/ip4/0.0.0.0/tcp/9000/p2p/12D3KooW...") - pub listen_addresses: Vec, - /// External addresses discovered via AutoNAT/Identify (preferred for NAT traversal) - /// These are addresses that peers outside NAT can use to reach us. - pub external_addresses: Vec, - /// Number of connected peers - pub connected_peers: usize, - /// List of connected peer IDs - pub peer_ids: Vec, - /// Topics we're subscribed to - pub subscribed_topics: Vec, - /// Number of peers subscribed to our topics (topic -> peer count) - pub topic_peer_counts: HashMap, -} +use std::collections::HashMap; + +use super::Service; +use crate::{ + AnyChainConfig, ByteArray, ChainKey, ComponentDigest, ServiceDigest, ServiceId, ServiceManager, + Trigger, TriggerData, WorkflowId, +}; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum SignerResponse { + Secp256k1 { + /// The derivation index used to create this key from the mnemonic + hd_index: u32, + /// The evm-style address ("0x" prefixed hex string) derived from the key + evm_address: String, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct AddServiceRequest { + pub service_manager: ServiceManager, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct GetSignerRequest { + pub service_manager: ServiceManager, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct AddChainRequest { + pub chain: ChainKey, + pub config: AnyChainConfig, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct DeleteServicesRequest { + pub service_managers: Vec, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct ListServicesResponse { + pub services: Vec, + pub service_ids: Vec, + pub component_digests: Vec, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct UploadComponentResponse { + pub digest: ComponentDigest, +} + +#[derive(Serialize, Deserialize, Clone, Debug, ToSchema)] +pub struct SaveServiceResponse { + pub hash: ServiceDigest, +} + +#[derive(Serialize, Deserialize, Debug, ToSchema)] +pub struct SimulatedTriggerRequest { + pub service_id: ServiceId, + pub workflow_id: WorkflowId, + pub trigger: Trigger, + #[schema(value_type = Object)] + pub data: TriggerData, + #[serde(default = "default_simulated_trigger_count")] + pub count: usize, + pub wait_for_completion: bool, +} + +fn default_simulated_trigger_count() -> usize { + 1 +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct DevTriggerStreamsInfo { + pub chains: HashMap, + #[serde(default)] + pub hypercore: HashMap, +} + +impl DevTriggerStreamsInfo { + /// Returns true when all chain streams (EVM, Cosmos) are connected and ready. + pub fn chains_finalized(&self) -> bool { + self.chains.values().all(|info| { + !info.any_active_rpcs_in_flight && info.is_connected && info.current_endpoint.is_some() + }) + } + + /// Returns true when all hypercore feeds have connected to at least one peer. + pub fn hypercore_finalized(&self) -> bool { + self.hypercore + .values() + .all(|info| matches!(info, DevHypercoreStreamState::Connected)) + } + + /// Returns true when all trigger streams (chains + hypercore) are fully ready. + pub fn finalized(&self) -> bool { + self.chains_finalized() && self.hypercore_finalized() + } + + pub fn any_active_subscriptions(&self) -> bool { + self.chains + .values() + .any(|info| !info.active_subscriptions.is_empty()) + } +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema, Clone, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DevHypercoreStreamState { + Waiting, + Connecting, + Connected, +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub struct DevTriggerStreamInfo { + pub current_endpoint: Option, + pub is_connected: bool, + pub any_active_rpcs_in_flight: bool, + pub active_subscriptions: HashMap, +} + +#[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] +pub enum DevTriggerStreamSubscriptionKind { + NewHeads, + Logs { + addresses: Vec>, + topics: Vec>, + }, + NewPendingTransactions, +} + +/// P2P network status for monitoring and readiness checks +#[derive(Debug, Clone, Default, Serialize, Deserialize, utoipa::ToSchema)] +pub struct P2pStatus { + /// Whether P2P networking is enabled + pub enabled: bool, + /// Local peer ID + pub local_peer_id: Option, + /// Listen addresses (multiaddrs with peer ID appended, e.g. "/ip4/0.0.0.0/tcp/9000/p2p/12D3KooW...") + pub listen_addresses: Vec, + /// External addresses discovered via AutoNAT/Identify (preferred for NAT traversal) + /// These are addresses that peers outside NAT can use to reach us. + pub external_addresses: Vec, + /// Number of connected peers + pub connected_peers: usize, + /// List of connected peer IDs + pub peer_ids: Vec, + /// Topics we're subscribed to + pub subscribed_topics: Vec, + /// Number of peers subscribed to our topics (topic -> peer count) + pub topic_peer_counts: HashMap, +} From 13d088f485dd856ef4320cd89732f88d5b2eef8e Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 20 Feb 2026 13:09:54 -0600 Subject: [PATCH 05/10] Wait for hypercore streams and mesh concurrently with longer timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sequential per-instance stream readiness checks (30s each × 3 instances) burned through the timeout budget before DHT discovery could complete. Now all instance checks and the mesh readiness check run in parallel with a shared 60s timeout. Also moved feed key verification before the connectivity waits to fail fast on misconfiguration. Co-Authored-By: Claude Opus 4.6 --- packages/layer-tests/src/e2e/runner.rs | 1622 ++++++++++++------------ 1 file changed, 821 insertions(+), 801 deletions(-) diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index 43521a3f6..3490f3df3 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -1,801 +1,821 @@ -// src/e2e/test_runner.rs - -use crate::deployment::ServiceDeployment; -use crate::e2e::config::Configs; -use crate::example_evm_client::example_trigger::ISimpleTrigger::TriggerInfo; -use crate::example_evm_client::example_trigger::NewTrigger; -use alloy_primitives::U256; -use alloy_provider::ext::AnvilApi; -use alloy_provider::Provider; -use alloy_sol_types::SolType; -use anyhow::{anyhow, bail, Context}; -use futures::{stream::FuturesUnordered, StreamExt}; -use ordermap::OrderMap; -use std::collections::HashMap; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use utils::alloy_helpers::SolidityEventFinder; -use wavs_types::{ - AtProtoAction, ChainKeyNamespace, SimulatedTriggerRequest, Submit, Trigger, TriggerData, - Workflow, WorkflowId, -}; - -use crate::e2e::handles::hypercore::HypercoreClients; -use crate::e2e::helpers::wait_for_hypercore_streams_to_finalize; -use crate::e2e::helpers::{ - change_service_for_test, cosmos_wait_for_task_to_land, wait_for_hypercore_mesh_ready, -}; -use crate::e2e::report::TestReport; -use crate::e2e::service_managers::ServiceManagers; -use crate::e2e::test_definition::{ - AggregatorDefinition, ChangeServiceDefinition, SubmitDefinition, -}; -use crate::e2e::test_registry::CosmosCodeMap; -use crate::{ - e2e::{ - clients::Clients, components::ComponentSources, test_definition::TestDefinition, - test_registry::TestRegistry, - }, - example_cosmos_client::SimpleCosmosTriggerClient, - example_evm_client::{LogSpamClient, SimpleEvmTriggerClient, TriggerId}, -}; -use serde_json::json; - -use super::helpers::{evm_wait_for_task_to_land, simulate_anvil_reorg}; -use super::test_definition::WorkflowDefinition; - -/// Simplified test runner that leverages services directly attached to test definitions -pub struct Runner { - configs: Arc, - clients: Arc, - registry: Arc, - component_sources: Arc, - hypercore_clients: HypercoreClients, - service_managers: ServiceManagers, - cosmos_code_map: CosmosCodeMap, - report: TestReport, -} - -/// Extract service handler address from an aggregator submit configuration -fn extract_aggregator_service_handler(submit: &Submit) -> Option { - match submit { - Submit::Aggregator { component, .. } => { - component - .config - .get("service_handler") - .and_then(|addr_str| { - match layer_climb::prelude::CosmosAddr::new_str(addr_str, None) { - Ok(cosmos_addr) => Some(layer_climb::prelude::Address::Cosmos(cosmos_addr)), - Err(_) => layer_climb::prelude::EvmAddr::new_str(addr_str) - .ok() - .map(layer_climb::prelude::Address::from), - } - }) - } - _ => None, - } -} - -impl Runner { - #[allow(clippy::too_many_arguments)] - pub fn new( - configs: Configs, - clients: Clients, - registry: TestRegistry, - component_sources: ComponentSources, - hypercore_clients: HypercoreClients, - service_managers: ServiceManagers, - cosmos_code_map: CosmosCodeMap, - report: TestReport, - ) -> Self { - Self { - configs: Arc::new(configs), - clients: Arc::new(clients), - registry: Arc::new(registry), - component_sources: Arc::new(component_sources), - hypercore_clients, - service_managers, - cosmos_code_map, - report, - } - } - - /// Run all tests in the registry - pub async fn run_tests(&mut self, mut all_services: HashMap) { - let test_groups = self.registry.list_all_grouped(self.configs.grouping); - - for (group, mut group_tests) in test_groups { - let services = group_tests - .iter() - .map(|test| all_services.get(&test.name).cloned().unwrap().service) - .collect::>(); - - // This essentially deploys the services for the group - // since it updates the services to "Active" - // which is detected by wavs - self.service_managers - .update_services(&self.clients, services) - .await; - - // However, we have some tests which demonstrate more specific service changes - // and so we need to re-update those before we can proceed - // - // First we just deploy the service changes (contracts, components, etc.) - let mut futures = FuturesUnordered::new(); - for test in group_tests.iter() { - if let Some(change_service) = test.change_service.clone() { - let service = all_services.get(&test.name).cloned().unwrap().service; - let clients = self.clients.clone(); - let component_sources = self.component_sources.clone(); - let cosmos_code_map = self.cosmos_code_map.clone(); - futures.push(async move { - let mut service = service; - change_service_for_test( - &mut service, - change_service.clone(), - &clients, - &component_sources, - cosmos_code_map, - ) - .await; - (service, change_service) - }); - } - } - - // Then we need to deploy the update to service managers - if futures.is_empty() { - tracing::info!("No changes to services in group {:?}", group); - } else { - tracing::warn!("Running service changes for group {:?}", group); - let mut services_to_change = Vec::new(); - while let Some((service, change_service)) = futures.next().await { - // update our local copy of the service and handle changes - let service_deployment = all_services - .get_mut(&service.name) - .expect("Service should exist in all_services"); - - service_deployment.service = service.clone(); - - // and the definition so that tests know what to look for - match change_service { - ChangeServiceDefinition::AddWorkflow { - workflow_id, - workflow, - } => { - // When a workflow is added, it includes a new submission contract - // Extract it from the service's workflow that was just added - let submission_address = service_deployment - .service - .workflows - .get(&workflow_id) - .and_then(|workflow| { - extract_aggregator_service_handler(&workflow.submit) - }); - - if let Some(address) = submission_address { - service_deployment - .submission_handlers - .insert(workflow_id.clone(), address); - } - - group_tests - .iter_mut() - .find(|test| test.name == service.name) - .unwrap() - .workflows - .insert(workflow_id.clone(), workflow); - } - ChangeServiceDefinition::Component { - workflow_id, - component, - } => { - group_tests - .iter_mut() - .find(|test| test.name == service.name) - .unwrap() - .workflows - .get_mut(&workflow_id) - .unwrap() - .component = component; - } - } - - services_to_change.push(service); - } - - self.service_managers - .update_services(&self.clients, services_to_change) - .await; - } - - // Create hypercore clients AFTER deploying services so WAVS is already - // doing DHT lookups when the test client announces. This avoids the stale-DHT - // problem where the client announces 10+ seconds before WAVS starts looking. - self.hypercore_clients - .create_clients() - .await - .expect("Failed to create hypercore clients"); - - // All services are now deployed and ready for the tests - // From here on in we're strictly testing the trigger->execute->aggregate->submit flow - - tracing::info!("Running group {:?} with {} tests", group, group_tests.len()); - let mut futures = FuturesUnordered::new(); - let hypercore_clients = &self.hypercore_clients; - - for test in group_tests { - let clients = self.clients.clone(); - let component_sources = self.component_sources.clone(); - let test = test.clone(); - let report = self.report.clone(); - let service = all_services.get(&test.name).cloned().unwrap(); - futures.push(async move { - Self::execute_test( - &test, - service, - clients, - component_sources, - hypercore_clients, - report, - ) - .await - }); - } - - while (futures.next().await).is_some() {} - } - } - - // Execute a single test with timings - async fn execute_test( - test: &TestDefinition, - service_deployment: ServiceDeployment, - clients: Arc, - component_sources: Arc, - hypercore_clients: &HypercoreClients, - report: TestReport, - ) { - report.start_test(test.name.clone()); - - run_test( - test, - service_deployment, - &clients, - &component_sources, - hypercore_clients, - ) - .await - .context(test.name.clone()) - .unwrap(); - - report.end_test(test.name.clone()); - } -} - -/// Run a single test -async fn run_test( - test: &TestDefinition, - service_deployment: ServiceDeployment, - clients: &Clients, - component_sources: &ComponentSources, - hypercore_clients: &HypercoreClients, -) -> anyhow::Result<()> { - // For multi-operator tests, wait for P2P mesh to form before triggering - if test.multi_operator && clients.http_clients.len() > 1 { - let expected_peers = clients.http_clients.len() - 1; - tracing::info!( - "Multi-operator test: waiting for P2P mesh formation ({} expected peers)", - expected_peers - ); - - // Wait for all operators to have connected to peers - for (idx, http_client) in clients.http_clients.iter().enumerate() { - let status = http_client - .wait_for_p2p_ready(expected_peers, Some(Duration::from_secs(30))) - .await - .map_err(|e| { - anyhow!( - "Operator {} P2P readiness check failed: {}. \ - Multi-operator tests require P2P mesh to be ready.", - idx, - e - ) - })?; - tracing::info!( - "Operator {} P2P ready: {} connected peers", - idx, - status.connected_peers - ); - } - } - - // Group workflows by trigger to handle multi-triggers - let mut trigger_groups: OrderMap<&Trigger, Vec<(&WorkflowId, &Workflow)>> = OrderMap::new(); - - for (workflow_id, workflow) in service_deployment.service.workflows.iter() { - trigger_groups - .entry(&workflow.trigger) - .or_default() - .push((workflow_id, workflow)); - } - - // Process each unique trigger once, then validate all associated workflows - for (trigger, workflows_group) in trigger_groups { - // Use the first workflow to execute the trigger - let (first_workflow_id, _) = workflows_group[0]; - - // Get the workflow data safely - let first_workflow = test - .workflows - .get(first_workflow_id) - .ok_or(anyhow!("Could not get workflow: {}", first_workflow_id))?; - - // Convert input data to bytes safely - let input_bytes = first_workflow.input_data.to_bytes(); - - // Execute the trigger once - let mut reorg_snapshot: Option = None; - let trigger_ids = match trigger { - Trigger::EvmContractEvent { - chain, - address, - event_hash: _, - } => { - let evm_client = clients.get_evm_client(chain); - let client = SimpleEvmTriggerClient::new(evm_client.clone(), *address); - - if first_workflow.expects_reorg() { - reorg_snapshot = Some(evm_client.provider.anvil_snapshot().await?); - } - let input = input_bytes.clone().expect("EVM triggers require an input"); - - let spam_client = if first_workflow.trigger_execution.log_spam_count > 0 { - let address = super::helpers::deploy_log_spam_contract(clients, chain).await?; - let client = LogSpamClient::new(evm_client.clone(), address); - Some(client) - } else { - None - }; - - #[derive(Clone, Copy, Debug)] - enum TxKind { - Trigger, - Spam, - } - - let mut pending: Vec<(TxKind, alloy_primitives::TxHash)> = Vec::new(); - - let pending_trigger = client - .contract - .addTrigger(input.clone().into()) - .send() - .await?; - pending.push((TxKind::Trigger, *pending_trigger.tx_hash())); - - if let Some(spam_client) = &spam_client { - let spam_count = first_workflow.trigger_execution.log_spam_count as u64; - tracing::info!( - "Emitting {} bulk spam logs using LogSpam contract", - spam_count - ); - - // Use bulk emission to spam N logs in a single transaction - let spam_hash = spam_client.emit_spam(0, spam_count).await?; - - tracing::info!("Bulk spam transaction sent: {:?}", spam_hash); - pending.push((TxKind::Spam, spam_hash)); - } - - let start = Instant::now(); - let mut receipts = Vec::new(); - - while !pending.is_empty() { - let mut remaining = Vec::new(); - - for (kind, tx_hash) in pending.drain(..) { - tracing::debug!("Checking receipt for transaction: {:?}", tx_hash); - match evm_client.provider.get_transaction_receipt(tx_hash).await? { - Some(receipt) => { - receipts.push((kind, receipt)); - } - None => remaining.push((kind, tx_hash)), - } - } - - if start.elapsed() > Duration::from_secs(60) { - tracing::error!( - "Timeout waiting for transactions to be mined. Pending: {}, Mined: {}", - remaining.len(), - receipts.len() - ); - bail!("Timed out waiting for transactions to be mined"); - } - - pending = remaining; - } - - let mut trigger_ids = Vec::new(); - for (kind, receipt) in receipts { - if matches!(kind, TxKind::Trigger) { - if let Some(event) = - SolidityEventFinder::::solidity_event(&receipt) - { - let trigger_info = TriggerInfo::abi_decode(&event.triggerData)?; - trigger_ids.push(TriggerId::new(trigger_info.triggerId)); - } - } - } - - if trigger_ids.is_empty() { - bail!("Failed to obtain trigger id from transaction receipts"); - } - - tracing::info!( - "Successfully extracted {} trigger IDs: {:?}", - trigger_ids.len(), - trigger_ids - ); - trigger_ids - } - Trigger::CosmosContractEvent { - chain, - address, - event_type: _, - } => { - let client = SimpleCosmosTriggerClient::new( - clients.get_cosmos_client(chain).await, - address.clone().into(), - ); - - let trigger_id = client - .add_trigger(input_bytes.expect("Cosmos triggers require an input")) - .await?; - - vec![TriggerId::new(trigger_id.u64())] - } - Trigger::BlockInterval { .. } => vec![TriggerId::new(1337)], - Trigger::Cron { .. } => vec![TriggerId::new(1338)], - Trigger::AtProtoEvent { .. } => { - let sequence: u64 = 1339; - let trigger_id = TriggerId::new(sequence); - - let record_payload = input_bytes.clone().unwrap_or_default(); - let record_text = String::from_utf8_lossy(&record_payload).to_string(); - - // Send simulated trigger to all WAVS instances - for http_client in clients.http_clients.iter() { - let atproto_data = TriggerData::AtProtoEvent { - sequence: sequence as i64, - timestamp: 0, - repo: "did:example:alice".to_string(), - collection: "app.bsky.feed.post".to_string(), - rkey: "rkey-1".to_string(), - action: AtProtoAction::Create, - cid: Some("bafytestcid".to_string()), - record: Some(json!({ "text": record_text.clone() })), - rev: Some("rev-test".to_string()), - op_index: Some(0), - }; - - let req = SimulatedTriggerRequest { - service_id: service_deployment.service.id(), - workflow_id: first_workflow_id.clone(), - trigger: trigger.clone(), - data: atproto_data, - count: 1, - wait_for_completion: true, - }; - - http_client.simulate_trigger(req).await?; - } - - vec![trigger_id] - } - Trigger::HypercoreAppend { feed_key } => { - // Try to get the hypercore test client for this test - let payload = input_bytes.clone().unwrap_or_default(); - - tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); - - if let Some(hypercore_client) = hypercore_clients.get(&test.name) { - let client_feed_key = hypercore_client.feed_key(); - tracing::info!( - "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", - test.name, - client_feed_key, - feed_key - ); - - for (idx, http_client) in clients.http_clients.iter().enumerate() { - tracing::info!( - "Waiting for hypercore stream readiness on instance {} for feed_key {}", - idx, - feed_key - ); - wait_for_hypercore_streams_to_finalize( - http_client, - feed_key, - Some(Duration::from_secs(30)), - ) - .await - .context("Failed to wait for hypercore stream to finalize")?; - } - - // Wait for hypercore mesh to stabilize - require at least 1 WAVS instance to connect. - // If 0 peers are connected the append will never replicate, so fail fast. - { - let min_required_peers = 1; - let total_operators = clients.http_clients.len(); - tracing::info!( - "Waiting for hypercore mesh to stabilize (min {} peer, {} total operators) before append", - min_required_peers, - total_operators - ); - - let peer_count = wait_for_hypercore_mesh_ready( - &hypercore_client, - min_required_peers, - Duration::from_secs(90), - ) - .await - .context("Hypercore mesh not ready: 0 peers connected, append will never replicate")?; - - tracing::info!( - "Hypercore mesh ready for append: {} connected peers (min required: {}, total operators: {})", - peer_count, - min_required_peers, - total_operators - ); - } - - // Verify feed keys match - if client_feed_key != *feed_key { - tracing::error!( - "FEED KEY MISMATCH! Client has: {}, Service has: {}", - client_feed_key, - feed_key - ); - return Err(anyhow::anyhow!( - "Feed key mismatch between client and service" - )); - } - - // Append data to the hypercore feed - tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); - let index = hypercore_client.append(payload).await?; - - vec![TriggerId::new(index)] - } else { - // Fallback to simulated trigger for backward compatibility - tracing::warn!( - "No hypercore client found for test '{}', using simulated trigger", - test.name - ); - - let trigger_id = TriggerId::new(0); - let hypercore_data = TriggerData::HypercoreAppend { - feed_key: feed_key.clone(), - index: trigger_id.u64(), - data: payload, - }; - - let req = SimulatedTriggerRequest { - service_id: service_deployment.service.id(), - workflow_id: first_workflow_id.clone(), - trigger: trigger.clone(), - data: hypercore_data, - count: 1, - wait_for_completion: true, - }; - - let http_client = clients - .http_clients - .first() - .ok_or_else(|| anyhow!("No HTTP clients available"))?; - http_client.simulate_trigger(req).await?; - - vec![trigger_id] - } - } - Trigger::Manual => unimplemented!("Manual trigger type is not implemented"), - }; - - tracing::info!( - "Starting workflow validation for {} workflows", - workflows_group.len() - ); - // Validate all workflows associated with this trigger - for (workflow_id, workflow) in workflows_group { - tracing::info!("Validating workflow: {}", workflow_id); - let WorkflowDefinition { - timeout, - expected_output, - .. - } = &test.workflows.get(workflow_id).ok_or(anyhow!( - "Could not get workflow definition from id: {}", - workflow_id - ))?; - - for trigger_id in trigger_ids.iter().copied() { - tracing::info!( - "Processing trigger_id: {} for workflow: {}", - trigger_id, - workflow_id - ); - let data = match &workflow.submit { - Submit::Aggregator { .. } => { - let workflow_def = test.workflows.get(workflow_id).ok_or_else(|| { - anyhow!("Could not get workflow definition from id: {}", workflow_id) - })?; - - let SubmitDefinition::Aggregator(aggregator) = &workflow_def.submit; - let AggregatorDefinition::ComponentBasedAggregator { chain, .. } = - aggregator; - - match chain.namespace.as_str() { - ChainKeyNamespace::COSMOS => { - let client = clients.get_cosmos_client(chain).await; - let submission_contract = service_deployment - .submission_handlers - .get(workflow_id) - .ok_or_else(|| { - anyhow!( - "No submission contract found for workflow {}", - workflow_id - ) - })?; - - let data = cosmos_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - *timeout, - ) - .await?; - - tracing::info!("Task result: {:?}", data); - - data - } - ChainKeyNamespace::EVM => { - let client = clients.get_evm_client(chain); - tracing::info!( - "Getting submit start block for workflow: {}", - workflow_id - ); - let submit_start_block = - client.provider.get_block_number().await.map_err(|e| { - anyhow!("Failed to get block number: {}", e) - })?; - tracing::info!("Submit start block: {}", submit_start_block); - - let submission_contract = service_deployment - .submission_handlers - .get(workflow_id) - .ok_or_else(|| { - anyhow!( - "No submission contract found for workflow {}", - workflow_id - ) - })?; - tracing::info!( - "Submission contract for workflow {}: {}", - workflow_id, - submission_contract - ); - - if first_workflow.expects_reorg() { - tracing::info!("Test '{}' will simulate re-org", test.name); - - // Simulate re-org before waiting for task - simulate_anvil_reorg( - &client, - reorg_snapshot.expect( - "Expected a reorg snapshot when simulating reorg", - ), - ) - .await?; - - // Wait for task - should return empty data on error due to re-org - tracing::info!( - "Waiting for task to land after re-org for trigger_id: {}", - trigger_id - ); - let result = evm_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - submit_start_block, - *timeout, - ) - .await; - - match result { - Ok(signed_data) => signed_data.data.to_vec(), - // If we get an error (transaction dropped due to re-org), - // return mocked signed data with empty content to match ExpectedOutput::Dropped - Err(_) => Vec::new(), - } - } else { - tracing::info!( - "Waiting for task to land (no re-org) for trigger_id: {}", - trigger_id - ); - let result = evm_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - submit_start_block, - *timeout, - ) - .await?; - tracing::info!("Task result (no re-org): {:?}", result.data); - result.data.to_vec() - } - } - _ => unimplemented!("Unsupported chain namespace for aggregator"), - } - } - Submit::None => unimplemented!("Submit::None is not implemented"), - }; - - tracing::info!("Validating expected output for workflow: {}", workflow_id); - expected_output.validate(test, clients, component_sources, &data)?; - tracing::info!( - "Successfully validated output for workflow: {}", - workflow_id - ); - } - } - tracing::info!("Test completed successfully!"); - } - - // Wait for the aggregator submit callback to complete on all WAVS instances - // before cleaning up the service. This ensures the after-submit callback - // has finished writing to the KV store. - // Only do this if: - // 1. Any workflow uses an aggregator submit - // 2. No workflow expects dropped output (e.g., reorg tests where submission is intentionally skipped) - let has_aggregator = service_deployment - .service - .workflows - .values() - .any(|w| matches!(w.submit, Submit::Aggregator { .. })); - - let expects_dropped = test.workflows.values().any(|w| w.expects_reorg()); - - if has_aggregator && !expects_dropped { - let service_id = service_deployment.service.id().to_string(); - tracing::info!( - "Waiting for submit callback to complete for service: {}", - service_id - ); - for (idx, http_client) in clients.http_clients.iter().enumerate() { - http_client - .wait_for_submit_callback(&service_id, None) - .await - .map_err(|e| { - anyhow!("Instance {} failed waiting for submit callback: {}", idx, e) - })?; - tracing::info!( - "Submit callback completed on instance {} for service: {}", - idx, - service_id - ); - } - } - - tracing::info!( - "Cleaning up service: {0:?}", - service_deployment.service.manager - ); - // Delete service from all WAVS instances - for http_client in clients.http_clients.iter() { - http_client - .delete_service(vec![service_deployment.service.manager.clone()]) - .await?; - } - - Ok(()) -} +// src/e2e/test_runner.rs + +use crate::deployment::ServiceDeployment; +use crate::e2e::config::Configs; +use crate::example_evm_client::example_trigger::ISimpleTrigger::TriggerInfo; +use crate::example_evm_client::example_trigger::NewTrigger; +use alloy_primitives::U256; +use alloy_provider::ext::AnvilApi; +use alloy_provider::Provider; +use alloy_sol_types::SolType; +use anyhow::{anyhow, bail, Context}; +use futures::{stream::FuturesUnordered, StreamExt}; +use ordermap::OrderMap; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use utils::alloy_helpers::SolidityEventFinder; +use wavs_types::{ + AtProtoAction, ChainKeyNamespace, SimulatedTriggerRequest, Submit, Trigger, TriggerData, + Workflow, WorkflowId, +}; + +use crate::e2e::handles::hypercore::HypercoreClients; +use crate::e2e::helpers::wait_for_hypercore_streams_to_finalize; +use crate::e2e::helpers::{ + change_service_for_test, cosmos_wait_for_task_to_land, wait_for_hypercore_mesh_ready, +}; +use crate::e2e::report::TestReport; +use crate::e2e::service_managers::ServiceManagers; +use crate::e2e::test_definition::{ + AggregatorDefinition, ChangeServiceDefinition, SubmitDefinition, +}; +use crate::e2e::test_registry::CosmosCodeMap; +use crate::{ + e2e::{ + clients::Clients, components::ComponentSources, test_definition::TestDefinition, + test_registry::TestRegistry, + }, + example_cosmos_client::SimpleCosmosTriggerClient, + example_evm_client::{LogSpamClient, SimpleEvmTriggerClient, TriggerId}, +}; +use serde_json::json; + +use super::helpers::{evm_wait_for_task_to_land, simulate_anvil_reorg}; +use super::test_definition::WorkflowDefinition; + +/// Simplified test runner that leverages services directly attached to test definitions +pub struct Runner { + configs: Arc, + clients: Arc, + registry: Arc, + component_sources: Arc, + hypercore_clients: HypercoreClients, + service_managers: ServiceManagers, + cosmos_code_map: CosmosCodeMap, + report: TestReport, +} + +/// Extract service handler address from an aggregator submit configuration +fn extract_aggregator_service_handler(submit: &Submit) -> Option { + match submit { + Submit::Aggregator { component, .. } => { + component + .config + .get("service_handler") + .and_then(|addr_str| { + match layer_climb::prelude::CosmosAddr::new_str(addr_str, None) { + Ok(cosmos_addr) => Some(layer_climb::prelude::Address::Cosmos(cosmos_addr)), + Err(_) => layer_climb::prelude::EvmAddr::new_str(addr_str) + .ok() + .map(layer_climb::prelude::Address::from), + } + }) + } + _ => None, + } +} + +impl Runner { + #[allow(clippy::too_many_arguments)] + pub fn new( + configs: Configs, + clients: Clients, + registry: TestRegistry, + component_sources: ComponentSources, + hypercore_clients: HypercoreClients, + service_managers: ServiceManagers, + cosmos_code_map: CosmosCodeMap, + report: TestReport, + ) -> Self { + Self { + configs: Arc::new(configs), + clients: Arc::new(clients), + registry: Arc::new(registry), + component_sources: Arc::new(component_sources), + hypercore_clients, + service_managers, + cosmos_code_map, + report, + } + } + + /// Run all tests in the registry + pub async fn run_tests(&mut self, mut all_services: HashMap) { + let test_groups = self.registry.list_all_grouped(self.configs.grouping); + + for (group, mut group_tests) in test_groups { + let services = group_tests + .iter() + .map(|test| all_services.get(&test.name).cloned().unwrap().service) + .collect::>(); + + // This essentially deploys the services for the group + // since it updates the services to "Active" + // which is detected by wavs + self.service_managers + .update_services(&self.clients, services) + .await; + + // However, we have some tests which demonstrate more specific service changes + // and so we need to re-update those before we can proceed + // + // First we just deploy the service changes (contracts, components, etc.) + let mut futures = FuturesUnordered::new(); + for test in group_tests.iter() { + if let Some(change_service) = test.change_service.clone() { + let service = all_services.get(&test.name).cloned().unwrap().service; + let clients = self.clients.clone(); + let component_sources = self.component_sources.clone(); + let cosmos_code_map = self.cosmos_code_map.clone(); + futures.push(async move { + let mut service = service; + change_service_for_test( + &mut service, + change_service.clone(), + &clients, + &component_sources, + cosmos_code_map, + ) + .await; + (service, change_service) + }); + } + } + + // Then we need to deploy the update to service managers + if futures.is_empty() { + tracing::info!("No changes to services in group {:?}", group); + } else { + tracing::warn!("Running service changes for group {:?}", group); + let mut services_to_change = Vec::new(); + while let Some((service, change_service)) = futures.next().await { + // update our local copy of the service and handle changes + let service_deployment = all_services + .get_mut(&service.name) + .expect("Service should exist in all_services"); + + service_deployment.service = service.clone(); + + // and the definition so that tests know what to look for + match change_service { + ChangeServiceDefinition::AddWorkflow { + workflow_id, + workflow, + } => { + // When a workflow is added, it includes a new submission contract + // Extract it from the service's workflow that was just added + let submission_address = service_deployment + .service + .workflows + .get(&workflow_id) + .and_then(|workflow| { + extract_aggregator_service_handler(&workflow.submit) + }); + + if let Some(address) = submission_address { + service_deployment + .submission_handlers + .insert(workflow_id.clone(), address); + } + + group_tests + .iter_mut() + .find(|test| test.name == service.name) + .unwrap() + .workflows + .insert(workflow_id.clone(), workflow); + } + ChangeServiceDefinition::Component { + workflow_id, + component, + } => { + group_tests + .iter_mut() + .find(|test| test.name == service.name) + .unwrap() + .workflows + .get_mut(&workflow_id) + .unwrap() + .component = component; + } + } + + services_to_change.push(service); + } + + self.service_managers + .update_services(&self.clients, services_to_change) + .await; + } + + // Create hypercore clients AFTER deploying services so WAVS is already + // doing DHT lookups when the test client announces. This avoids the stale-DHT + // problem where the client announces 10+ seconds before WAVS starts looking. + self.hypercore_clients + .create_clients() + .await + .expect("Failed to create hypercore clients"); + + // All services are now deployed and ready for the tests + // From here on in we're strictly testing the trigger->execute->aggregate->submit flow + + tracing::info!("Running group {:?} with {} tests", group, group_tests.len()); + let mut futures = FuturesUnordered::new(); + let hypercore_clients = &self.hypercore_clients; + + for test in group_tests { + let clients = self.clients.clone(); + let component_sources = self.component_sources.clone(); + let test = test.clone(); + let report = self.report.clone(); + let service = all_services.get(&test.name).cloned().unwrap(); + futures.push(async move { + Self::execute_test( + &test, + service, + clients, + component_sources, + hypercore_clients, + report, + ) + .await + }); + } + + while (futures.next().await).is_some() {} + } + } + + // Execute a single test with timings + async fn execute_test( + test: &TestDefinition, + service_deployment: ServiceDeployment, + clients: Arc, + component_sources: Arc, + hypercore_clients: &HypercoreClients, + report: TestReport, + ) { + report.start_test(test.name.clone()); + + run_test( + test, + service_deployment, + &clients, + &component_sources, + hypercore_clients, + ) + .await + .context(test.name.clone()) + .unwrap(); + + report.end_test(test.name.clone()); + } +} + +/// Run a single test +async fn run_test( + test: &TestDefinition, + service_deployment: ServiceDeployment, + clients: &Clients, + component_sources: &ComponentSources, + hypercore_clients: &HypercoreClients, +) -> anyhow::Result<()> { + // For multi-operator tests, wait for P2P mesh to form before triggering + if test.multi_operator && clients.http_clients.len() > 1 { + let expected_peers = clients.http_clients.len() - 1; + tracing::info!( + "Multi-operator test: waiting for P2P mesh formation ({} expected peers)", + expected_peers + ); + + // Wait for all operators to have connected to peers + for (idx, http_client) in clients.http_clients.iter().enumerate() { + let status = http_client + .wait_for_p2p_ready(expected_peers, Some(Duration::from_secs(30))) + .await + .map_err(|e| { + anyhow!( + "Operator {} P2P readiness check failed: {}. \ + Multi-operator tests require P2P mesh to be ready.", + idx, + e + ) + })?; + tracing::info!( + "Operator {} P2P ready: {} connected peers", + idx, + status.connected_peers + ); + } + } + + // Group workflows by trigger to handle multi-triggers + let mut trigger_groups: OrderMap<&Trigger, Vec<(&WorkflowId, &Workflow)>> = OrderMap::new(); + + for (workflow_id, workflow) in service_deployment.service.workflows.iter() { + trigger_groups + .entry(&workflow.trigger) + .or_default() + .push((workflow_id, workflow)); + } + + // Process each unique trigger once, then validate all associated workflows + for (trigger, workflows_group) in trigger_groups { + // Use the first workflow to execute the trigger + let (first_workflow_id, _) = workflows_group[0]; + + // Get the workflow data safely + let first_workflow = test + .workflows + .get(first_workflow_id) + .ok_or(anyhow!("Could not get workflow: {}", first_workflow_id))?; + + // Convert input data to bytes safely + let input_bytes = first_workflow.input_data.to_bytes(); + + // Execute the trigger once + let mut reorg_snapshot: Option = None; + let trigger_ids = match trigger { + Trigger::EvmContractEvent { + chain, + address, + event_hash: _, + } => { + let evm_client = clients.get_evm_client(chain); + let client = SimpleEvmTriggerClient::new(evm_client.clone(), *address); + + if first_workflow.expects_reorg() { + reorg_snapshot = Some(evm_client.provider.anvil_snapshot().await?); + } + let input = input_bytes.clone().expect("EVM triggers require an input"); + + let spam_client = if first_workflow.trigger_execution.log_spam_count > 0 { + let address = super::helpers::deploy_log_spam_contract(clients, chain).await?; + let client = LogSpamClient::new(evm_client.clone(), address); + Some(client) + } else { + None + }; + + #[derive(Clone, Copy, Debug)] + enum TxKind { + Trigger, + Spam, + } + + let mut pending: Vec<(TxKind, alloy_primitives::TxHash)> = Vec::new(); + + let pending_trigger = client + .contract + .addTrigger(input.clone().into()) + .send() + .await?; + pending.push((TxKind::Trigger, *pending_trigger.tx_hash())); + + if let Some(spam_client) = &spam_client { + let spam_count = first_workflow.trigger_execution.log_spam_count as u64; + tracing::info!( + "Emitting {} bulk spam logs using LogSpam contract", + spam_count + ); + + // Use bulk emission to spam N logs in a single transaction + let spam_hash = spam_client.emit_spam(0, spam_count).await?; + + tracing::info!("Bulk spam transaction sent: {:?}", spam_hash); + pending.push((TxKind::Spam, spam_hash)); + } + + let start = Instant::now(); + let mut receipts = Vec::new(); + + while !pending.is_empty() { + let mut remaining = Vec::new(); + + for (kind, tx_hash) in pending.drain(..) { + tracing::debug!("Checking receipt for transaction: {:?}", tx_hash); + match evm_client.provider.get_transaction_receipt(tx_hash).await? { + Some(receipt) => { + receipts.push((kind, receipt)); + } + None => remaining.push((kind, tx_hash)), + } + } + + if start.elapsed() > Duration::from_secs(60) { + tracing::error!( + "Timeout waiting for transactions to be mined. Pending: {}, Mined: {}", + remaining.len(), + receipts.len() + ); + bail!("Timed out waiting for transactions to be mined"); + } + + pending = remaining; + } + + let mut trigger_ids = Vec::new(); + for (kind, receipt) in receipts { + if matches!(kind, TxKind::Trigger) { + if let Some(event) = + SolidityEventFinder::::solidity_event(&receipt) + { + let trigger_info = TriggerInfo::abi_decode(&event.triggerData)?; + trigger_ids.push(TriggerId::new(trigger_info.triggerId)); + } + } + } + + if trigger_ids.is_empty() { + bail!("Failed to obtain trigger id from transaction receipts"); + } + + tracing::info!( + "Successfully extracted {} trigger IDs: {:?}", + trigger_ids.len(), + trigger_ids + ); + trigger_ids + } + Trigger::CosmosContractEvent { + chain, + address, + event_type: _, + } => { + let client = SimpleCosmosTriggerClient::new( + clients.get_cosmos_client(chain).await, + address.clone().into(), + ); + + let trigger_id = client + .add_trigger(input_bytes.expect("Cosmos triggers require an input")) + .await?; + + vec![TriggerId::new(trigger_id.u64())] + } + Trigger::BlockInterval { .. } => vec![TriggerId::new(1337)], + Trigger::Cron { .. } => vec![TriggerId::new(1338)], + Trigger::AtProtoEvent { .. } => { + let sequence: u64 = 1339; + let trigger_id = TriggerId::new(sequence); + + let record_payload = input_bytes.clone().unwrap_or_default(); + let record_text = String::from_utf8_lossy(&record_payload).to_string(); + + // Send simulated trigger to all WAVS instances + for http_client in clients.http_clients.iter() { + let atproto_data = TriggerData::AtProtoEvent { + sequence: sequence as i64, + timestamp: 0, + repo: "did:example:alice".to_string(), + collection: "app.bsky.feed.post".to_string(), + rkey: "rkey-1".to_string(), + action: AtProtoAction::Create, + cid: Some("bafytestcid".to_string()), + record: Some(json!({ "text": record_text.clone() })), + rev: Some("rev-test".to_string()), + op_index: Some(0), + }; + + let req = SimulatedTriggerRequest { + service_id: service_deployment.service.id(), + workflow_id: first_workflow_id.clone(), + trigger: trigger.clone(), + data: atproto_data, + count: 1, + wait_for_completion: true, + }; + + http_client.simulate_trigger(req).await?; + } + + vec![trigger_id] + } + Trigger::HypercoreAppend { feed_key } => { + // Try to get the hypercore test client for this test + let payload = input_bytes.clone().unwrap_or_default(); + + tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); + + if let Some(hypercore_client) = hypercore_clients.get(&test.name) { + let client_feed_key = hypercore_client.feed_key(); + tracing::info!( + "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", + test.name, + client_feed_key, + feed_key + ); + + // Verify feed keys match before waiting for connectivity + if client_feed_key != *feed_key { + tracing::error!( + "FEED KEY MISMATCH! Client has: {}, Service has: {}", + client_feed_key, + feed_key + ); + return Err(anyhow::anyhow!( + "Feed key mismatch between client and service" + )); + } + + // Wait for all instances' hypercore streams AND the test client mesh + // to be ready concurrently. Both check the same underlying DHT + // connectivity from different sides, so running them in parallel + // avoids wasting the timeout budget on sequential per-instance waits. + let connectivity_timeout = Duration::from_secs(60); + let min_required_peers = 1; + let total_operators = clients.http_clients.len(); + + tracing::info!( + "Waiting for hypercore streams and mesh (min {} peer, {} total operators, timeout {}s)", + min_required_peers, + total_operators, + connectivity_timeout.as_secs(), + ); + + let stream_futs: Vec<_> = clients + .http_clients + .iter() + .enumerate() + .map(|(idx, http_client)| async move { + tracing::info!( + "Waiting for hypercore stream readiness on instance {} for feed_key {}", + idx, + feed_key + ); + wait_for_hypercore_streams_to_finalize( + http_client, + feed_key, + Some(connectivity_timeout), + ) + .await + .with_context(|| { + format!( + "Hypercore stream failed to finalize on instance {idx}" + ) + }) + }) + .collect(); + + let mesh_fut = wait_for_hypercore_mesh_ready( + &hypercore_client, + min_required_peers, + connectivity_timeout, + ); + + let (streams_result, mesh_result) = tokio::join!( + futures::future::try_join_all(stream_futs), + mesh_fut, + ); + + streams_result + .context("Failed to wait for hypercore streams to finalize")?; + let peer_count = mesh_result + .context("Hypercore mesh not ready: 0 peers connected, append will never replicate")?; + + tracing::info!( + "Hypercore streams and mesh ready: {} connected peers (min required: {}, total operators: {})", + peer_count, + min_required_peers, + total_operators + ); + + // Append data to the hypercore feed + tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); + let index = hypercore_client.append(payload).await?; + + vec![TriggerId::new(index)] + } else { + // Fallback to simulated trigger for backward compatibility + tracing::warn!( + "No hypercore client found for test '{}', using simulated trigger", + test.name + ); + + let trigger_id = TriggerId::new(0); + let hypercore_data = TriggerData::HypercoreAppend { + feed_key: feed_key.clone(), + index: trigger_id.u64(), + data: payload, + }; + + let req = SimulatedTriggerRequest { + service_id: service_deployment.service.id(), + workflow_id: first_workflow_id.clone(), + trigger: trigger.clone(), + data: hypercore_data, + count: 1, + wait_for_completion: true, + }; + + let http_client = clients + .http_clients + .first() + .ok_or_else(|| anyhow!("No HTTP clients available"))?; + http_client.simulate_trigger(req).await?; + + vec![trigger_id] + } + } + Trigger::Manual => unimplemented!("Manual trigger type is not implemented"), + }; + + tracing::info!( + "Starting workflow validation for {} workflows", + workflows_group.len() + ); + // Validate all workflows associated with this trigger + for (workflow_id, workflow) in workflows_group { + tracing::info!("Validating workflow: {}", workflow_id); + let WorkflowDefinition { + timeout, + expected_output, + .. + } = &test.workflows.get(workflow_id).ok_or(anyhow!( + "Could not get workflow definition from id: {}", + workflow_id + ))?; + + for trigger_id in trigger_ids.iter().copied() { + tracing::info!( + "Processing trigger_id: {} for workflow: {}", + trigger_id, + workflow_id + ); + let data = match &workflow.submit { + Submit::Aggregator { .. } => { + let workflow_def = test.workflows.get(workflow_id).ok_or_else(|| { + anyhow!("Could not get workflow definition from id: {}", workflow_id) + })?; + + let SubmitDefinition::Aggregator(aggregator) = &workflow_def.submit; + let AggregatorDefinition::ComponentBasedAggregator { chain, .. } = + aggregator; + + match chain.namespace.as_str() { + ChainKeyNamespace::COSMOS => { + let client = clients.get_cosmos_client(chain).await; + let submission_contract = service_deployment + .submission_handlers + .get(workflow_id) + .ok_or_else(|| { + anyhow!( + "No submission contract found for workflow {}", + workflow_id + ) + })?; + + let data = cosmos_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + *timeout, + ) + .await?; + + tracing::info!("Task result: {:?}", data); + + data + } + ChainKeyNamespace::EVM => { + let client = clients.get_evm_client(chain); + tracing::info!( + "Getting submit start block for workflow: {}", + workflow_id + ); + let submit_start_block = + client.provider.get_block_number().await.map_err(|e| { + anyhow!("Failed to get block number: {}", e) + })?; + tracing::info!("Submit start block: {}", submit_start_block); + + let submission_contract = service_deployment + .submission_handlers + .get(workflow_id) + .ok_or_else(|| { + anyhow!( + "No submission contract found for workflow {}", + workflow_id + ) + })?; + tracing::info!( + "Submission contract for workflow {}: {}", + workflow_id, + submission_contract + ); + + if first_workflow.expects_reorg() { + tracing::info!("Test '{}' will simulate re-org", test.name); + + // Simulate re-org before waiting for task + simulate_anvil_reorg( + &client, + reorg_snapshot.expect( + "Expected a reorg snapshot when simulating reorg", + ), + ) + .await?; + + // Wait for task - should return empty data on error due to re-org + tracing::info!( + "Waiting for task to land after re-org for trigger_id: {}", + trigger_id + ); + let result = evm_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + submit_start_block, + *timeout, + ) + .await; + + match result { + Ok(signed_data) => signed_data.data.to_vec(), + // If we get an error (transaction dropped due to re-org), + // return mocked signed data with empty content to match ExpectedOutput::Dropped + Err(_) => Vec::new(), + } + } else { + tracing::info!( + "Waiting for task to land (no re-org) for trigger_id: {}", + trigger_id + ); + let result = evm_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + submit_start_block, + *timeout, + ) + .await?; + tracing::info!("Task result (no re-org): {:?}", result.data); + result.data.to_vec() + } + } + _ => unimplemented!("Unsupported chain namespace for aggregator"), + } + } + Submit::None => unimplemented!("Submit::None is not implemented"), + }; + + tracing::info!("Validating expected output for workflow: {}", workflow_id); + expected_output.validate(test, clients, component_sources, &data)?; + tracing::info!( + "Successfully validated output for workflow: {}", + workflow_id + ); + } + } + tracing::info!("Test completed successfully!"); + } + + // Wait for the aggregator submit callback to complete on all WAVS instances + // before cleaning up the service. This ensures the after-submit callback + // has finished writing to the KV store. + // Only do this if: + // 1. Any workflow uses an aggregator submit + // 2. No workflow expects dropped output (e.g., reorg tests where submission is intentionally skipped) + let has_aggregator = service_deployment + .service + .workflows + .values() + .any(|w| matches!(w.submit, Submit::Aggregator { .. })); + + let expects_dropped = test.workflows.values().any(|w| w.expects_reorg()); + + if has_aggregator && !expects_dropped { + let service_id = service_deployment.service.id().to_string(); + tracing::info!( + "Waiting for submit callback to complete for service: {}", + service_id + ); + for (idx, http_client) in clients.http_clients.iter().enumerate() { + http_client + .wait_for_submit_callback(&service_id, None) + .await + .map_err(|e| { + anyhow!("Instance {} failed waiting for submit callback: {}", idx, e) + })?; + tracing::info!( + "Submit callback completed on instance {} for service: {}", + idx, + service_id + ); + } + } + + tracing::info!( + "Cleaning up service: {0:?}", + service_deployment.service.manager + ); + // Delete service from all WAVS instances + for http_client in clients.http_clients.iter() { + http_client + .delete_service(vec![service_deployment.service.manager.clone()]) + .await?; + } + + Ok(()) +} From f17c70ecf6b5e36ffa17d19a70ce097121a81a9b Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 20 Feb 2026 14:10:51 -0600 Subject: [PATCH 06/10] Add periodic DHT re-lookup for hyperswarm peer discovery The hyperswarm DHT only executes announce/lookup once after bootstrapping. If one peer's lookup fires before the other has announced, they never discover each other. Fix by periodically re-issuing announce+lookup via SwarmHandle while no peers are connected, with proper cleanup on both the WAVS server and test client sides. Co-Authored-By: Claude Opus 4.6 --- .../layer-tests/src/e2e/handles/hypercore.rs | 642 +++++++++--------- .../trigger/streams/hypercore_stream.rs | 565 +++++++-------- 2 files changed, 632 insertions(+), 575 deletions(-) diff --git a/packages/layer-tests/src/e2e/handles/hypercore.rs b/packages/layer-tests/src/e2e/handles/hypercore.rs index 6f5723a82..121d779cd 100644 --- a/packages/layer-tests/src/e2e/handles/hypercore.rs +++ b/packages/layer-tests/src/e2e/handles/hypercore.rs @@ -1,309 +1,333 @@ -//! Hypercore test client for e2e tests. -//! -//! Provides a simple interface to create hypercore feeds, append data, -//! and use hyperswarm for peer discovery during tests. - -use ::hypercore_protocol::discovery_key; -use hypercore::{Hypercore, HypercoreBuilder, PartialKeypair, SigningKey, Storage, VerifyingKey}; -use hyperswarm::{Config as SwarmConfig, Hyperswarm, TopicConfig}; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, -}; -use tempfile::TempDir; -use tokio::sync::Mutex; -use tokio::task::JoinHandle; -use wavs::subsystems::trigger::streams::hypercore_protocol; - -/// Manages hypercore test client lifecycle (deferred creation and retrieval). -/// -/// During test registration, pending entries are inserted with the signing key -/// and bootstrap address. Clients are created lazily right before tests run -/// so DHT announcements stay fresh. -#[derive(Default)] -pub struct HypercoreClients { - /// Pending: (hyperswarm_bootstrap, signing_key_bytes) - pending: HashMap, Vec)>, - clients: HashMap>, -} - -impl HypercoreClients { - pub fn new() -> Self { - Self::default() - } - - /// Queue a test for deferred hypercore client creation. - pub fn insert_pending( - &mut self, - test_name: String, - hyperswarm_bootstrap: Option, - signing_key_bytes: Vec, - ) { - self.pending - .insert(test_name, (hyperswarm_bootstrap, signing_key_bytes)); - } - - /// Create hypercore clients for all pending tests, draining the pending queue. - /// Called right before tests run to ensure DHT announcements are fresh. - /// No-op if the queue has already been drained. - pub async fn create_clients(&mut self) -> anyhow::Result<()> { - for (test_name, (bootstrap, key_bytes)) in self.pending.drain() { - tracing::info!( - "Creating hypercore client for test '{}' right before test execution", - test_name - ); - let client = HypercoreTestClient::new(&test_name, bootstrap, &key_bytes).await?; - self.clients.insert(test_name, Arc::new(client)); - } - Ok(()) - } - - /// Get a hypercore test client by test name. - pub fn get(&self, test_name: &str) -> Option> { - self.clients.get(test_name).cloned() - } -} - -/// Test client for creating and managing hypercore feeds in e2e tests. -pub struct HypercoreTestClient { - /// The hypercore feed - feed: Arc>, - /// Hex-encoded feed key (public key) - feed_key: String, - /// Handle for the hyperswarm task - swarm_handle: JoinHandle<()>, - /// TempDir storage - must be kept alive for the lifetime of the client - _storage_dir: TempDir, - /// Connection count for testing mesh formation - connection_count_for_swarm: Arc, -} - -// Properly clean up the swarm task when the client is dropped -impl Drop for HypercoreTestClient { - fn drop(&mut self) { - tracing::info!( - "Dropping HypercoreTestClient for feed_key: {}, aborting swarm task", - self.feed_key - ); - self.swarm_handle.abort(); - } -} - -impl HypercoreTestClient { - /// Create a new hypercore feed with a pre-generated signing key. - /// - /// This is used when we need the feed_key early (for trigger registration) - /// but want to delay creating the full client until services are ready. - pub async fn new( - test_name: &str, - hyperswarm_bootstrap: Option, - signing_key_bytes: &[u8], - ) -> anyhow::Result { - // Create unique tempdir for this test - let storage_dir = TempDir::new()?; - let storage_path: PathBuf = storage_dir.path().to_path_buf(); - - tracing::info!( - "Creating hypercore test client with pre-generated key for '{}' with storage at: {}", - test_name, - storage_path.display() - ); - - // Create hypercore storage - let storage = Storage::new_disk(&storage_path, false) - .await - .map_err(|e| anyhow::anyhow!("Failed to create hypercore storage: {e:?}"))?; - - // Reconstruct the signing key from bytes - // Convert slice to array for SigningKey::from_bytes - let key_array: [u8; 32] = signing_key_bytes - .try_into() - .map_err(|_| anyhow::anyhow!("Invalid signing key length, expected 32 bytes"))?; - let signing_key = SigningKey::from_bytes(&key_array); - - let public_key_bytes = signing_key.verifying_key().to_bytes(); - let feed_key_bytes = public_key_bytes; - let feed_key = const_hex::encode(public_key_bytes); - - tracing::info!("Using hypercore feed key: {}", feed_key); - - // Reconstruct VerifyingKey from bytes for owned value - let public = VerifyingKey::from_bytes(&public_key_bytes) - .map_err(|e| anyhow::anyhow!("Failed to create verifying key: {e:?}"))?; - - // Create a PartialKeypair with both public and secret keys (for writable feed) - let key_pair = PartialKeypair { - public, - secret: Some(signing_key), - }; - - // Build hypercore with the generated keypair - let core = HypercoreBuilder::new(storage) - .key_pair(key_pair) - .build() - .await - .map_err(|e| anyhow::anyhow!("Failed to build hypercore: {e:?}"))?; - - // Set up hyperswarm for peer discovery - let topic = discovery_key(&public_key_bytes); - - let mut swarm = Hyperswarm::bind(build_swarm_config(hyperswarm_bootstrap.as_deref())) - .await - .map_err(|e| anyhow::anyhow!("Failed to bind hyperswarm: {e:?}"))?; - - // Announce and lookup for this feed's discovery key - swarm.configure(topic, TopicConfig::announce_and_lookup()); - - tracing::info!( - "Hyperswarm configured for discovery key: {}, topic: {:?}", - const_hex::encode(topic), - topic - ); - - let feed = Arc::new(Mutex::new(core)); - let swarm_feed = Arc::clone(&feed); - - // Spawn hyperswarm task to handle incoming connections - let feed_key_for_swarm = feed_key.clone(); - let feed_key_bytes_for_swarm = feed_key_bytes; - let connection_count_for_swarm = Arc::new(AtomicUsize::new(0)); - - // Clone the Arc for the spawned task (we keep the original for the struct) - let swarm_connection_count = Arc::clone(&connection_count_for_swarm); - let swarm_handle = tokio::spawn(async move { - let mut swarm = swarm; - tracing::info!( - "Hypercore swarm task started, listening for peers for feed_key: {}", - feed_key_for_swarm - ); - - use futures_lite::StreamExt; - while let Some(result) = swarm.next().await { - match result { - Ok(stream) => { - swarm_connection_count.fetch_add(1, Ordering::SeqCst); - tracing::debug!( - "Hyperswarm peer discovery attempt (initiator={}, peer_addr={:?}) for feed_key: {}", - stream.is_initiator(), - stream.peer_addr(), - feed_key_for_swarm - ); - tracing::info!( - "Hyperswarm connection established (initiator={}, peer_addr={:?}) for feed_key: {}", - stream.is_initiator(), - stream.peer_addr(), - feed_key_for_swarm - ); - let feed = Arc::clone(&swarm_feed); - let is_initiator = stream.is_initiator(); - let feed_key_bytes = feed_key_bytes_for_swarm; - - // Spawn a task for each peer connection - let connection_count_for_peer = Arc::clone(&swarm_connection_count); - tokio::spawn(async move { - let result = hypercore_protocol::run_protocol( - stream, - is_initiator, - feed, - feed_key_bytes, - None, - ) - .await; - - // Decrement connection count when peer connection closes - connection_count_for_peer.fetch_sub(1, Ordering::SeqCst); - - if let Err(err) = result { - tracing::error!( - "Hyperswarm connection failed for feed_key {}: {:?}", - const_hex::encode(feed_key_bytes), - err - ); - } else { - tracing::debug!( - "Hypercore protocol peer connection closed cleanly" - ); - } - }); - } - Err(err) => { - tracing::error!( - "Hyperswarm connection failed for feed_key {}: {:?}", - feed_key_for_swarm, - err - ); - } - } - } - - tracing::info!("Hypercore swarm task ended"); - }); - - Ok(Self { - feed, - feed_key, - swarm_handle, - _storage_dir: storage_dir, - connection_count_for_swarm, - }) - } - - /// Get the hex-encoded feed key (public key). - /// - /// This should be used when registering hypercore triggers - /// in service definitions. - pub fn feed_key(&self) -> String { - self.feed_key.clone() - } - - /// Get the current number of connected peers for this hypercore feed. - /// - /// This is used in tests to wait for mesh formation before proceeding. - pub fn connected_peer_count(&self) -> usize { - self.connection_count_for_swarm - .load(std::sync::atomic::Ordering::Relaxed) - } - - /// Append data to the hypercore feed. - /// - /// Returns the index of the appended block. - pub async fn append(&self, data: Vec) -> anyhow::Result { - let mut feed = self.feed.lock().await; - let outcome = feed - .append(&data) - .await - .map_err(|e| anyhow::anyhow!("Failed to append to hypercore: {e:?}"))?; - - // AppendOutcome contains the length, we need to return the index - let index = outcome.length.saturating_sub(1); - - tracing::info!( - "Appended {} bytes to hypercore feed at index {}", - data.len(), - index - ); - - Ok(index) - } -} - -fn build_swarm_config(hyperswarm_bootstrap: Option<&str>) -> SwarmConfig { - if let Some(addr) = hyperswarm_bootstrap { - match addr.parse::() { - Ok(addr) => { - tracing::info!("Using hyperswarm bootstrap: {}", addr); - return SwarmConfig::default() - .set_bootstrap_nodes(&[addr]) - .with_defaults(); - } - Err(err) => { - tracing::warn!("Invalid hyperswarm bootstrap address '{}': {err}", addr); - } - } - } - - SwarmConfig::all() -} +//! Hypercore test client for e2e tests. +//! +//! Provides a simple interface to create hypercore feeds, append data, +//! and use hyperswarm for peer discovery during tests. + +use ::hypercore_protocol::discovery_key; +use hypercore::{Hypercore, HypercoreBuilder, PartialKeypair, SigningKey, Storage, VerifyingKey}; +use hyperswarm::{Config as SwarmConfig, Hyperswarm, TopicConfig}; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; +use std::time::Duration; +use tempfile::TempDir; +use tokio::sync::Mutex; +use tokio::task::JoinHandle; +use wavs::subsystems::trigger::streams::hypercore_protocol; + +/// Manages hypercore test client lifecycle (deferred creation and retrieval). +/// +/// During test registration, pending entries are inserted with the signing key +/// and bootstrap address. Clients are created lazily right before tests run +/// so DHT announcements stay fresh. +#[derive(Default)] +pub struct HypercoreClients { + /// Pending: (hyperswarm_bootstrap, signing_key_bytes) + pending: HashMap, Vec)>, + clients: HashMap>, +} + +impl HypercoreClients { + pub fn new() -> Self { + Self::default() + } + + /// Queue a test for deferred hypercore client creation. + pub fn insert_pending( + &mut self, + test_name: String, + hyperswarm_bootstrap: Option, + signing_key_bytes: Vec, + ) { + self.pending + .insert(test_name, (hyperswarm_bootstrap, signing_key_bytes)); + } + + /// Create hypercore clients for all pending tests, draining the pending queue. + /// Called right before tests run to ensure DHT announcements are fresh. + /// No-op if the queue has already been drained. + pub async fn create_clients(&mut self) -> anyhow::Result<()> { + for (test_name, (bootstrap, key_bytes)) in self.pending.drain() { + tracing::info!( + "Creating hypercore client for test '{}' right before test execution", + test_name + ); + let client = HypercoreTestClient::new(&test_name, bootstrap, &key_bytes).await?; + self.clients.insert(test_name, Arc::new(client)); + } + Ok(()) + } + + /// Get a hypercore test client by test name. + pub fn get(&self, test_name: &str) -> Option> { + self.clients.get(test_name).cloned() + } +} + +/// Test client for creating and managing hypercore feeds in e2e tests. +pub struct HypercoreTestClient { + /// The hypercore feed + feed: Arc>, + /// Hex-encoded feed key (public key) + feed_key: String, + /// Handle for the hyperswarm task + swarm_handle: JoinHandle<()>, + /// Handle for the periodic re-lookup task + relookup_handle: JoinHandle<()>, + /// TempDir storage - must be kept alive for the lifetime of the client + _storage_dir: TempDir, + /// Connection count for testing mesh formation + connection_count_for_swarm: Arc, +} + +// Properly clean up the swarm and re-lookup tasks when the client is dropped +impl Drop for HypercoreTestClient { + fn drop(&mut self) { + tracing::info!( + "Dropping HypercoreTestClient for feed_key: {}, aborting swarm tasks", + self.feed_key + ); + self.swarm_handle.abort(); + self.relookup_handle.abort(); + } +} + +impl HypercoreTestClient { + /// Create a new hypercore feed with a pre-generated signing key. + /// + /// This is used when we need the feed_key early (for trigger registration) + /// but want to delay creating the full client until services are ready. + pub async fn new( + test_name: &str, + hyperswarm_bootstrap: Option, + signing_key_bytes: &[u8], + ) -> anyhow::Result { + // Create unique tempdir for this test + let storage_dir = TempDir::new()?; + let storage_path: PathBuf = storage_dir.path().to_path_buf(); + + tracing::info!( + "Creating hypercore test client with pre-generated key for '{}' with storage at: {}", + test_name, + storage_path.display() + ); + + // Create hypercore storage + let storage = Storage::new_disk(&storage_path, false) + .await + .map_err(|e| anyhow::anyhow!("Failed to create hypercore storage: {e:?}"))?; + + // Reconstruct the signing key from bytes + // Convert slice to array for SigningKey::from_bytes + let key_array: [u8; 32] = signing_key_bytes + .try_into() + .map_err(|_| anyhow::anyhow!("Invalid signing key length, expected 32 bytes"))?; + let signing_key = SigningKey::from_bytes(&key_array); + + let public_key_bytes = signing_key.verifying_key().to_bytes(); + let feed_key_bytes = public_key_bytes; + let feed_key = const_hex::encode(public_key_bytes); + + tracing::info!("Using hypercore feed key: {}", feed_key); + + // Reconstruct VerifyingKey from bytes for owned value + let public = VerifyingKey::from_bytes(&public_key_bytes) + .map_err(|e| anyhow::anyhow!("Failed to create verifying key: {e:?}"))?; + + // Create a PartialKeypair with both public and secret keys (for writable feed) + let key_pair = PartialKeypair { + public, + secret: Some(signing_key), + }; + + // Build hypercore with the generated keypair + let core = HypercoreBuilder::new(storage) + .key_pair(key_pair) + .build() + .await + .map_err(|e| anyhow::anyhow!("Failed to build hypercore: {e:?}"))?; + + // Set up hyperswarm for peer discovery + let topic = discovery_key(&public_key_bytes); + + let mut swarm = Hyperswarm::bind(build_swarm_config(hyperswarm_bootstrap.as_deref())) + .await + .map_err(|e| anyhow::anyhow!("Failed to bind hyperswarm: {e:?}"))?; + + // Announce and lookup for this feed's discovery key + swarm.configure(topic, TopicConfig::announce_and_lookup()); + + tracing::info!( + "Hyperswarm configured for discovery key: {}, topic: {:?}", + const_hex::encode(topic), + topic + ); + + let feed = Arc::new(Mutex::new(core)); + let swarm_feed = Arc::clone(&feed); + + // Spawn hyperswarm task to handle incoming connections + let feed_key_for_swarm = feed_key.clone(); + let feed_key_bytes_for_swarm = feed_key_bytes; + let connection_count_for_swarm = Arc::new(AtomicUsize::new(0)); + + // The hyperswarm DHT only executes announce/lookup once after + // bootstrapping. Periodically re-issue them so peers that announce + // later are still discovered. Stops once a peer connects. + let swarm_handle_for_relookup = swarm.handle(); + let connection_count_for_relookup = Arc::clone(&connection_count_for_swarm); + let relookup_handle = tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + interval.tick().await; // skip immediate first tick + loop { + interval.tick().await; + if connection_count_for_relookup.load(Ordering::Relaxed) > 0 { + tracing::debug!("Peer connected, stopping DHT re-lookup"); + break; + } + swarm_handle_for_relookup.configure(topic, TopicConfig::default()); + swarm_handle_for_relookup.configure(topic, TopicConfig::announce_and_lookup()); + } + }); + + // Clone the Arc for the spawned task (we keep the original for the struct) + let swarm_connection_count = Arc::clone(&connection_count_for_swarm); + let swarm_handle = tokio::spawn(async move { + let mut swarm = swarm; + tracing::info!( + "Hypercore swarm task started, listening for peers for feed_key: {}", + feed_key_for_swarm + ); + + use futures_lite::StreamExt; + while let Some(result) = swarm.next().await { + match result { + Ok(stream) => { + swarm_connection_count.fetch_add(1, Ordering::SeqCst); + tracing::debug!( + "Hyperswarm peer discovery attempt (initiator={}, peer_addr={:?}) for feed_key: {}", + stream.is_initiator(), + stream.peer_addr(), + feed_key_for_swarm + ); + tracing::info!( + "Hyperswarm connection established (initiator={}, peer_addr={:?}) for feed_key: {}", + stream.is_initiator(), + stream.peer_addr(), + feed_key_for_swarm + ); + let feed = Arc::clone(&swarm_feed); + let is_initiator = stream.is_initiator(); + let feed_key_bytes = feed_key_bytes_for_swarm; + + // Spawn a task for each peer connection + let connection_count_for_peer = Arc::clone(&swarm_connection_count); + tokio::spawn(async move { + let result = hypercore_protocol::run_protocol( + stream, + is_initiator, + feed, + feed_key_bytes, + None, + ) + .await; + + // Decrement connection count when peer connection closes + connection_count_for_peer.fetch_sub(1, Ordering::SeqCst); + + if let Err(err) = result { + tracing::error!( + "Hyperswarm connection failed for feed_key {}: {:?}", + const_hex::encode(feed_key_bytes), + err + ); + } else { + tracing::debug!( + "Hypercore protocol peer connection closed cleanly" + ); + } + }); + } + Err(err) => { + tracing::error!( + "Hyperswarm connection failed for feed_key {}: {:?}", + feed_key_for_swarm, + err + ); + } + } + } + + tracing::info!("Hypercore swarm task ended"); + }); + + Ok(Self { + feed, + feed_key, + swarm_handle, + relookup_handle, + _storage_dir: storage_dir, + connection_count_for_swarm, + }) + } + + /// Get the hex-encoded feed key (public key). + /// + /// This should be used when registering hypercore triggers + /// in service definitions. + pub fn feed_key(&self) -> String { + self.feed_key.clone() + } + + /// Get the current number of connected peers for this hypercore feed. + /// + /// This is used in tests to wait for mesh formation before proceeding. + pub fn connected_peer_count(&self) -> usize { + self.connection_count_for_swarm + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Append data to the hypercore feed. + /// + /// Returns the index of the appended block. + pub async fn append(&self, data: Vec) -> anyhow::Result { + let mut feed = self.feed.lock().await; + let outcome = feed + .append(&data) + .await + .map_err(|e| anyhow::anyhow!("Failed to append to hypercore: {e:?}"))?; + + // AppendOutcome contains the length, we need to return the index + let index = outcome.length.saturating_sub(1); + + tracing::info!( + "Appended {} bytes to hypercore feed at index {}", + data.len(), + index + ); + + Ok(index) + } +} + +fn build_swarm_config(hyperswarm_bootstrap: Option<&str>) -> SwarmConfig { + if let Some(addr) = hyperswarm_bootstrap { + match addr.parse::() { + Ok(addr) => { + tracing::info!("Using hyperswarm bootstrap: {}", addr); + return SwarmConfig::default() + .set_bootstrap_nodes(&[addr]) + .with_defaults(); + } + Err(err) => { + tracing::warn!("Invalid hyperswarm bootstrap address '{}': {err}", addr); + } + } + } + + SwarmConfig::all() +} diff --git a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs index 7a1012794..693a5b952 100644 --- a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs +++ b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs @@ -1,266 +1,299 @@ -//! Hypercore trigger stream for WAVS. -//! -//! Opens a hypercore, subscribes to append events, fetches new blocks, and -//! emits `StreamTriggers::Hypercore`. Replication uses Hyperswarm discovery -//! and spawns the replication protocol to ingest data. - -use ::hypercore_protocol::discovery_key; -use futures::Stream; -use hypercore::{replication::Event, Hypercore, HypercoreBuilder, PartialKeypair, Storage}; -use hyperswarm::{Config as SwarmConfig, Hyperswarm, TopicConfig}; -use std::net::SocketAddr; -use std::{path::PathBuf, pin::Pin, sync::Arc}; -use tokio::sync::Mutex; -use utils::telemetry::TriggerMetrics; - -use crate::subsystems::trigger::error::TriggerError; - -use super::{hypercore_protocol, StreamTriggers}; - -#[derive(Debug, Clone)] -pub struct HypercoreAppendEvent { - pub feed_key: String, - pub index: u64, - pub data: Vec, -} - -#[derive(Debug, Clone)] -pub struct HypercoreStreamConfig { - pub storage_dir: PathBuf, - pub feed_key: String, - pub hyperswarm_bootstrap: Option, -} - -pub async fn start_hypercore_stream( - config: HypercoreStreamConfig, - metrics: TriggerMetrics, - shutdown: tokio::sync::broadcast::Receiver<()>, -) -> Result< - ( - Pin> + Send>>, - tokio::sync::oneshot::Receiver<()>, - ), - TriggerError, -> { - std::fs::create_dir_all(&config.storage_dir).map_err(|err| { - TriggerError::Hypercore(format!( - "create storage dir {}: {}", - config.storage_dir.display(), - err - )) - })?; - - let storage = Storage::new_disk(&config.storage_dir, false) - .await - .map_err(|err| TriggerError::Hypercore(format!("open storage: {err:?}")))?; - - let (core, feed_key_bytes) = build_core_with_feed_key(storage, &config.feed_key).await?; - let feed_key = config.feed_key.clone(); - - let mut next_index = core.info().length; - let core = Arc::new(Mutex::new(core)); - let stream_core = Arc::clone(&core); - let mut receiver = { - let core = stream_core.lock().await; - core.event_subscribe() - }; - - let event_stream = async_stream::stream! { - loop { - match receiver.recv().await { - Ok(event) => match event { - Event::Have(have) => { - if have.drop { - continue; - } - let end = have.start.saturating_add(have.length); - for index in have.start..end { - if index < next_index { - continue; - } - let data = { - let mut core = stream_core.lock().await; - core.get(index).await - }; - match data { - Ok(Some(data)) => { - next_index = index.saturating_add(1); - tracing::info!( - "Hypercore append received: index={}, bytes={}", - index, - data.len() - ); - yield Ok(StreamTriggers::Hypercore { - event: HypercoreAppendEvent { - feed_key: feed_key.clone(), - index, - data, - }, - }); - } - Ok(None) => { - metrics.increment_total_errors("hypercore_missing_block"); - } - Err(err) => { - metrics.increment_total_errors("hypercore_get_error"); - yield Err(TriggerError::Hypercore(format!( - "hypercore get {}: {err:?}", - index - ))); - } - } - } - } - Event::DataUpgrade(_) | Event::Get(_) => {} - }, - Err(err) => { - metrics.increment_total_errors("hypercore_event_receive"); - yield Err(TriggerError::Hypercore(format!( - "hypercore event receive: {err:?}" - ))); - break; - } - } - } - }; - - let peer_connected_rx = start_swarm_replication( - feed_key_bytes, - Arc::clone(&core), - shutdown, - config.hyperswarm_bootstrap.clone(), - ) - .await?; - - Ok((Box::pin(event_stream), peer_connected_rx)) -} - -async fn build_core_with_feed_key( - storage: Storage, - feed_key_hex: &str, -) -> Result<(Hypercore, [u8; 32]), TriggerError> { - let feed_key_bytes = const_hex::decode(feed_key_hex.trim()) - .map_err(|err| TriggerError::Hypercore(format!("invalid feed key hex: {err:?}")))?; - let feed_key: [u8; 32] = feed_key_bytes - .as_slice() - .try_into() - .map_err(|_| TriggerError::Hypercore("invalid feed key length".to_string()))?; - let public = hypercore::VerifyingKey::from_bytes(&feed_key) - .map_err(|err| TriggerError::Hypercore(format!("invalid feed key: {err:?}")))?; - - let key_pair = PartialKeypair { - public, - secret: None, - }; - let core = HypercoreBuilder::new(storage) - .key_pair(key_pair) - .build() - .await - .map_err(|err| TriggerError::Hypercore(format!("build hypercore: {err:?}")))?; - - Ok((core, feed_key)) -} - -async fn start_swarm_replication( - feed_key: [u8; 32], - core: Arc>, - mut shutdown: tokio::sync::broadcast::Receiver<()>, - hyperswarm_bootstrap: Option, -) -> Result, TriggerError> { - let topic = discovery_key(&feed_key); - - tracing::info!( - "Starting hyperswarm replication for feed_key: {}, discovery_key: {:?}", - const_hex::encode(feed_key), - topic - ); - - let mut swarm = Hyperswarm::bind(build_swarm_config(hyperswarm_bootstrap.as_deref())) - .await - .map_err(|err| TriggerError::Hypercore(format!("bind hyperswarm: {err:?}")))?; - - swarm.configure(topic, TopicConfig::announce_and_lookup()); - - tracing::info!( - "Configured hyperswarm with announce_and_lookup for discovery_key: {:?}", - topic - ); - - let (replication_ready_tx, replication_ready_rx) = tokio::sync::oneshot::channel::<()>(); - - // Hyperswarm is async-std based but exposes futures-compatible streams, so it - // can be polled directly from the tokio runtime that owns hypercore. - tokio::spawn(async move { - tracing::info!("Hyperswarm task started, waiting for peer connections..."); - // Only the first peer's replication readiness is signalled. - let mut replication_ready_tx = Some(replication_ready_tx); - - loop { - tokio::select! { - _ = shutdown.recv() => { - tracing::info!("Hyperswarm task received shutdown signal"); - break; - } - stream = futures_lite::StreamExt::next(&mut swarm) => { - let stream = match stream { - Some(Ok(stream)) => stream, - Some(Err(err)) => { - tracing::warn!("Hyperswarm connection error: {err:?}"); - continue; - } - None => { - tracing::info!("Hyperswarm stream ended"); - break; - } - }; - - let peer_addr = stream.peer_addr(); - tracing::info!( - "Hyperswarm connection established (initiator={}, peer_addr={:?})", - stream.is_initiator(), - peer_addr - ); - - let replication_core = Arc::clone(&core); - let is_initiator = stream.is_initiator(); - // Pass the sender only to the first peer's protocol session. - let ready_tx = replication_ready_tx.take(); - - tokio::spawn(async move { - if let Err(err) = hypercore_protocol::run_protocol( - stream, - is_initiator, - replication_core, - feed_key, - ready_tx, - ) - .await - { - tracing::warn!("Hypercore protocol swarm peer error: {err:?}"); - } - }); - } - } - } - }); - - Ok(replication_ready_rx) -} - -fn build_swarm_config(hyperswarm_bootstrap: Option<&str>) -> SwarmConfig { - if let Some(addr) = hyperswarm_bootstrap { - match addr.parse::() { - Ok(addr) => { - tracing::info!("Using hyperswarm bootstrap: {}", addr); - return SwarmConfig::default() - .set_bootstrap_nodes(&[addr]) - .with_defaults(); - } - Err(err) => { - tracing::warn!("Invalid hyperswarm bootstrap address '{}': {err}", addr); - } - } - } - - SwarmConfig::all() -} +//! Hypercore trigger stream for WAVS. +//! +//! Opens a hypercore, subscribes to append events, fetches new blocks, and +//! emits `StreamTriggers::Hypercore`. Replication uses Hyperswarm discovery +//! and spawns the replication protocol to ingest data. + +use ::hypercore_protocol::discovery_key; +use futures::Stream; +use hypercore::{replication::Event, Hypercore, HypercoreBuilder, PartialKeypair, Storage}; +use hyperswarm::{Config as SwarmConfig, Hyperswarm, TopicConfig}; +use std::net::SocketAddr; +use std::{path::PathBuf, pin::Pin, sync::Arc, time::Duration}; +use tokio::sync::Mutex; +use utils::telemetry::TriggerMetrics; + +use crate::subsystems::trigger::error::TriggerError; + +use super::{hypercore_protocol, StreamTriggers}; + +#[derive(Debug, Clone)] +pub struct HypercoreAppendEvent { + pub feed_key: String, + pub index: u64, + pub data: Vec, +} + +#[derive(Debug, Clone)] +pub struct HypercoreStreamConfig { + pub storage_dir: PathBuf, + pub feed_key: String, + pub hyperswarm_bootstrap: Option, +} + +pub async fn start_hypercore_stream( + config: HypercoreStreamConfig, + metrics: TriggerMetrics, + shutdown: tokio::sync::broadcast::Receiver<()>, +) -> Result< + ( + Pin> + Send>>, + tokio::sync::oneshot::Receiver<()>, + ), + TriggerError, +> { + std::fs::create_dir_all(&config.storage_dir).map_err(|err| { + TriggerError::Hypercore(format!( + "create storage dir {}: {}", + config.storage_dir.display(), + err + )) + })?; + + let storage = Storage::new_disk(&config.storage_dir, false) + .await + .map_err(|err| TriggerError::Hypercore(format!("open storage: {err:?}")))?; + + let (core, feed_key_bytes) = build_core_with_feed_key(storage, &config.feed_key).await?; + let feed_key = config.feed_key.clone(); + + let mut next_index = core.info().length; + let core = Arc::new(Mutex::new(core)); + let stream_core = Arc::clone(&core); + let mut receiver = { + let core = stream_core.lock().await; + core.event_subscribe() + }; + + let event_stream = async_stream::stream! { + loop { + match receiver.recv().await { + Ok(event) => match event { + Event::Have(have) => { + if have.drop { + continue; + } + let end = have.start.saturating_add(have.length); + for index in have.start..end { + if index < next_index { + continue; + } + let data = { + let mut core = stream_core.lock().await; + core.get(index).await + }; + match data { + Ok(Some(data)) => { + next_index = index.saturating_add(1); + tracing::info!( + "Hypercore append received: index={}, bytes={}", + index, + data.len() + ); + yield Ok(StreamTriggers::Hypercore { + event: HypercoreAppendEvent { + feed_key: feed_key.clone(), + index, + data, + }, + }); + } + Ok(None) => { + metrics.increment_total_errors("hypercore_missing_block"); + } + Err(err) => { + metrics.increment_total_errors("hypercore_get_error"); + yield Err(TriggerError::Hypercore(format!( + "hypercore get {}: {err:?}", + index + ))); + } + } + } + } + Event::DataUpgrade(_) | Event::Get(_) => {} + }, + Err(err) => { + metrics.increment_total_errors("hypercore_event_receive"); + yield Err(TriggerError::Hypercore(format!( + "hypercore event receive: {err:?}" + ))); + break; + } + } + } + }; + + let peer_connected_rx = start_swarm_replication( + feed_key_bytes, + Arc::clone(&core), + shutdown, + config.hyperswarm_bootstrap.clone(), + ) + .await?; + + Ok((Box::pin(event_stream), peer_connected_rx)) +} + +async fn build_core_with_feed_key( + storage: Storage, + feed_key_hex: &str, +) -> Result<(Hypercore, [u8; 32]), TriggerError> { + let feed_key_bytes = const_hex::decode(feed_key_hex.trim()) + .map_err(|err| TriggerError::Hypercore(format!("invalid feed key hex: {err:?}")))?; + let feed_key: [u8; 32] = feed_key_bytes + .as_slice() + .try_into() + .map_err(|_| TriggerError::Hypercore("invalid feed key length".to_string()))?; + let public = hypercore::VerifyingKey::from_bytes(&feed_key) + .map_err(|err| TriggerError::Hypercore(format!("invalid feed key: {err:?}")))?; + + let key_pair = PartialKeypair { + public, + secret: None, + }; + let core = HypercoreBuilder::new(storage) + .key_pair(key_pair) + .build() + .await + .map_err(|err| TriggerError::Hypercore(format!("build hypercore: {err:?}")))?; + + Ok((core, feed_key)) +} + +async fn start_swarm_replication( + feed_key: [u8; 32], + core: Arc>, + mut shutdown: tokio::sync::broadcast::Receiver<()>, + hyperswarm_bootstrap: Option, +) -> Result, TriggerError> { + let topic = discovery_key(&feed_key); + + tracing::info!( + "Starting hyperswarm replication for feed_key: {}, discovery_key: {:?}", + const_hex::encode(feed_key), + topic + ); + + let mut swarm = Hyperswarm::bind(build_swarm_config(hyperswarm_bootstrap.as_deref())) + .await + .map_err(|err| TriggerError::Hypercore(format!("bind hyperswarm: {err:?}")))?; + + swarm.configure(topic, TopicConfig::announce_and_lookup()); + + tracing::info!( + "Configured hyperswarm with announce_and_lookup for discovery_key: {:?}", + topic + ); + + let (replication_ready_tx, replication_ready_rx) = tokio::sync::oneshot::channel::<()>(); + + // The hyperswarm DHT only executes announce/lookup commands once after + // bootstrapping. If no peers have announced yet at that point, the lookup + // finds nothing and never retries. Work around this by periodically + // re-issuing announce+lookup via the SwarmHandle while no peers are connected. + let peer_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let swarm_handle = swarm.handle(); + let relookup_shutdown = shutdown.resubscribe(); + tokio::spawn({ + let peer_count = Arc::clone(&peer_count); + let mut shutdown = relookup_shutdown; + async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + interval.tick().await; // skip immediate first tick + loop { + tokio::select! { + _ = shutdown.recv() => break, + _ = interval.tick() => { + if peer_count.load(std::sync::atomic::Ordering::Relaxed) > 0 { + continue; + } + // Clear then re-set to force new announce/lookup + swarm_handle.configure(topic, TopicConfig::default()); + swarm_handle.configure(topic, TopicConfig::announce_and_lookup()); + } + } + } + } + }); + + // Hyperswarm is async-std based but exposes futures-compatible streams, so it + // can be polled directly from the tokio runtime that owns hypercore. + tokio::spawn(async move { + tracing::info!("Hyperswarm task started, waiting for peer connections..."); + // Only the first peer's replication readiness is signalled. + let mut replication_ready_tx = Some(replication_ready_tx); + + loop { + tokio::select! { + _ = shutdown.recv() => { + tracing::info!("Hyperswarm task received shutdown signal"); + break; + } + stream = futures_lite::StreamExt::next(&mut swarm) => { + let stream = match stream { + Some(Ok(stream)) => stream, + Some(Err(err)) => { + tracing::warn!("Hyperswarm connection error: {err:?}"); + continue; + } + None => { + tracing::info!("Hyperswarm stream ended"); + break; + } + }; + + peer_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let peer_addr = stream.peer_addr(); + tracing::info!( + "Hyperswarm connection established (initiator={}, peer_addr={:?})", + stream.is_initiator(), + peer_addr + ); + + let replication_core = Arc::clone(&core); + let is_initiator = stream.is_initiator(); + // Pass the sender only to the first peer's protocol session. + let ready_tx = replication_ready_tx.take(); + let peer_count_for_protocol = Arc::clone(&peer_count); + + tokio::spawn(async move { + if let Err(err) = hypercore_protocol::run_protocol( + stream, + is_initiator, + replication_core, + feed_key, + ready_tx, + ) + .await + { + tracing::warn!("Hypercore protocol swarm peer error: {err:?}"); + } + peer_count_for_protocol.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + }); + } + } + } + }); + + Ok(replication_ready_rx) +} + +fn build_swarm_config(hyperswarm_bootstrap: Option<&str>) -> SwarmConfig { + if let Some(addr) = hyperswarm_bootstrap { + match addr.parse::() { + Ok(addr) => { + tracing::info!("Using hyperswarm bootstrap: {}", addr); + return SwarmConfig::default() + .set_bootstrap_nodes(&[addr]) + .with_defaults(); + } + Err(err) => { + tracing::warn!("Invalid hyperswarm bootstrap address '{}': {err}", addr); + } + } + } + + SwarmConfig::all() +} From 7be3d551f639d5a35d7a950b8cdcc100e3510c3f Mon Sep 17 00:00:00 2001 From: ismellike Date: Fri, 20 Feb 2026 14:11:03 -0600 Subject: [PATCH 07/10] cargo fmt --- packages/layer-tests/src/e2e/runner.rs | 1640 ++++++++++++------------ 1 file changed, 819 insertions(+), 821 deletions(-) diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index 3490f3df3..6dba35fb0 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -1,821 +1,819 @@ -// src/e2e/test_runner.rs - -use crate::deployment::ServiceDeployment; -use crate::e2e::config::Configs; -use crate::example_evm_client::example_trigger::ISimpleTrigger::TriggerInfo; -use crate::example_evm_client::example_trigger::NewTrigger; -use alloy_primitives::U256; -use alloy_provider::ext::AnvilApi; -use alloy_provider::Provider; -use alloy_sol_types::SolType; -use anyhow::{anyhow, bail, Context}; -use futures::{stream::FuturesUnordered, StreamExt}; -use ordermap::OrderMap; -use std::collections::HashMap; -use std::sync::Arc; -use std::time::{Duration, Instant}; -use utils::alloy_helpers::SolidityEventFinder; -use wavs_types::{ - AtProtoAction, ChainKeyNamespace, SimulatedTriggerRequest, Submit, Trigger, TriggerData, - Workflow, WorkflowId, -}; - -use crate::e2e::handles::hypercore::HypercoreClients; -use crate::e2e::helpers::wait_for_hypercore_streams_to_finalize; -use crate::e2e::helpers::{ - change_service_for_test, cosmos_wait_for_task_to_land, wait_for_hypercore_mesh_ready, -}; -use crate::e2e::report::TestReport; -use crate::e2e::service_managers::ServiceManagers; -use crate::e2e::test_definition::{ - AggregatorDefinition, ChangeServiceDefinition, SubmitDefinition, -}; -use crate::e2e::test_registry::CosmosCodeMap; -use crate::{ - e2e::{ - clients::Clients, components::ComponentSources, test_definition::TestDefinition, - test_registry::TestRegistry, - }, - example_cosmos_client::SimpleCosmosTriggerClient, - example_evm_client::{LogSpamClient, SimpleEvmTriggerClient, TriggerId}, -}; -use serde_json::json; - -use super::helpers::{evm_wait_for_task_to_land, simulate_anvil_reorg}; -use super::test_definition::WorkflowDefinition; - -/// Simplified test runner that leverages services directly attached to test definitions -pub struct Runner { - configs: Arc, - clients: Arc, - registry: Arc, - component_sources: Arc, - hypercore_clients: HypercoreClients, - service_managers: ServiceManagers, - cosmos_code_map: CosmosCodeMap, - report: TestReport, -} - -/// Extract service handler address from an aggregator submit configuration -fn extract_aggregator_service_handler(submit: &Submit) -> Option { - match submit { - Submit::Aggregator { component, .. } => { - component - .config - .get("service_handler") - .and_then(|addr_str| { - match layer_climb::prelude::CosmosAddr::new_str(addr_str, None) { - Ok(cosmos_addr) => Some(layer_climb::prelude::Address::Cosmos(cosmos_addr)), - Err(_) => layer_climb::prelude::EvmAddr::new_str(addr_str) - .ok() - .map(layer_climb::prelude::Address::from), - } - }) - } - _ => None, - } -} - -impl Runner { - #[allow(clippy::too_many_arguments)] - pub fn new( - configs: Configs, - clients: Clients, - registry: TestRegistry, - component_sources: ComponentSources, - hypercore_clients: HypercoreClients, - service_managers: ServiceManagers, - cosmos_code_map: CosmosCodeMap, - report: TestReport, - ) -> Self { - Self { - configs: Arc::new(configs), - clients: Arc::new(clients), - registry: Arc::new(registry), - component_sources: Arc::new(component_sources), - hypercore_clients, - service_managers, - cosmos_code_map, - report, - } - } - - /// Run all tests in the registry - pub async fn run_tests(&mut self, mut all_services: HashMap) { - let test_groups = self.registry.list_all_grouped(self.configs.grouping); - - for (group, mut group_tests) in test_groups { - let services = group_tests - .iter() - .map(|test| all_services.get(&test.name).cloned().unwrap().service) - .collect::>(); - - // This essentially deploys the services for the group - // since it updates the services to "Active" - // which is detected by wavs - self.service_managers - .update_services(&self.clients, services) - .await; - - // However, we have some tests which demonstrate more specific service changes - // and so we need to re-update those before we can proceed - // - // First we just deploy the service changes (contracts, components, etc.) - let mut futures = FuturesUnordered::new(); - for test in group_tests.iter() { - if let Some(change_service) = test.change_service.clone() { - let service = all_services.get(&test.name).cloned().unwrap().service; - let clients = self.clients.clone(); - let component_sources = self.component_sources.clone(); - let cosmos_code_map = self.cosmos_code_map.clone(); - futures.push(async move { - let mut service = service; - change_service_for_test( - &mut service, - change_service.clone(), - &clients, - &component_sources, - cosmos_code_map, - ) - .await; - (service, change_service) - }); - } - } - - // Then we need to deploy the update to service managers - if futures.is_empty() { - tracing::info!("No changes to services in group {:?}", group); - } else { - tracing::warn!("Running service changes for group {:?}", group); - let mut services_to_change = Vec::new(); - while let Some((service, change_service)) = futures.next().await { - // update our local copy of the service and handle changes - let service_deployment = all_services - .get_mut(&service.name) - .expect("Service should exist in all_services"); - - service_deployment.service = service.clone(); - - // and the definition so that tests know what to look for - match change_service { - ChangeServiceDefinition::AddWorkflow { - workflow_id, - workflow, - } => { - // When a workflow is added, it includes a new submission contract - // Extract it from the service's workflow that was just added - let submission_address = service_deployment - .service - .workflows - .get(&workflow_id) - .and_then(|workflow| { - extract_aggregator_service_handler(&workflow.submit) - }); - - if let Some(address) = submission_address { - service_deployment - .submission_handlers - .insert(workflow_id.clone(), address); - } - - group_tests - .iter_mut() - .find(|test| test.name == service.name) - .unwrap() - .workflows - .insert(workflow_id.clone(), workflow); - } - ChangeServiceDefinition::Component { - workflow_id, - component, - } => { - group_tests - .iter_mut() - .find(|test| test.name == service.name) - .unwrap() - .workflows - .get_mut(&workflow_id) - .unwrap() - .component = component; - } - } - - services_to_change.push(service); - } - - self.service_managers - .update_services(&self.clients, services_to_change) - .await; - } - - // Create hypercore clients AFTER deploying services so WAVS is already - // doing DHT lookups when the test client announces. This avoids the stale-DHT - // problem where the client announces 10+ seconds before WAVS starts looking. - self.hypercore_clients - .create_clients() - .await - .expect("Failed to create hypercore clients"); - - // All services are now deployed and ready for the tests - // From here on in we're strictly testing the trigger->execute->aggregate->submit flow - - tracing::info!("Running group {:?} with {} tests", group, group_tests.len()); - let mut futures = FuturesUnordered::new(); - let hypercore_clients = &self.hypercore_clients; - - for test in group_tests { - let clients = self.clients.clone(); - let component_sources = self.component_sources.clone(); - let test = test.clone(); - let report = self.report.clone(); - let service = all_services.get(&test.name).cloned().unwrap(); - futures.push(async move { - Self::execute_test( - &test, - service, - clients, - component_sources, - hypercore_clients, - report, - ) - .await - }); - } - - while (futures.next().await).is_some() {} - } - } - - // Execute a single test with timings - async fn execute_test( - test: &TestDefinition, - service_deployment: ServiceDeployment, - clients: Arc, - component_sources: Arc, - hypercore_clients: &HypercoreClients, - report: TestReport, - ) { - report.start_test(test.name.clone()); - - run_test( - test, - service_deployment, - &clients, - &component_sources, - hypercore_clients, - ) - .await - .context(test.name.clone()) - .unwrap(); - - report.end_test(test.name.clone()); - } -} - -/// Run a single test -async fn run_test( - test: &TestDefinition, - service_deployment: ServiceDeployment, - clients: &Clients, - component_sources: &ComponentSources, - hypercore_clients: &HypercoreClients, -) -> anyhow::Result<()> { - // For multi-operator tests, wait for P2P mesh to form before triggering - if test.multi_operator && clients.http_clients.len() > 1 { - let expected_peers = clients.http_clients.len() - 1; - tracing::info!( - "Multi-operator test: waiting for P2P mesh formation ({} expected peers)", - expected_peers - ); - - // Wait for all operators to have connected to peers - for (idx, http_client) in clients.http_clients.iter().enumerate() { - let status = http_client - .wait_for_p2p_ready(expected_peers, Some(Duration::from_secs(30))) - .await - .map_err(|e| { - anyhow!( - "Operator {} P2P readiness check failed: {}. \ - Multi-operator tests require P2P mesh to be ready.", - idx, - e - ) - })?; - tracing::info!( - "Operator {} P2P ready: {} connected peers", - idx, - status.connected_peers - ); - } - } - - // Group workflows by trigger to handle multi-triggers - let mut trigger_groups: OrderMap<&Trigger, Vec<(&WorkflowId, &Workflow)>> = OrderMap::new(); - - for (workflow_id, workflow) in service_deployment.service.workflows.iter() { - trigger_groups - .entry(&workflow.trigger) - .or_default() - .push((workflow_id, workflow)); - } - - // Process each unique trigger once, then validate all associated workflows - for (trigger, workflows_group) in trigger_groups { - // Use the first workflow to execute the trigger - let (first_workflow_id, _) = workflows_group[0]; - - // Get the workflow data safely - let first_workflow = test - .workflows - .get(first_workflow_id) - .ok_or(anyhow!("Could not get workflow: {}", first_workflow_id))?; - - // Convert input data to bytes safely - let input_bytes = first_workflow.input_data.to_bytes(); - - // Execute the trigger once - let mut reorg_snapshot: Option = None; - let trigger_ids = match trigger { - Trigger::EvmContractEvent { - chain, - address, - event_hash: _, - } => { - let evm_client = clients.get_evm_client(chain); - let client = SimpleEvmTriggerClient::new(evm_client.clone(), *address); - - if first_workflow.expects_reorg() { - reorg_snapshot = Some(evm_client.provider.anvil_snapshot().await?); - } - let input = input_bytes.clone().expect("EVM triggers require an input"); - - let spam_client = if first_workflow.trigger_execution.log_spam_count > 0 { - let address = super::helpers::deploy_log_spam_contract(clients, chain).await?; - let client = LogSpamClient::new(evm_client.clone(), address); - Some(client) - } else { - None - }; - - #[derive(Clone, Copy, Debug)] - enum TxKind { - Trigger, - Spam, - } - - let mut pending: Vec<(TxKind, alloy_primitives::TxHash)> = Vec::new(); - - let pending_trigger = client - .contract - .addTrigger(input.clone().into()) - .send() - .await?; - pending.push((TxKind::Trigger, *pending_trigger.tx_hash())); - - if let Some(spam_client) = &spam_client { - let spam_count = first_workflow.trigger_execution.log_spam_count as u64; - tracing::info!( - "Emitting {} bulk spam logs using LogSpam contract", - spam_count - ); - - // Use bulk emission to spam N logs in a single transaction - let spam_hash = spam_client.emit_spam(0, spam_count).await?; - - tracing::info!("Bulk spam transaction sent: {:?}", spam_hash); - pending.push((TxKind::Spam, spam_hash)); - } - - let start = Instant::now(); - let mut receipts = Vec::new(); - - while !pending.is_empty() { - let mut remaining = Vec::new(); - - for (kind, tx_hash) in pending.drain(..) { - tracing::debug!("Checking receipt for transaction: {:?}", tx_hash); - match evm_client.provider.get_transaction_receipt(tx_hash).await? { - Some(receipt) => { - receipts.push((kind, receipt)); - } - None => remaining.push((kind, tx_hash)), - } - } - - if start.elapsed() > Duration::from_secs(60) { - tracing::error!( - "Timeout waiting for transactions to be mined. Pending: {}, Mined: {}", - remaining.len(), - receipts.len() - ); - bail!("Timed out waiting for transactions to be mined"); - } - - pending = remaining; - } - - let mut trigger_ids = Vec::new(); - for (kind, receipt) in receipts { - if matches!(kind, TxKind::Trigger) { - if let Some(event) = - SolidityEventFinder::::solidity_event(&receipt) - { - let trigger_info = TriggerInfo::abi_decode(&event.triggerData)?; - trigger_ids.push(TriggerId::new(trigger_info.triggerId)); - } - } - } - - if trigger_ids.is_empty() { - bail!("Failed to obtain trigger id from transaction receipts"); - } - - tracing::info!( - "Successfully extracted {} trigger IDs: {:?}", - trigger_ids.len(), - trigger_ids - ); - trigger_ids - } - Trigger::CosmosContractEvent { - chain, - address, - event_type: _, - } => { - let client = SimpleCosmosTriggerClient::new( - clients.get_cosmos_client(chain).await, - address.clone().into(), - ); - - let trigger_id = client - .add_trigger(input_bytes.expect("Cosmos triggers require an input")) - .await?; - - vec![TriggerId::new(trigger_id.u64())] - } - Trigger::BlockInterval { .. } => vec![TriggerId::new(1337)], - Trigger::Cron { .. } => vec![TriggerId::new(1338)], - Trigger::AtProtoEvent { .. } => { - let sequence: u64 = 1339; - let trigger_id = TriggerId::new(sequence); - - let record_payload = input_bytes.clone().unwrap_or_default(); - let record_text = String::from_utf8_lossy(&record_payload).to_string(); - - // Send simulated trigger to all WAVS instances - for http_client in clients.http_clients.iter() { - let atproto_data = TriggerData::AtProtoEvent { - sequence: sequence as i64, - timestamp: 0, - repo: "did:example:alice".to_string(), - collection: "app.bsky.feed.post".to_string(), - rkey: "rkey-1".to_string(), - action: AtProtoAction::Create, - cid: Some("bafytestcid".to_string()), - record: Some(json!({ "text": record_text.clone() })), - rev: Some("rev-test".to_string()), - op_index: Some(0), - }; - - let req = SimulatedTriggerRequest { - service_id: service_deployment.service.id(), - workflow_id: first_workflow_id.clone(), - trigger: trigger.clone(), - data: atproto_data, - count: 1, - wait_for_completion: true, - }; - - http_client.simulate_trigger(req).await?; - } - - vec![trigger_id] - } - Trigger::HypercoreAppend { feed_key } => { - // Try to get the hypercore test client for this test - let payload = input_bytes.clone().unwrap_or_default(); - - tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); - - if let Some(hypercore_client) = hypercore_clients.get(&test.name) { - let client_feed_key = hypercore_client.feed_key(); - tracing::info!( - "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", - test.name, - client_feed_key, - feed_key - ); - - // Verify feed keys match before waiting for connectivity - if client_feed_key != *feed_key { - tracing::error!( - "FEED KEY MISMATCH! Client has: {}, Service has: {}", - client_feed_key, - feed_key - ); - return Err(anyhow::anyhow!( - "Feed key mismatch between client and service" - )); - } - - // Wait for all instances' hypercore streams AND the test client mesh - // to be ready concurrently. Both check the same underlying DHT - // connectivity from different sides, so running them in parallel - // avoids wasting the timeout budget on sequential per-instance waits. - let connectivity_timeout = Duration::from_secs(60); - let min_required_peers = 1; - let total_operators = clients.http_clients.len(); - - tracing::info!( - "Waiting for hypercore streams and mesh (min {} peer, {} total operators, timeout {}s)", - min_required_peers, - total_operators, - connectivity_timeout.as_secs(), - ); - - let stream_futs: Vec<_> = clients - .http_clients - .iter() - .enumerate() - .map(|(idx, http_client)| async move { - tracing::info!( - "Waiting for hypercore stream readiness on instance {} for feed_key {}", - idx, - feed_key - ); - wait_for_hypercore_streams_to_finalize( - http_client, - feed_key, - Some(connectivity_timeout), - ) - .await - .with_context(|| { - format!( - "Hypercore stream failed to finalize on instance {idx}" - ) - }) - }) - .collect(); - - let mesh_fut = wait_for_hypercore_mesh_ready( - &hypercore_client, - min_required_peers, - connectivity_timeout, - ); - - let (streams_result, mesh_result) = tokio::join!( - futures::future::try_join_all(stream_futs), - mesh_fut, - ); - - streams_result - .context("Failed to wait for hypercore streams to finalize")?; - let peer_count = mesh_result - .context("Hypercore mesh not ready: 0 peers connected, append will never replicate")?; - - tracing::info!( - "Hypercore streams and mesh ready: {} connected peers (min required: {}, total operators: {})", - peer_count, - min_required_peers, - total_operators - ); - - // Append data to the hypercore feed - tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); - let index = hypercore_client.append(payload).await?; - - vec![TriggerId::new(index)] - } else { - // Fallback to simulated trigger for backward compatibility - tracing::warn!( - "No hypercore client found for test '{}', using simulated trigger", - test.name - ); - - let trigger_id = TriggerId::new(0); - let hypercore_data = TriggerData::HypercoreAppend { - feed_key: feed_key.clone(), - index: trigger_id.u64(), - data: payload, - }; - - let req = SimulatedTriggerRequest { - service_id: service_deployment.service.id(), - workflow_id: first_workflow_id.clone(), - trigger: trigger.clone(), - data: hypercore_data, - count: 1, - wait_for_completion: true, - }; - - let http_client = clients - .http_clients - .first() - .ok_or_else(|| anyhow!("No HTTP clients available"))?; - http_client.simulate_trigger(req).await?; - - vec![trigger_id] - } - } - Trigger::Manual => unimplemented!("Manual trigger type is not implemented"), - }; - - tracing::info!( - "Starting workflow validation for {} workflows", - workflows_group.len() - ); - // Validate all workflows associated with this trigger - for (workflow_id, workflow) in workflows_group { - tracing::info!("Validating workflow: {}", workflow_id); - let WorkflowDefinition { - timeout, - expected_output, - .. - } = &test.workflows.get(workflow_id).ok_or(anyhow!( - "Could not get workflow definition from id: {}", - workflow_id - ))?; - - for trigger_id in trigger_ids.iter().copied() { - tracing::info!( - "Processing trigger_id: {} for workflow: {}", - trigger_id, - workflow_id - ); - let data = match &workflow.submit { - Submit::Aggregator { .. } => { - let workflow_def = test.workflows.get(workflow_id).ok_or_else(|| { - anyhow!("Could not get workflow definition from id: {}", workflow_id) - })?; - - let SubmitDefinition::Aggregator(aggregator) = &workflow_def.submit; - let AggregatorDefinition::ComponentBasedAggregator { chain, .. } = - aggregator; - - match chain.namespace.as_str() { - ChainKeyNamespace::COSMOS => { - let client = clients.get_cosmos_client(chain).await; - let submission_contract = service_deployment - .submission_handlers - .get(workflow_id) - .ok_or_else(|| { - anyhow!( - "No submission contract found for workflow {}", - workflow_id - ) - })?; - - let data = cosmos_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - *timeout, - ) - .await?; - - tracing::info!("Task result: {:?}", data); - - data - } - ChainKeyNamespace::EVM => { - let client = clients.get_evm_client(chain); - tracing::info!( - "Getting submit start block for workflow: {}", - workflow_id - ); - let submit_start_block = - client.provider.get_block_number().await.map_err(|e| { - anyhow!("Failed to get block number: {}", e) - })?; - tracing::info!("Submit start block: {}", submit_start_block); - - let submission_contract = service_deployment - .submission_handlers - .get(workflow_id) - .ok_or_else(|| { - anyhow!( - "No submission contract found for workflow {}", - workflow_id - ) - })?; - tracing::info!( - "Submission contract for workflow {}: {}", - workflow_id, - submission_contract - ); - - if first_workflow.expects_reorg() { - tracing::info!("Test '{}' will simulate re-org", test.name); - - // Simulate re-org before waiting for task - simulate_anvil_reorg( - &client, - reorg_snapshot.expect( - "Expected a reorg snapshot when simulating reorg", - ), - ) - .await?; - - // Wait for task - should return empty data on error due to re-org - tracing::info!( - "Waiting for task to land after re-org for trigger_id: {}", - trigger_id - ); - let result = evm_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - submit_start_block, - *timeout, - ) - .await; - - match result { - Ok(signed_data) => signed_data.data.to_vec(), - // If we get an error (transaction dropped due to re-org), - // return mocked signed data with empty content to match ExpectedOutput::Dropped - Err(_) => Vec::new(), - } - } else { - tracing::info!( - "Waiting for task to land (no re-org) for trigger_id: {}", - trigger_id - ); - let result = evm_wait_for_task_to_land( - client, - submission_contract.clone().try_into().unwrap(), - trigger_id, - submit_start_block, - *timeout, - ) - .await?; - tracing::info!("Task result (no re-org): {:?}", result.data); - result.data.to_vec() - } - } - _ => unimplemented!("Unsupported chain namespace for aggregator"), - } - } - Submit::None => unimplemented!("Submit::None is not implemented"), - }; - - tracing::info!("Validating expected output for workflow: {}", workflow_id); - expected_output.validate(test, clients, component_sources, &data)?; - tracing::info!( - "Successfully validated output for workflow: {}", - workflow_id - ); - } - } - tracing::info!("Test completed successfully!"); - } - - // Wait for the aggregator submit callback to complete on all WAVS instances - // before cleaning up the service. This ensures the after-submit callback - // has finished writing to the KV store. - // Only do this if: - // 1. Any workflow uses an aggregator submit - // 2. No workflow expects dropped output (e.g., reorg tests where submission is intentionally skipped) - let has_aggregator = service_deployment - .service - .workflows - .values() - .any(|w| matches!(w.submit, Submit::Aggregator { .. })); - - let expects_dropped = test.workflows.values().any(|w| w.expects_reorg()); - - if has_aggregator && !expects_dropped { - let service_id = service_deployment.service.id().to_string(); - tracing::info!( - "Waiting for submit callback to complete for service: {}", - service_id - ); - for (idx, http_client) in clients.http_clients.iter().enumerate() { - http_client - .wait_for_submit_callback(&service_id, None) - .await - .map_err(|e| { - anyhow!("Instance {} failed waiting for submit callback: {}", idx, e) - })?; - tracing::info!( - "Submit callback completed on instance {} for service: {}", - idx, - service_id - ); - } - } - - tracing::info!( - "Cleaning up service: {0:?}", - service_deployment.service.manager - ); - // Delete service from all WAVS instances - for http_client in clients.http_clients.iter() { - http_client - .delete_service(vec![service_deployment.service.manager.clone()]) - .await?; - } - - Ok(()) -} +// src/e2e/test_runner.rs + +use crate::deployment::ServiceDeployment; +use crate::e2e::config::Configs; +use crate::example_evm_client::example_trigger::ISimpleTrigger::TriggerInfo; +use crate::example_evm_client::example_trigger::NewTrigger; +use alloy_primitives::U256; +use alloy_provider::ext::AnvilApi; +use alloy_provider::Provider; +use alloy_sol_types::SolType; +use anyhow::{anyhow, bail, Context}; +use futures::{stream::FuturesUnordered, StreamExt}; +use ordermap::OrderMap; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use utils::alloy_helpers::SolidityEventFinder; +use wavs_types::{ + AtProtoAction, ChainKeyNamespace, SimulatedTriggerRequest, Submit, Trigger, TriggerData, + Workflow, WorkflowId, +}; + +use crate::e2e::handles::hypercore::HypercoreClients; +use crate::e2e::helpers::wait_for_hypercore_streams_to_finalize; +use crate::e2e::helpers::{ + change_service_for_test, cosmos_wait_for_task_to_land, wait_for_hypercore_mesh_ready, +}; +use crate::e2e::report::TestReport; +use crate::e2e::service_managers::ServiceManagers; +use crate::e2e::test_definition::{ + AggregatorDefinition, ChangeServiceDefinition, SubmitDefinition, +}; +use crate::e2e::test_registry::CosmosCodeMap; +use crate::{ + e2e::{ + clients::Clients, components::ComponentSources, test_definition::TestDefinition, + test_registry::TestRegistry, + }, + example_cosmos_client::SimpleCosmosTriggerClient, + example_evm_client::{LogSpamClient, SimpleEvmTriggerClient, TriggerId}, +}; +use serde_json::json; + +use super::helpers::{evm_wait_for_task_to_land, simulate_anvil_reorg}; +use super::test_definition::WorkflowDefinition; + +/// Simplified test runner that leverages services directly attached to test definitions +pub struct Runner { + configs: Arc, + clients: Arc, + registry: Arc, + component_sources: Arc, + hypercore_clients: HypercoreClients, + service_managers: ServiceManagers, + cosmos_code_map: CosmosCodeMap, + report: TestReport, +} + +/// Extract service handler address from an aggregator submit configuration +fn extract_aggregator_service_handler(submit: &Submit) -> Option { + match submit { + Submit::Aggregator { component, .. } => { + component + .config + .get("service_handler") + .and_then(|addr_str| { + match layer_climb::prelude::CosmosAddr::new_str(addr_str, None) { + Ok(cosmos_addr) => Some(layer_climb::prelude::Address::Cosmos(cosmos_addr)), + Err(_) => layer_climb::prelude::EvmAddr::new_str(addr_str) + .ok() + .map(layer_climb::prelude::Address::from), + } + }) + } + _ => None, + } +} + +impl Runner { + #[allow(clippy::too_many_arguments)] + pub fn new( + configs: Configs, + clients: Clients, + registry: TestRegistry, + component_sources: ComponentSources, + hypercore_clients: HypercoreClients, + service_managers: ServiceManagers, + cosmos_code_map: CosmosCodeMap, + report: TestReport, + ) -> Self { + Self { + configs: Arc::new(configs), + clients: Arc::new(clients), + registry: Arc::new(registry), + component_sources: Arc::new(component_sources), + hypercore_clients, + service_managers, + cosmos_code_map, + report, + } + } + + /// Run all tests in the registry + pub async fn run_tests(&mut self, mut all_services: HashMap) { + let test_groups = self.registry.list_all_grouped(self.configs.grouping); + + for (group, mut group_tests) in test_groups { + let services = group_tests + .iter() + .map(|test| all_services.get(&test.name).cloned().unwrap().service) + .collect::>(); + + // This essentially deploys the services for the group + // since it updates the services to "Active" + // which is detected by wavs + self.service_managers + .update_services(&self.clients, services) + .await; + + // However, we have some tests which demonstrate more specific service changes + // and so we need to re-update those before we can proceed + // + // First we just deploy the service changes (contracts, components, etc.) + let mut futures = FuturesUnordered::new(); + for test in group_tests.iter() { + if let Some(change_service) = test.change_service.clone() { + let service = all_services.get(&test.name).cloned().unwrap().service; + let clients = self.clients.clone(); + let component_sources = self.component_sources.clone(); + let cosmos_code_map = self.cosmos_code_map.clone(); + futures.push(async move { + let mut service = service; + change_service_for_test( + &mut service, + change_service.clone(), + &clients, + &component_sources, + cosmos_code_map, + ) + .await; + (service, change_service) + }); + } + } + + // Then we need to deploy the update to service managers + if futures.is_empty() { + tracing::info!("No changes to services in group {:?}", group); + } else { + tracing::warn!("Running service changes for group {:?}", group); + let mut services_to_change = Vec::new(); + while let Some((service, change_service)) = futures.next().await { + // update our local copy of the service and handle changes + let service_deployment = all_services + .get_mut(&service.name) + .expect("Service should exist in all_services"); + + service_deployment.service = service.clone(); + + // and the definition so that tests know what to look for + match change_service { + ChangeServiceDefinition::AddWorkflow { + workflow_id, + workflow, + } => { + // When a workflow is added, it includes a new submission contract + // Extract it from the service's workflow that was just added + let submission_address = service_deployment + .service + .workflows + .get(&workflow_id) + .and_then(|workflow| { + extract_aggregator_service_handler(&workflow.submit) + }); + + if let Some(address) = submission_address { + service_deployment + .submission_handlers + .insert(workflow_id.clone(), address); + } + + group_tests + .iter_mut() + .find(|test| test.name == service.name) + .unwrap() + .workflows + .insert(workflow_id.clone(), workflow); + } + ChangeServiceDefinition::Component { + workflow_id, + component, + } => { + group_tests + .iter_mut() + .find(|test| test.name == service.name) + .unwrap() + .workflows + .get_mut(&workflow_id) + .unwrap() + .component = component; + } + } + + services_to_change.push(service); + } + + self.service_managers + .update_services(&self.clients, services_to_change) + .await; + } + + // Create hypercore clients AFTER deploying services so WAVS is already + // doing DHT lookups when the test client announces. This avoids the stale-DHT + // problem where the client announces 10+ seconds before WAVS starts looking. + self.hypercore_clients + .create_clients() + .await + .expect("Failed to create hypercore clients"); + + // All services are now deployed and ready for the tests + // From here on in we're strictly testing the trigger->execute->aggregate->submit flow + + tracing::info!("Running group {:?} with {} tests", group, group_tests.len()); + let mut futures = FuturesUnordered::new(); + let hypercore_clients = &self.hypercore_clients; + + for test in group_tests { + let clients = self.clients.clone(); + let component_sources = self.component_sources.clone(); + let test = test.clone(); + let report = self.report.clone(); + let service = all_services.get(&test.name).cloned().unwrap(); + futures.push(async move { + Self::execute_test( + &test, + service, + clients, + component_sources, + hypercore_clients, + report, + ) + .await + }); + } + + while (futures.next().await).is_some() {} + } + } + + // Execute a single test with timings + async fn execute_test( + test: &TestDefinition, + service_deployment: ServiceDeployment, + clients: Arc, + component_sources: Arc, + hypercore_clients: &HypercoreClients, + report: TestReport, + ) { + report.start_test(test.name.clone()); + + run_test( + test, + service_deployment, + &clients, + &component_sources, + hypercore_clients, + ) + .await + .context(test.name.clone()) + .unwrap(); + + report.end_test(test.name.clone()); + } +} + +/// Run a single test +async fn run_test( + test: &TestDefinition, + service_deployment: ServiceDeployment, + clients: &Clients, + component_sources: &ComponentSources, + hypercore_clients: &HypercoreClients, +) -> anyhow::Result<()> { + // For multi-operator tests, wait for P2P mesh to form before triggering + if test.multi_operator && clients.http_clients.len() > 1 { + let expected_peers = clients.http_clients.len() - 1; + tracing::info!( + "Multi-operator test: waiting for P2P mesh formation ({} expected peers)", + expected_peers + ); + + // Wait for all operators to have connected to peers + for (idx, http_client) in clients.http_clients.iter().enumerate() { + let status = http_client + .wait_for_p2p_ready(expected_peers, Some(Duration::from_secs(30))) + .await + .map_err(|e| { + anyhow!( + "Operator {} P2P readiness check failed: {}. \ + Multi-operator tests require P2P mesh to be ready.", + idx, + e + ) + })?; + tracing::info!( + "Operator {} P2P ready: {} connected peers", + idx, + status.connected_peers + ); + } + } + + // Group workflows by trigger to handle multi-triggers + let mut trigger_groups: OrderMap<&Trigger, Vec<(&WorkflowId, &Workflow)>> = OrderMap::new(); + + for (workflow_id, workflow) in service_deployment.service.workflows.iter() { + trigger_groups + .entry(&workflow.trigger) + .or_default() + .push((workflow_id, workflow)); + } + + // Process each unique trigger once, then validate all associated workflows + for (trigger, workflows_group) in trigger_groups { + // Use the first workflow to execute the trigger + let (first_workflow_id, _) = workflows_group[0]; + + // Get the workflow data safely + let first_workflow = test + .workflows + .get(first_workflow_id) + .ok_or(anyhow!("Could not get workflow: {}", first_workflow_id))?; + + // Convert input data to bytes safely + let input_bytes = first_workflow.input_data.to_bytes(); + + // Execute the trigger once + let mut reorg_snapshot: Option = None; + let trigger_ids = match trigger { + Trigger::EvmContractEvent { + chain, + address, + event_hash: _, + } => { + let evm_client = clients.get_evm_client(chain); + let client = SimpleEvmTriggerClient::new(evm_client.clone(), *address); + + if first_workflow.expects_reorg() { + reorg_snapshot = Some(evm_client.provider.anvil_snapshot().await?); + } + let input = input_bytes.clone().expect("EVM triggers require an input"); + + let spam_client = if first_workflow.trigger_execution.log_spam_count > 0 { + let address = super::helpers::deploy_log_spam_contract(clients, chain).await?; + let client = LogSpamClient::new(evm_client.clone(), address); + Some(client) + } else { + None + }; + + #[derive(Clone, Copy, Debug)] + enum TxKind { + Trigger, + Spam, + } + + let mut pending: Vec<(TxKind, alloy_primitives::TxHash)> = Vec::new(); + + let pending_trigger = client + .contract + .addTrigger(input.clone().into()) + .send() + .await?; + pending.push((TxKind::Trigger, *pending_trigger.tx_hash())); + + if let Some(spam_client) = &spam_client { + let spam_count = first_workflow.trigger_execution.log_spam_count as u64; + tracing::info!( + "Emitting {} bulk spam logs using LogSpam contract", + spam_count + ); + + // Use bulk emission to spam N logs in a single transaction + let spam_hash = spam_client.emit_spam(0, spam_count).await?; + + tracing::info!("Bulk spam transaction sent: {:?}", spam_hash); + pending.push((TxKind::Spam, spam_hash)); + } + + let start = Instant::now(); + let mut receipts = Vec::new(); + + while !pending.is_empty() { + let mut remaining = Vec::new(); + + for (kind, tx_hash) in pending.drain(..) { + tracing::debug!("Checking receipt for transaction: {:?}", tx_hash); + match evm_client.provider.get_transaction_receipt(tx_hash).await? { + Some(receipt) => { + receipts.push((kind, receipt)); + } + None => remaining.push((kind, tx_hash)), + } + } + + if start.elapsed() > Duration::from_secs(60) { + tracing::error!( + "Timeout waiting for transactions to be mined. Pending: {}, Mined: {}", + remaining.len(), + receipts.len() + ); + bail!("Timed out waiting for transactions to be mined"); + } + + pending = remaining; + } + + let mut trigger_ids = Vec::new(); + for (kind, receipt) in receipts { + if matches!(kind, TxKind::Trigger) { + if let Some(event) = + SolidityEventFinder::::solidity_event(&receipt) + { + let trigger_info = TriggerInfo::abi_decode(&event.triggerData)?; + trigger_ids.push(TriggerId::new(trigger_info.triggerId)); + } + } + } + + if trigger_ids.is_empty() { + bail!("Failed to obtain trigger id from transaction receipts"); + } + + tracing::info!( + "Successfully extracted {} trigger IDs: {:?}", + trigger_ids.len(), + trigger_ids + ); + trigger_ids + } + Trigger::CosmosContractEvent { + chain, + address, + event_type: _, + } => { + let client = SimpleCosmosTriggerClient::new( + clients.get_cosmos_client(chain).await, + address.clone().into(), + ); + + let trigger_id = client + .add_trigger(input_bytes.expect("Cosmos triggers require an input")) + .await?; + + vec![TriggerId::new(trigger_id.u64())] + } + Trigger::BlockInterval { .. } => vec![TriggerId::new(1337)], + Trigger::Cron { .. } => vec![TriggerId::new(1338)], + Trigger::AtProtoEvent { .. } => { + let sequence: u64 = 1339; + let trigger_id = TriggerId::new(sequence); + + let record_payload = input_bytes.clone().unwrap_or_default(); + let record_text = String::from_utf8_lossy(&record_payload).to_string(); + + // Send simulated trigger to all WAVS instances + for http_client in clients.http_clients.iter() { + let atproto_data = TriggerData::AtProtoEvent { + sequence: sequence as i64, + timestamp: 0, + repo: "did:example:alice".to_string(), + collection: "app.bsky.feed.post".to_string(), + rkey: "rkey-1".to_string(), + action: AtProtoAction::Create, + cid: Some("bafytestcid".to_string()), + record: Some(json!({ "text": record_text.clone() })), + rev: Some("rev-test".to_string()), + op_index: Some(0), + }; + + let req = SimulatedTriggerRequest { + service_id: service_deployment.service.id(), + workflow_id: first_workflow_id.clone(), + trigger: trigger.clone(), + data: atproto_data, + count: 1, + wait_for_completion: true, + }; + + http_client.simulate_trigger(req).await?; + } + + vec![trigger_id] + } + Trigger::HypercoreAppend { feed_key } => { + // Try to get the hypercore test client for this test + let payload = input_bytes.clone().unwrap_or_default(); + + tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); + + if let Some(hypercore_client) = hypercore_clients.get(&test.name) { + let client_feed_key = hypercore_client.feed_key(); + tracing::info!( + "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", + test.name, + client_feed_key, + feed_key + ); + + // Verify feed keys match before waiting for connectivity + if client_feed_key != *feed_key { + tracing::error!( + "FEED KEY MISMATCH! Client has: {}, Service has: {}", + client_feed_key, + feed_key + ); + return Err(anyhow::anyhow!( + "Feed key mismatch between client and service" + )); + } + + // Wait for all instances' hypercore streams AND the test client mesh + // to be ready concurrently. Both check the same underlying DHT + // connectivity from different sides, so running them in parallel + // avoids wasting the timeout budget on sequential per-instance waits. + let connectivity_timeout = Duration::from_secs(60); + let min_required_peers = 1; + let total_operators = clients.http_clients.len(); + + tracing::info!( + "Waiting for hypercore streams and mesh (min {} peer, {} total operators, timeout {}s)", + min_required_peers, + total_operators, + connectivity_timeout.as_secs(), + ); + + let stream_futs: Vec<_> = clients + .http_clients + .iter() + .enumerate() + .map(|(idx, http_client)| async move { + tracing::info!( + "Waiting for hypercore stream readiness on instance {} for feed_key {}", + idx, + feed_key + ); + wait_for_hypercore_streams_to_finalize( + http_client, + feed_key, + Some(connectivity_timeout), + ) + .await + .with_context(|| { + format!( + "Hypercore stream failed to finalize on instance {idx}" + ) + }) + }) + .collect(); + + let mesh_fut = wait_for_hypercore_mesh_ready( + &hypercore_client, + min_required_peers, + connectivity_timeout, + ); + + let (streams_result, mesh_result) = + tokio::join!(futures::future::try_join_all(stream_futs), mesh_fut,); + + streams_result.context("Failed to wait for hypercore streams to finalize")?; + let peer_count = mesh_result.context( + "Hypercore mesh not ready: 0 peers connected, append will never replicate", + )?; + + tracing::info!( + "Hypercore streams and mesh ready: {} connected peers (min required: {}, total operators: {})", + peer_count, + min_required_peers, + total_operators + ); + + // Append data to the hypercore feed + tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); + let index = hypercore_client.append(payload).await?; + + vec![TriggerId::new(index)] + } else { + // Fallback to simulated trigger for backward compatibility + tracing::warn!( + "No hypercore client found for test '{}', using simulated trigger", + test.name + ); + + let trigger_id = TriggerId::new(0); + let hypercore_data = TriggerData::HypercoreAppend { + feed_key: feed_key.clone(), + index: trigger_id.u64(), + data: payload, + }; + + let req = SimulatedTriggerRequest { + service_id: service_deployment.service.id(), + workflow_id: first_workflow_id.clone(), + trigger: trigger.clone(), + data: hypercore_data, + count: 1, + wait_for_completion: true, + }; + + let http_client = clients + .http_clients + .first() + .ok_or_else(|| anyhow!("No HTTP clients available"))?; + http_client.simulate_trigger(req).await?; + + vec![trigger_id] + } + } + Trigger::Manual => unimplemented!("Manual trigger type is not implemented"), + }; + + tracing::info!( + "Starting workflow validation for {} workflows", + workflows_group.len() + ); + // Validate all workflows associated with this trigger + for (workflow_id, workflow) in workflows_group { + tracing::info!("Validating workflow: {}", workflow_id); + let WorkflowDefinition { + timeout, + expected_output, + .. + } = &test.workflows.get(workflow_id).ok_or(anyhow!( + "Could not get workflow definition from id: {}", + workflow_id + ))?; + + for trigger_id in trigger_ids.iter().copied() { + tracing::info!( + "Processing trigger_id: {} for workflow: {}", + trigger_id, + workflow_id + ); + let data = match &workflow.submit { + Submit::Aggregator { .. } => { + let workflow_def = test.workflows.get(workflow_id).ok_or_else(|| { + anyhow!("Could not get workflow definition from id: {}", workflow_id) + })?; + + let SubmitDefinition::Aggregator(aggregator) = &workflow_def.submit; + let AggregatorDefinition::ComponentBasedAggregator { chain, .. } = + aggregator; + + match chain.namespace.as_str() { + ChainKeyNamespace::COSMOS => { + let client = clients.get_cosmos_client(chain).await; + let submission_contract = service_deployment + .submission_handlers + .get(workflow_id) + .ok_or_else(|| { + anyhow!( + "No submission contract found for workflow {}", + workflow_id + ) + })?; + + let data = cosmos_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + *timeout, + ) + .await?; + + tracing::info!("Task result: {:?}", data); + + data + } + ChainKeyNamespace::EVM => { + let client = clients.get_evm_client(chain); + tracing::info!( + "Getting submit start block for workflow: {}", + workflow_id + ); + let submit_start_block = + client.provider.get_block_number().await.map_err(|e| { + anyhow!("Failed to get block number: {}", e) + })?; + tracing::info!("Submit start block: {}", submit_start_block); + + let submission_contract = service_deployment + .submission_handlers + .get(workflow_id) + .ok_or_else(|| { + anyhow!( + "No submission contract found for workflow {}", + workflow_id + ) + })?; + tracing::info!( + "Submission contract for workflow {}: {}", + workflow_id, + submission_contract + ); + + if first_workflow.expects_reorg() { + tracing::info!("Test '{}' will simulate re-org", test.name); + + // Simulate re-org before waiting for task + simulate_anvil_reorg( + &client, + reorg_snapshot.expect( + "Expected a reorg snapshot when simulating reorg", + ), + ) + .await?; + + // Wait for task - should return empty data on error due to re-org + tracing::info!( + "Waiting for task to land after re-org for trigger_id: {}", + trigger_id + ); + let result = evm_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + submit_start_block, + *timeout, + ) + .await; + + match result { + Ok(signed_data) => signed_data.data.to_vec(), + // If we get an error (transaction dropped due to re-org), + // return mocked signed data with empty content to match ExpectedOutput::Dropped + Err(_) => Vec::new(), + } + } else { + tracing::info!( + "Waiting for task to land (no re-org) for trigger_id: {}", + trigger_id + ); + let result = evm_wait_for_task_to_land( + client, + submission_contract.clone().try_into().unwrap(), + trigger_id, + submit_start_block, + *timeout, + ) + .await?; + tracing::info!("Task result (no re-org): {:?}", result.data); + result.data.to_vec() + } + } + _ => unimplemented!("Unsupported chain namespace for aggregator"), + } + } + Submit::None => unimplemented!("Submit::None is not implemented"), + }; + + tracing::info!("Validating expected output for workflow: {}", workflow_id); + expected_output.validate(test, clients, component_sources, &data)?; + tracing::info!( + "Successfully validated output for workflow: {}", + workflow_id + ); + } + } + tracing::info!("Test completed successfully!"); + } + + // Wait for the aggregator submit callback to complete on all WAVS instances + // before cleaning up the service. This ensures the after-submit callback + // has finished writing to the KV store. + // Only do this if: + // 1. Any workflow uses an aggregator submit + // 2. No workflow expects dropped output (e.g., reorg tests where submission is intentionally skipped) + let has_aggregator = service_deployment + .service + .workflows + .values() + .any(|w| matches!(w.submit, Submit::Aggregator { .. })); + + let expects_dropped = test.workflows.values().any(|w| w.expects_reorg()); + + if has_aggregator && !expects_dropped { + let service_id = service_deployment.service.id().to_string(); + tracing::info!( + "Waiting for submit callback to complete for service: {}", + service_id + ); + for (idx, http_client) in clients.http_clients.iter().enumerate() { + http_client + .wait_for_submit_callback(&service_id, None) + .await + .map_err(|e| { + anyhow!("Instance {} failed waiting for submit callback: {}", idx, e) + })?; + tracing::info!( + "Submit callback completed on instance {} for service: {}", + idx, + service_id + ); + } + } + + tracing::info!( + "Cleaning up service: {0:?}", + service_deployment.service.manager + ); + // Delete service from all WAVS instances + for http_client in clients.http_clients.iter() { + http_client + .delete_service(vec![service_deployment.service.manager.clone()]) + .await?; + } + + Ok(()) +} From 9084fb0db277979a7e0debad20753f40b776c988 Mon Sep 17 00:00:00 2001 From: ismellike Date: Thu, 19 Mar 2026 13:33:38 -0500 Subject: [PATCH 08/10] Do not fallback to simulated trigger --- packages/layer-tests/src/e2e/runner.rs | 152 ++++++++++--------------- 1 file changed, 61 insertions(+), 91 deletions(-) diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index 6dba35fb0..a393b6a07 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -498,123 +498,93 @@ async fn run_test( tracing::info!("Hypercore trigger detected with feed_key: {}", feed_key); - if let Some(hypercore_client) = hypercore_clients.get(&test.name) { - let client_feed_key = hypercore_client.feed_key(); - tracing::info!( + let hypercore_client = hypercore_clients.get(&test.name).ok_or(anyhow::anyhow!( + "No hypercore client found for test '{}'.", + test.name + ))?; + let client_feed_key = hypercore_client.feed_key(); + tracing::info!( "Using real hypercore feed for test '{}', client feed_key: {}, service feed_key: {}", test.name, client_feed_key, feed_key ); - // Verify feed keys match before waiting for connectivity - if client_feed_key != *feed_key { - tracing::error!( - "FEED KEY MISMATCH! Client has: {}, Service has: {}", - client_feed_key, - feed_key - ); - return Err(anyhow::anyhow!( - "Feed key mismatch between client and service" - )); - } + // Verify feed keys match before waiting for connectivity + if client_feed_key != *feed_key { + tracing::error!( + "FEED KEY MISMATCH! Client has: {}, Service has: {}", + client_feed_key, + feed_key + ); + return Err(anyhow::anyhow!( + "Feed key mismatch between client and service" + )); + } - // Wait for all instances' hypercore streams AND the test client mesh - // to be ready concurrently. Both check the same underlying DHT - // connectivity from different sides, so running them in parallel - // avoids wasting the timeout budget on sequential per-instance waits. - let connectivity_timeout = Duration::from_secs(60); - let min_required_peers = 1; - let total_operators = clients.http_clients.len(); + // Wait for all instances' hypercore streams AND the test client mesh + // to be ready concurrently. Both check the same underlying DHT + // connectivity from different sides, so running them in parallel + // avoids wasting the timeout budget on sequential per-instance waits. + let connectivity_timeout = Duration::from_secs(60); + let min_required_peers = 1; + let total_operators = clients.http_clients.len(); - tracing::info!( + tracing::info!( "Waiting for hypercore streams and mesh (min {} peer, {} total operators, timeout {}s)", min_required_peers, total_operators, connectivity_timeout.as_secs(), ); - let stream_futs: Vec<_> = clients - .http_clients - .iter() - .enumerate() - .map(|(idx, http_client)| async move { - tracing::info!( - "Waiting for hypercore stream readiness on instance {} for feed_key {}", - idx, - feed_key - ); - wait_for_hypercore_streams_to_finalize( - http_client, - feed_key, - Some(connectivity_timeout), - ) - .await - .with_context(|| { - format!( - "Hypercore stream failed to finalize on instance {idx}" - ) - }) + let stream_futs: Vec<_> = clients + .http_clients + .iter() + .enumerate() + .map(|(idx, http_client)| async move { + tracing::info!( + "Waiting for hypercore stream readiness on instance {} for feed_key {}", + idx, + feed_key + ); + wait_for_hypercore_streams_to_finalize( + http_client, + feed_key, + Some(connectivity_timeout), + ) + .await + .with_context(|| { + format!("Hypercore stream failed to finalize on instance {idx}") }) - .collect(); + }) + .collect(); - let mesh_fut = wait_for_hypercore_mesh_ready( - &hypercore_client, - min_required_peers, - connectivity_timeout, - ); + let mesh_fut = wait_for_hypercore_mesh_ready( + &hypercore_client, + min_required_peers, + connectivity_timeout, + ); - let (streams_result, mesh_result) = - tokio::join!(futures::future::try_join_all(stream_futs), mesh_fut,); + let (streams_result, mesh_result) = + tokio::join!(futures::future::try_join_all(stream_futs), mesh_fut,); - streams_result.context("Failed to wait for hypercore streams to finalize")?; - let peer_count = mesh_result.context( - "Hypercore mesh not ready: 0 peers connected, append will never replicate", - )?; + streams_result.context("Failed to wait for hypercore streams to finalize")?; + let peer_count = mesh_result.context( + "Hypercore mesh not ready: 0 peers connected, append will never replicate", + )?; - tracing::info!( + tracing::info!( "Hypercore streams and mesh ready: {} connected peers (min required: {}, total operators: {})", peer_count, min_required_peers, total_operators ); - // Append data to the hypercore feed - tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); - let index = hypercore_client.append(payload).await?; - - vec![TriggerId::new(index)] - } else { - // Fallback to simulated trigger for backward compatibility - tracing::warn!( - "No hypercore client found for test '{}', using simulated trigger", - test.name - ); - - let trigger_id = TriggerId::new(0); - let hypercore_data = TriggerData::HypercoreAppend { - feed_key: feed_key.clone(), - index: trigger_id.u64(), - data: payload, - }; - - let req = SimulatedTriggerRequest { - service_id: service_deployment.service.id(), - workflow_id: first_workflow_id.clone(), - trigger: trigger.clone(), - data: hypercore_data, - count: 1, - wait_for_completion: true, - }; - - let http_client = clients - .http_clients - .first() - .ok_or_else(|| anyhow!("No HTTP clients available"))?; - http_client.simulate_trigger(req).await?; + // Append data to the hypercore feed + tracing::info!("Appending {} bytes to hypercore feed...", payload.len()); + let index = hypercore_client.append(payload).await?; - vec![trigger_id] - } + vec![TriggerId::new(index)] } Trigger::Manual => unimplemented!("Manual trigger type is not implemented"), }; From f6c71cb3595ffe0662d8507c7dd903d7438b48d3 Mon Sep 17 00:00:00 2001 From: ismellike Date: Thu, 19 Mar 2026 14:08:57 -0500 Subject: [PATCH 09/10] Improve timing --- packages/layer-tests/src/e2e/handles/hypercore.rs | 6 +++++- packages/layer-tests/src/e2e/runner.rs | 8 ++++++++ .../src/subsystems/trigger/streams/hypercore_stream.rs | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/layer-tests/src/e2e/handles/hypercore.rs b/packages/layer-tests/src/e2e/handles/hypercore.rs index 121d779cd..7e19fa278 100644 --- a/packages/layer-tests/src/e2e/handles/hypercore.rs +++ b/packages/layer-tests/src/e2e/handles/hypercore.rs @@ -62,6 +62,10 @@ impl HypercoreClients { Ok(()) } + pub fn is_empty(&self) -> bool { + self.clients.is_empty() && self.pending.is_empty() + } + /// Get a hypercore test client by test name. pub fn get(&self, test_name: &str) -> Option> { self.clients.get(test_name).cloned() @@ -181,7 +185,7 @@ impl HypercoreTestClient { let swarm_handle_for_relookup = swarm.handle(); let connection_count_for_relookup = Arc::clone(&connection_count_for_swarm); let relookup_handle = tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(5)); + let mut interval = tokio::time::interval(Duration::from_secs(2)); interval.tick().await; // skip immediate first tick loop { interval.tick().await; diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index a393b6a07..014448644 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -217,6 +217,14 @@ impl Runner { .await .expect("Failed to create hypercore clients"); + // Give the test client's DHT time to fully bootstrap and announce + // before WAVS operators look it up. Without this, operators may + // issue lookups before the client has announced, leading to empty + // results until the next re-lookup cycle. + if !self.hypercore_clients.is_empty() { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + // All services are now deployed and ready for the tests // From here on in we're strictly testing the trigger->execute->aggregate->submit flow diff --git a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs index 693a5b952..761d9a2fd 100644 --- a/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs +++ b/packages/wavs/src/subsystems/trigger/streams/hypercore_stream.rs @@ -199,7 +199,7 @@ async fn start_swarm_replication( let peer_count = Arc::clone(&peer_count); let mut shutdown = relookup_shutdown; async move { - let mut interval = tokio::time::interval(Duration::from_secs(5)); + let mut interval = tokio::time::interval(Duration::from_secs(2)); interval.tick().await; // skip immediate first tick loop { tokio::select! { From 276cedb237b51808a5048c5105f56f163539af50 Mon Sep 17 00:00:00 2001 From: ismellike Date: Thu, 19 Mar 2026 14:27:09 -0500 Subject: [PATCH 10/10] Update runner.rs --- packages/layer-tests/src/e2e/runner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/layer-tests/src/e2e/runner.rs b/packages/layer-tests/src/e2e/runner.rs index 014448644..8edcf4ea3 100644 --- a/packages/layer-tests/src/e2e/runner.rs +++ b/packages/layer-tests/src/e2e/runner.rs @@ -534,7 +534,7 @@ async fn run_test( // to be ready concurrently. Both check the same underlying DHT // connectivity from different sides, so running them in parallel // avoids wasting the timeout budget on sequential per-instance waits. - let connectivity_timeout = Duration::from_secs(60); + let connectivity_timeout = Duration::from_secs(120); let min_required_peers = 1; let total_operators = clients.http_clients.len();