diff --git a/nexus/db-queries/src/db/datastore/multicast/groups.rs b/nexus/db-queries/src/db/datastore/multicast/groups.rs index e078d657dd5..0c5dbf68596 100644 --- a/nexus/db-queries/src/db/datastore/multicast/groups.rs +++ b/nexus/db-queries/src/db/datastore/multicast/groups.rs @@ -905,39 +905,30 @@ mod tests { use nexus_types::identity::Resource; use omicron_common::address::{IpRange, Ipv4Range}; use omicron_test_utils::dev; - use omicron_uuid_kinds::{GenericUuid, InstanceUuid, SledUuid}; + use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use crate::db::datastore::Error; use crate::db::datastore::LookupType; use crate::db::model::{ IncompleteIpPoolResource, IpPool, IpPoolReservationType, - IpPoolResourceType, IpVersion, MulticastGroupMemberState, - }; - use crate::db::pub_test_utils::helpers::{ - SledUpdateBuilder, create_instance_with_vmm, create_project, + IpPoolResourceType, IpVersion, }; + use crate::db::pub_test_utils::helpers::create_instance_with_vmm; use crate::db::pub_test_utils::{TestDatabase, multicast}; - async fn create_test_sled(datastore: &DataStore) -> SledUuid { - let sled_id = SledUuid::new_v4(); - let sled_update = SledUpdateBuilder::new().sled_id(sled_id).build(); - datastore.sled_upsert(sled_update).await.unwrap(); - sled_id - } - #[tokio::test] - async fn test_multicast_group_datastore_pool_exhaustion() { - let logctx = - dev::test_setup_log("test_multicast_group_pool_exhaustion"); + async fn test_multicast_group_pool_exhaustion_and_reuse() { + let logctx = dev::test_setup_log( + "test_multicast_group_pool_exhaustion_and_reuse", + ); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); + // Create multicast IP pool with small range (2 addresses) let pool_identity = IdentityMetadataCreateParams { name: "exhaust-pool".parse().unwrap(), description: "Pool exhaustion test".to_string(), }; - - // Create multicast IP pool with very small range (2 addresses) let ip_pool = datastore .ip_pool_create( &opctx, @@ -956,7 +947,6 @@ mod tests { LookupType::ById(ip_pool.id()), ); let range = IpRange::V4( - // Only 2 addresses Ipv4Range::new( Ipv4Addr::new(224, 100, 2, 1), Ipv4Addr::new(224, 100, 2, 2), @@ -979,56 +969,116 @@ mod tests { .await .expect("Should link multicast pool to silo"); - // Allocate first address - let params1 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "first-group".parse().unwrap(), - description: "First group".to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, + // Exhaust the pool by allocating both addresses + let group1 = { + let params = MulticastGroupCreate { + identity: IdentityMetadataCreateParams { + name: "first-group".parse().unwrap(), + description: "First group".to_string(), + }, + multicast_ip: None, + mvlan: None, + has_sources: false, + ip_version: None, + }; + datastore + .multicast_group_create( + &opctx, + ¶ms, + Some(authz_pool.clone()), + ) + .await + .expect("Should create first group") }; - datastore - .multicast_group_create(&opctx, ¶ms1, Some(authz_pool.clone())) - .await - .expect("Should create first group"); + let first_ip = group1.multicast_ip.ip(); - // Allocate second address - let params2 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "second-group".parse().unwrap(), - description: "Second group".to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, - }; - datastore - .multicast_group_create(&opctx, ¶ms2, Some(authz_pool.clone())) - .await - .expect("Should create second group"); + { + let params = MulticastGroupCreate { + identity: IdentityMetadataCreateParams { + name: "second-group".parse().unwrap(), + description: "Second group".to_string(), + }, + multicast_ip: None, + mvlan: None, + has_sources: false, + ip_version: None, + }; + datastore + .multicast_group_create( + &opctx, + ¶ms, + Some(authz_pool.clone()), + ) + .await + .expect("Should create second group"); + } - // Third allocation should fail due to exhaustion - let params3 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "third-group".parse().unwrap(), - description: "Should fail".to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, - }; - let res3 = datastore - .multicast_group_create(&opctx, ¶ms3, Some(authz_pool.clone())) - .await; - assert!( - res3.is_err(), - "Third allocation should fail due to pool exhaustion" - ); + // Verify exhaustion: third allocation should fail + { + let params = MulticastGroupCreate { + identity: IdentityMetadataCreateParams { + name: "third-group".parse().unwrap(), + description: "Should fail".to_string(), + }, + multicast_ip: None, + mvlan: None, + has_sources: false, + ip_version: None, + }; + let result = datastore + .multicast_group_create( + &opctx, + ¶ms, + Some(authz_pool.clone()), + ) + .await; + assert!( + result.is_err(), + "Third allocation should fail due to pool exhaustion" + ); + } + + // Delete first group and verify IP reuse + { + let deleted = datastore + .deallocate_external_multicast_group( + &opctx, + MulticastGroupUuid::from_untyped_uuid(group1.id()), + ) + .await + .expect("Should deallocate first group"); + assert!(deleted, "Should successfully deallocate the group"); + + let params = MulticastGroupCreate { + identity: IdentityMetadataCreateParams { + name: "reuse-group".parse().unwrap(), + description: "Should reuse freed IP".to_string(), + }, + multicast_ip: None, + mvlan: None, + has_sources: false, + ip_version: None, + }; + let reused_group = datastore + .multicast_group_create( + &opctx, + ¶ms, + Some(authz_pool.clone()), + ) + .await + .expect("Should create group after deletion freed IP"); + + assert_eq!( + reused_group.multicast_ip.ip(), + first_ip, + "Should reuse the same IP address" + ); + assert_ne!( + group1.id(), + reused_group.id(), + "Should be different group instances" + ); + } db.terminate().await; logctx.cleanup_successful(); @@ -1286,221 +1336,6 @@ mod tests { logctx.cleanup_successful(); } - #[tokio::test] - async fn test_multicast_group_member_state_transitions_datastore() { - let logctx = dev::test_setup_log( - "test_multicast_group_member_state_transitions_datastore", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // Set up multicast IP pool and group - let pool_identity = IdentityMetadataCreateParams { - name: "state-test-pool".parse().unwrap(), - description: "Pool for state transition testing".to_string(), - }; - let ip_pool = datastore - .ip_pool_create( - &opctx, - IpPool::new_multicast( - &pool_identity, - IpVersion::V4, - IpPoolReservationType::ExternalSilos, - ), - ) - .await - .expect("Should create multicast IP pool"); - - let authz_pool = authz::IpPool::new( - authz::FLEET, - ip_pool.id(), - LookupType::ById(ip_pool.id()), - ); - let range = IpRange::V4( - Ipv4Range::new( - Ipv4Addr::new(224, 4, 1, 1), - Ipv4Addr::new(224, 4, 1, 10), - ) - .unwrap(), - ); - datastore - .ip_pool_add_range(&opctx, &authz_pool, &ip_pool, &range) - .await - .expect("Should add multicast range to pool"); - - let silo_id = opctx.authn.silo_required().unwrap().id(); - let link = IncompleteIpPoolResource { - ip_pool_id: ip_pool.id(), - resource_type: IpPoolResourceType::Silo, - resource_id: silo_id, - is_default: false, - }; - datastore - .ip_pool_link_silo(&opctx, link) - .await - .expect("Should link pool to silo"); - - // Create multicast group (datastore-only; not exercising reconciler) - let group_params = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "state-test-group".parse().unwrap(), - description: "Group for testing member state transitions" - .to_string(), - }, - multicast_ip: None, // Let it allocate from pool - mvlan: None, - has_sources: false, - ip_version: None, - }; - let group = datastore - .multicast_group_create( - &opctx, - &group_params, - Some(authz_pool.clone()), - ) - .await - .expect("Should create multicast group"); - - // Create test project and instance (datastore-only) - let (authz_project, _project) = - create_project(&opctx, &datastore, "state-test-proj").await; - let sled_id = create_test_sled(&datastore).await; - let (instance, _vmm) = create_instance_with_vmm( - &opctx, - &datastore, - &authz_project, - "state-test-instance", - sled_id, - ) - .await; - let test_instance_id = instance.into_untyped_uuid(); - - // Transition group to "Active" state before adding members - datastore - .multicast_group_set_active( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - ) - .await - .expect("Should transition group to 'Active' state"); - - // Create member record in "Joining" state using datastore API - let member = datastore - .multicast_group_member_attach_to_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(test_instance_id), - Some(&[] as &[IpAddr]), - ) - .await - .expect("Should create member record"); - - assert_eq!(member.state, MulticastGroupMemberState::Joining); - assert_eq!(member.parent_id, test_instance_id); - - // Case: Transition from "Joining" → "Joined" (simulating what the reconciler would do) - datastore - .multicast_group_member_set_state( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(test_instance_id), - MulticastGroupMemberState::Joined, - ) - .await - .expect("Should transition to 'Joined'"); - - // Verify member is now "Active" - let pagparams = &DataPageParams { - marker: None, - limit: std::num::NonZeroU32::new(100).unwrap(), - direction: dropshot::PaginationOrder::Ascending, - }; - - let members = datastore - .multicast_group_members_list( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - pagparams, - ) - .await - .expect("Should list members"); - - assert_eq!(members.len(), 1); - assert_eq!(members[0].state, MulticastGroupMemberState::Joined); - - // Case: Transition member to "Left" state (without permanent deletion) - datastore - .multicast_group_member_set_state( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(test_instance_id), - MulticastGroupMemberState::Left, - ) - .await - .expect("Should transition to 'Left' state"); - - // Verify member is now in "Left" state - let all_members = datastore - .multicast_group_members_list_by_id( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - pagparams, - ) - .await - .expect("Should list all members"); - - assert_eq!(all_members.len(), 1); - - // Verify only "Active" members are shown (filter out Left members) - let all_members = datastore - .multicast_group_members_list( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - pagparams, - ) - .await - .expect("Should list all members"); - - // Filter for "Active" members (non-"Left" state) - let active_members: Vec<_> = all_members - .into_iter() - .filter(|m| m.state != MulticastGroupMemberState::Left) - .collect(); - - assert_eq!( - active_members.len(), - 0, - "Active member list should filter out Left members" - ); - - // Complete removal (→ "Left") - datastore - .multicast_group_member_set_state( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(test_instance_id), - MulticastGroupMemberState::Left, - ) - .await - .expect("Should transition to Left"); - - // Member should still exist in database and be in "Left" state - let members = datastore - .multicast_group_members_list_by_id( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - pagparams, - ) - .await - .expect("Should list members"); - - assert_eq!(members.len(), 1); - assert_eq!(members[0].state, MulticastGroupMemberState::Left); - - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn test_multicast_group_ip_reuse_after_deletion() { let logctx = @@ -1614,143 +1449,6 @@ mod tests { logctx.cleanup_successful(); } - #[tokio::test] - async fn test_multicast_group_pool_exhaustion_delete_create_cycle() { - let logctx = dev::test_setup_log( - "test_multicast_group_pool_exhaustion_delete_create_cycle", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - // Set up small pool (only 1 address) - let pool_identity = IdentityMetadataCreateParams { - name: "cycle-test-pool".parse().unwrap(), - description: "Pool for exhaustion-delete-create cycle testing" - .to_string(), - }; - let ip_pool = datastore - .ip_pool_create( - &opctx, - IpPool::new_multicast( - &pool_identity, - IpVersion::V4, - IpPoolReservationType::ExternalSilos, - ), - ) - .await - .expect("Should create multicast IP pool"); - - let authz_pool = authz::IpPool::new( - authz::FLEET, - ip_pool.id(), - external::LookupType::ById(ip_pool.id()), - ); - let range = IpRange::V4( - Ipv4Range::new( - Ipv4Addr::new(224, 20, 1, 50), // Only 1 address - Ipv4Addr::new(224, 20, 1, 50), - ) - .unwrap(), - ); - datastore - .ip_pool_add_range(&opctx, &authz_pool, &ip_pool, &range) - .await - .expect("Should add multicast range to pool"); - - let silo_id = opctx.authn.silo_required().unwrap().id(); - let link = IncompleteIpPoolResource { - ip_pool_id: ip_pool.id(), - resource_type: IpPoolResourceType::Silo, - resource_id: silo_id, - is_default: false, - }; - datastore - .ip_pool_link_silo(&opctx, link) - .await - .expect("Should link pool to silo"); - - // Exhaust the pool - let params1 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "cycle-test-1".parse().unwrap(), - description: "First group to exhaust pool".to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, - }; - - let group1 = datastore - .multicast_group_create(&opctx, ¶ms1, Some(authz_pool.clone())) - .await - .expect("Should create first group"); - let allocated_ip = group1.multicast_ip.ip(); - - // Try to create another group - should fail due to exhaustion - let params2 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "cycle-test-2".parse().unwrap(), - description: "Second group should fail".to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, - }; - - let res2 = datastore - .multicast_group_create(&opctx, ¶ms2, Some(authz_pool.clone())) - .await; - assert!( - res2.is_err(), - "Second group creation should fail due to pool exhaustion" - ); - - // Delete the first group to free up the IP - let deleted = datastore - .deallocate_external_multicast_group( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group1.id()), - ) - .await - .expect("Should deallocate first group"); - assert_eq!(deleted, true, "Should successfully deallocate the group"); - - // Now creating a new group should succeed - let params3 = MulticastGroupCreate { - identity: IdentityMetadataCreateParams { - name: "cycle-test-3".parse().unwrap(), - description: "Third group should succeed after deletion" - .to_string(), - }, - multicast_ip: None, - mvlan: None, - has_sources: false, - ip_version: None, - }; - - let group3 = datastore - .multicast_group_create(&opctx, ¶ms3, Some(authz_pool.clone())) - .await - .expect("Should create third group after first was deleted"); - - // Should reuse the same IP address - assert_eq!( - group3.multicast_ip.ip(), - allocated_ip, - "Should reuse the same IP address" - ); - assert_ne!( - group1.id(), - group3.id(), - "Should be different group instances" - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - #[tokio::test] async fn test_multicast_group_deallocation_return_values() { let logctx = dev::test_setup_log( diff --git a/nexus/db-queries/src/db/datastore/multicast/members.rs b/nexus/db-queries/src/db/datastore/multicast/members.rs index a8844686c49..4f1eb460860 100644 --- a/nexus/db-queries/src/db/datastore/multicast/members.rs +++ b/nexus/db-queries/src/db/datastore/multicast/members.rs @@ -3072,9 +3072,10 @@ mod tests { } #[tokio::test] - async fn test_member_attach_reactivation_from_left() { - let logctx = - dev::test_setup_log("test_member_attach_reactivation_from_left"); + async fn test_member_attach_reactivation_source_handling() { + let logctx = dev::test_setup_log( + "test_member_attach_reactivation_source_handling", + ); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); @@ -3086,236 +3087,165 @@ mod tests { ) .await; - // Create active group let group = multicast::create_test_group_with_state( &opctx, &datastore, &setup, "test-group", "224.10.1.9", - true, // make_active - ) - .await; - - // Create instance - let (instance, _vmm) = create_instance_with_vmm( - &opctx, - &datastore, - &setup.authz_project, - "test-instance", - setup.sled_id, + true, ) .await; - let instance_id = *instance.as_untyped_uuid(); + let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); - // First attach with source IPs - let initial_sources: Vec = - vec!["10.1.1.1".parse().unwrap(), "10.1.1.2".parse().unwrap()]; - let member1 = datastore - .multicast_group_member_attach_to_instance( + // Preserve sources: None keeps existing source_ips + { + let instance = create_stopped_instance_record( &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - Some(initial_sources.as_slice()), + &datastore, + &setup.authz_project, + "preserve-instance", ) - .await - .expect("First attach should succeed"); - - // Verify `source_ips` were stored - // Database stores IpNetwork, so convert for comparison - let member_init = datastore - .multicast_group_member_get_by_id(&opctx, member1.id, false) - .await - .expect("Should get member") - .expect("Member should exist"); - let stored_ips: Vec = - member_init.source_ips.iter().map(|n| n.ip()).collect(); - assert_eq!( - stored_ips, initial_sources, - "Initial source_ips should be stored" - ); - - // Transition member to "Left" state and clear sled_id (simulating - // instance stop) - // - // This does not set `time_deleted`, instead only stopped instances can - // be reactivated - datastore - .multicast_group_members_detach_by_instance( - &opctx, - InstanceUuid::from_untyped_uuid(instance_id), - ) - .await - .expect("Should transition member to 'Left' and clear sled_id"); + .await; - // Verify member is now in Left state without time_deleted - let member_stopped = datastore - .multicast_group_member_get_by_id(&opctx, member1.id, false) - .await - .expect("Should get member") - .expect("Member should still exist (not soft-deleted)"); - assert_eq!(member_stopped.state, MulticastGroupMemberState::Left); - assert!( - member_stopped.time_deleted.is_none(), - "'time_deleted' should not be set for stopped instances" - ); - assert!(member_stopped.sled_id.is_none(), "sled_id should be cleared"); + let original_sources: Vec = + vec!["10.1.1.1".parse().unwrap(), "10.1.1.2".parse().unwrap()]; + let member = datastore + .multicast_group_member_attach_to_instance( + &opctx, + group_id, + instance, + Some(original_sources.as_slice()), + ) + .await + .expect("Should attach"); - // Reactivate by attaching again (simulating instance restart) - // Use `None` to preserve the existing source IPs - let member2 = datastore - .multicast_group_member_attach_to_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - None, // Preserve existing sources - ) - .await - .expect("Reactivation should succeed"); + datastore + .multicast_group_members_detach_by_instance(&opctx, instance) + .await + .expect("Should detach"); - // Should return same member ID (reactivated existing member) - assert_eq!(member1.id, member2.id, "Should reactivate same member"); + let reactivated = datastore + .multicast_group_member_attach_to_instance( + &opctx, group_id, instance, + None, // Preserve existing sources + ) + .await + .expect("Reactivation should succeed"); + + assert_eq!( + member.id, reactivated.id, + "Should reactivate same member" + ); + let stored_ips: Vec = + reactivated.source_ips.iter().map(|n| n.ip()).collect(); + assert_eq!( + stored_ips, original_sources, + "None should preserve existing source_ips" + ); + assert_eq!(reactivated.state, MulticastGroupMemberState::Joining); + } - // Verify member is back in "Joining" state with `time_deleted` still NULL - let member = datastore - .multicast_group_member_get_by_group_and_instance( + // Replace sources: Some([new]) replaces existing source_ips + { + let instance = create_stopped_instance_record( &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), + &datastore, + &setup.authz_project, + "replace-instance", ) - .await - .expect("Should get member") - .expect("Member should exist"); - - assert_eq!(member.state, MulticastGroupMemberState::Joining); - assert_eq!(member.id, member1.id); - assert!( - member.time_deleted.is_none(), - "time_deleted should remain NULL (never set by detach_by_instance)" - ); - // Verify `source_ips` preserved on reactivation with empty sources - // Database stores IpNetwork, so convert for comparison - let stored_ips: Vec = - member.source_ips.iter().map(|n| n.ip()).collect(); - assert_eq!( - stored_ips, initial_sources, - "Reactivation with empty sources should preserve existing source_ips" - ); - - db.terminate().await; - logctx.cleanup_successful(); - } - - #[tokio::test] - async fn test_member_attach_reactivation_replaces_sources() { - let logctx = dev::test_setup_log( - "test_member_attach_reactivation_replaces_sources", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - let setup = multicast::create_test_setup( - &opctx, - &datastore, - "replace-sources-pool", - "replace-sources-project", - ) - .await; - - // Create active group - let group = multicast::create_test_group_with_state( - &opctx, - &datastore, - &setup, - "test-group", - "224.10.1.20", - true, // make_active - ) - .await; + .await; - // Create instance - let (instance, _vmm) = create_instance_with_vmm( - &opctx, - &datastore, - &setup.authz_project, - "test-instance", - setup.sled_id, - ) - .await; - let instance_id = *instance.as_untyped_uuid(); + let original_sources: Vec = + vec!["10.0.0.1".parse().unwrap(), "10.0.0.2".parse().unwrap()]; + let member = datastore + .multicast_group_member_attach_to_instance( + &opctx, + group_id, + instance, + Some(original_sources.as_slice()), + ) + .await + .expect("Should attach"); - // Initial attach with source IPs [A, B] - let original_sources: Vec = - vec!["10.0.0.1".parse().unwrap(), "10.0.0.2".parse().unwrap()]; - datastore - .multicast_group_member_attach_to_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - Some(original_sources.as_slice()), - ) - .await - .expect("Should attach instance"); + datastore + .multicast_group_members_detach_by_instance(&opctx, instance) + .await + .expect("Should detach"); - // Verify original sources stored - // Database stores IpNetwork, so convert for comparison - let member_initial = datastore - .multicast_group_member_get_by_group_and_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - ) - .await - .expect("Should get member") - .expect("Member should exist"); - let stored_ips: Vec = - member_initial.source_ips.iter().map(|n| n.ip()).collect(); - assert_eq!(stored_ips, original_sources); + let replacement_sources: Vec = + vec!["10.0.0.3".parse().unwrap(), "10.0.0.4".parse().unwrap()]; + let reactivated = datastore + .multicast_group_member_attach_to_instance( + &opctx, + group_id, + instance, + Some(replacement_sources.as_slice()), + ) + .await + .expect("Reactivation should succeed"); + + assert_eq!( + member.id, reactivated.id, + "Should reactivate same member when replacing sources" + ); + let stored_ips: Vec = + reactivated.source_ips.iter().map(|n| n.ip()).collect(); + assert_eq!( + stored_ips, replacement_sources, + "Some([new]) should replace existing sources" + ); + assert_ne!(stored_ips, original_sources); + } - // Transition to "Left" (simulating instance stop) - datastore - .multicast_group_members_detach_by_instance( + // Clear sources: Some([]) clears source_ips (switch to ASM) + { + let instance = create_stopped_instance_record( &opctx, - InstanceUuid::from_untyped_uuid(instance_id), + &datastore, + &setup.authz_project, + "clear-instance", ) - .await - .expect("Should detach"); + .await; - // Reactivate with a different set of non-empty sources [C, D] - let replacement_sources: Vec = - vec!["10.0.0.3".parse().unwrap(), "10.0.0.4".parse().unwrap()]; - datastore - .multicast_group_member_attach_to_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - Some(replacement_sources.as_slice()), - ) - .await - .expect("Reactivation should succeed"); + let original_sources: Vec = + vec!["10.5.5.1".parse().unwrap(), "10.5.5.2".parse().unwrap()]; + let member = datastore + .multicast_group_member_attach_to_instance( + &opctx, + group_id, + instance, + Some(original_sources.as_slice()), + ) + .await + .expect("Should attach"); - // Verify `source_ips` were replaced (not preserved) - // Database stores IpNetwork, so convert for comparison - let member_reactivated = datastore - .multicast_group_member_get_by_group_and_instance( - &opctx, - MulticastGroupUuid::from_untyped_uuid(group.id()), - InstanceUuid::from_untyped_uuid(instance_id), - ) - .await - .expect("Should get member") - .expect("Member should exist"); - let stored_ips: Vec = - member_reactivated.source_ips.iter().map(|n| n.ip()).collect(); + datastore + .multicast_group_members_detach_by_instance(&opctx, instance) + .await + .expect("Should detach"); - assert_eq!( - stored_ips, replacement_sources, - "Reactivation with non-empty sources should REPLACE existing sources" - ); - assert_ne!( - stored_ips, original_sources, - "Original sources should not be preserved when new sources provided" - ); + let reactivated = datastore + .multicast_group_member_attach_to_instance( + &opctx, + group_id, + instance, + Some(NO_SOURCE_IPS), // Clear sources + ) + .await + .expect("Reactivation should succeed"); + + assert_eq!( + member.id, reactivated.id, + "Should reactivate same member when clearing sources" + ); + assert_eq!( + reactivated.source_ips.len(), + 0, + "Some([]) should clear source_ips" + ); + assert_eq!(reactivated.state, MulticastGroupMemberState::Joining); + } db.terminate().await; logctx.cleanup_successful(); @@ -3747,209 +3677,4 @@ mod tests { db.terminate().await; logctx.cleanup_successful(); } - - /// Test that `None` preserves source_ips on reactivation. - /// - /// This verifies the distinction between: - /// - `None` → preserve existing source_ips - /// - `Some([])` → clear source_ips (switch to ASM) - #[tokio::test] - async fn test_member_attach_preserves_sources_on_reactivation() { - let logctx = dev::test_setup_log( - "test_member_attach_preserves_sources_on_reactivation", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - let setup = multicast::create_test_setup( - &opctx, - &datastore, - "add-preserve-sources-pool", - "add-preserve-sources-project", - ) - .await; - - // Create active group - let group = multicast::create_test_group_with_state( - &opctx, - &datastore, - &setup, - "test-group", - "224.10.1.1", - true, // make_active - ) - .await; - let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); - - // Create stopped instance - let instance = create_stopped_instance_record( - &opctx, - &datastore, - &setup.authz_project, - "test-instance", - ) - .await; - - // Add member with `source_ips` via HTTP API path - let original_sources = - vec!["10.5.5.1".parse().unwrap(), "10.5.5.2".parse().unwrap()]; - let member = datastore - .multicast_group_member_attach_to_instance( - &opctx, - group_id, - instance, - Some(original_sources.as_slice()), - ) - .await - .expect("Should add member with sources"); - - assert_eq!( - member.source_ips.len(), - 2, - "Member should have 2 source IPs" - ); - - // Transition to "Left" state - datastore - .multicast_group_members_detach_by_instance(&opctx, instance) - .await - .expect("Should detach"); - - // Verify member is in "Left" state - let left_member = datastore - .multicast_group_member_get_by_id(&opctx, member.id, false) - .await - .expect("Should get member") - .expect("Member should exist"); - assert_eq!(left_member.state, MulticastGroupMemberState::Left); - // Source IPs should still be stored (just in "Left" state) - assert_eq!(left_member.source_ips.len(), 2); - - // Reactivate via HTTP API path with `None` (preserve existing sources) - let reactivated = datastore - .multicast_group_member_attach_to_instance( - &opctx, group_id, instance, - None, // None = preserve existing source_ips - ) - .await - .expect("Should reactivate member"); - - // Verify `source_ips` were preserved (not cleared) - assert_eq!( - reactivated.source_ips.len(), - 2, - "Source IPs should be preserved on reactivation with None" - ); - let reactivated_ips: Vec = - reactivated.source_ips.iter().map(|n| n.ip()).collect(); - assert!(reactivated_ips.contains(&"10.5.5.1".parse().unwrap())); - assert!(reactivated_ips.contains(&"10.5.5.2".parse().unwrap())); - - // Verify state is back to Joining - assert_eq!(reactivated.state, MulticastGroupMemberState::Joining); - - db.terminate().await; - logctx.cleanup_successful(); - } - - /// Test that `Some([])` clears `source_ips` on reactivation (switch to ASM). - /// - /// This verifies the distinction between: - /// - `None` → preserve existing `source_ips` - /// - `Some([])` → clear `source_ips` (switch to ASM) - #[tokio::test] - async fn test_member_attach_clears_sources_on_reactivation() { - let logctx = dev::test_setup_log( - "test_member_attach_clears_sources_on_reactivation", - ); - let db = TestDatabase::new_with_datastore(&logctx.log).await; - let (opctx, datastore) = (db.opctx(), db.datastore()); - - let setup = multicast::create_test_setup( - &opctx, - &datastore, - "add-clear-sources-pool", - "add-clear-sources-project", - ) - .await; - - // Create active group - let group = multicast::create_test_group_with_state( - &opctx, - &datastore, - &setup, - "test-group", - "224.10.1.1", - true, // make_active - ) - .await; - let group_id = MulticastGroupUuid::from_untyped_uuid(group.id()); - - // Create stopped instance - let instance = create_stopped_instance_record( - &opctx, - &datastore, - &setup.authz_project, - "test-instance", - ) - .await; - - // Add member with `source_ips` - let original_sources = - vec!["10.5.5.1".parse().unwrap(), "10.5.5.2".parse().unwrap()]; - let member = datastore - .multicast_group_member_attach_to_instance( - &opctx, - group_id, - instance, - Some(original_sources.as_slice()), - ) - .await - .expect("Should add member with sources"); - - assert_eq!( - member.source_ips.len(), - 2, - "Member should have 2 source IPs" - ); - - // Transition to "Left" state - datastore - .multicast_group_members_detach_by_instance(&opctx, instance) - .await - .expect("Should detach"); - - // Verify member is in "Left" state with sources still stored - let left_member = datastore - .multicast_group_member_get_by_id(&opctx, member.id, false) - .await - .expect("Should get member") - .expect("Member should exist"); - assert_eq!(left_member.state, MulticastGroupMemberState::Left); - assert_eq!(left_member.source_ips.len(), 2); - - // Reactivate to clear sources (switch to ASM) - let reactivated = datastore - .multicast_group_member_attach_to_instance( - &opctx, - group_id, - instance, - Some(NO_SOURCE_IPS), // Some([]) = clear source_ips - ) - .await - .expect("Should reactivate member"); - - // Verify `source_ips` were cleared - assert_eq!( - reactivated.source_ips.len(), - 0, - "Source IPs should be cleared on reactivation with Some([])" - ); - - // Verify state is back to "Joining" - assert_eq!(reactivated.state, MulticastGroupMemberState::Joining); - - db.terminate().await; - logctx.cleanup_successful(); - } } diff --git a/nexus/tests/integration_tests/multicast/api.rs b/nexus/tests/integration_tests/multicast/api.rs index 87d99682fb2..d7328a0ada4 100644 --- a/nexus/tests/integration_tests/multicast/api.rs +++ b/nexus/tests/integration_tests/multicast/api.rs @@ -59,8 +59,7 @@ async fn test_multicast_api_behavior(cptestctx: &ControlPlaneTestContext) { ) .await; - // Case: Stopped instances (all APIs should handle stopped instances - // identically) + // Case: Stopped instances (all APIs should handle stopped instances identically) // API Path: Instance created stopped, then added to group let instance1_params = InstanceCreate { @@ -128,8 +127,8 @@ async fn test_multicast_api_behavior(cptestctx: &ControlPlaneTestContext) { // Reconciler runs, sees instance_valid=false (stopped/no VMM) // Reconciler immediately transitions "Joining"→"Left" (no DPD programming) // - // This verifies the reconciler correctly handles stopped instances without - // requiring inventory/DPD readiness (unlike running instances). + // This verifies stopped instances don't require sled assignment or DPD + // underlay programming, as they go straight to "Left" state. for (i, instance) in [&instance1, &instance2].iter().enumerate() { wait_for_member_state( cptestctx, @@ -147,8 +146,7 @@ async fn test_multicast_api_behavior(cptestctx: &ControlPlaneTestContext) { ); } - // Case: Idempotency test (adding already-existing member should be - // safe for all APIs) + // Case: Idempotency (adding an existing member is safe) // Try to add instance1 again using instance join (should be idempotent) let duplicate_join_url = format!( @@ -464,176 +462,189 @@ async fn test_join_by_ip_ssm_with_sources(cptestctx: &ControlPlaneTestContext) { wait_for_group_deleted(cptestctx, &expected_group_name).await; } -/// Test SSM join-by-IP without sources should fail. -/// SSM addresses (232.0.0.0/8) require source IPs for implicit creation. +/// Test SSM source validation: all scenarios where SSM joins should fail. +/// +/// SSM addresses (232.0.0.0/8) require source IPs for every member subscription. +/// This test validates three failure scenarios with a single setup: +/// +/// 1. Join-by-IP with new SSM group without sources (implicit creation blocked) +/// 2. Join existing SSM group without sources (via ID, name, and IP lookup) +/// 3. Join-by-IP with empty sources array (treated same as no sources) #[nexus_test] -async fn test_join_by_ip_ssm_without_sources_fails( - cptestctx: &ControlPlaneTestContext, -) { +async fn test_ssm_source_validation(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - let project_name = "join-by-ip-ssm-fail-project"; - let instance_name = "join-by-ip-ssm-fail-inst"; + let project_name = "ssm-source-validation-project"; - // Setup + // Setup: project, pools, and instances for the rest of the test ops::join3( create_project(client, project_name), create_default_ip_pools(client), create_multicast_ip_pool_with_range( client, - "ssm-fail-pool", + "ssm-validation-pool", (232, 30, 0, 1), (232, 30, 0, 255), ), ) .await; - create_instance(client, project_name, instance_name).await; - - // Try to join SSM IP without sources; should fail - let ssm_ip = "232.30.0.50"; - let join_url = format!( - "/v1/instances/{instance_name}/multicast-groups/{ssm_ip}?project={project_name}" - ); - let join_body = InstanceMulticastGroupJoin { - source_ips: None, // No sources! - ip_version: None, - }; - - let error = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url) - .body(Some(&join_body)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("SSM without sources should fail"); - - let error_body: dropshot::HttpErrorResponseBody = - error.parsed_body().unwrap(); - assert_eq!( - error_body.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for SSM without sources, got: {:?}", - error_body.error_code - ); + // Create instances for all test cases + create_instance(client, project_name, "ssm-inst-creator").await; + create_instance(client, project_name, "ssm-inst-joiner").await; - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; -} + // Shared body for "no sources" cases + let join_body_no_sources = + InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; -/// Test joining an existing SSM group without sources fails for all lookup methods. -/// -/// When an SSM group exists (created by first instance with sources), a second -/// instance cannot join without providing sources - regardless of lookup method. -/// This test consolidates validation for join-by-ID, join-by-name, and join-by-IP. -#[nexus_test] -async fn test_join_existing_ssm_group_without_sources_fails( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "ssm-join-existing-fail-project"; + // Case: Join-by-IP SSM without sources fails (implicit creation blocked) + { + let ssm_ip = "232.30.0.50"; + let join_url = format!( + "/v1/instances/ssm-inst-joiner/multicast-groups/{ssm_ip}?project={project_name}" + ); - // Setup: SSM pool - ops::join3( - create_project(client, project_name), - create_default_ip_pools(client), - create_multicast_ip_pool_with_range( - client, - "ssm-join-existing-fail-pool", - (232, 40, 0, 1), - (232, 40, 0, 255), - ), - ) - .await; + let response = NexusRequest::new( + RequestBuilder::new(client, Method::PUT, &join_url) + .body(Some(&join_body_no_sources)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("SSM without sources should fail"); - create_instance(client, project_name, "ssm-inst-creator").await; - create_instance(client, project_name, "ssm-inst-joiner").await; + let error: dropshot::HttpErrorResponseBody = + response.parsed_body().unwrap(); + assert_eq!( + error.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for SSM without sources, got: {:?}", + error.error_code + ); + } - // First instance creates SSM group with sources - let ssm_ip = "232.40.0.100"; - let source_ip: IpAddr = "10.40.0.1".parse().unwrap(); - let join_url_1 = format!( + // Case: Join existing SSM group without sources fails (all lookup methods) + // + // First, create an SSM group with sources using ssm-inst-creator + let ssm_ip = "232.30.0.100"; + let source_ip: IpAddr = "10.30.0.1".parse().unwrap(); + let join_url_creator = format!( "/v1/instances/ssm-inst-creator/multicast-groups/{ssm_ip}?project={project_name}" ); - - let join_body_1 = InstanceMulticastGroupJoin { + let join_body_with_sources = InstanceMulticastGroupJoin { source_ips: Some(vec![source_ip]), ip_version: None, }; - let member_1: MulticastGroupMember = - put_upsert(client, &join_url_1, &join_body_1).await; + let member_creator: MulticastGroupMember = + put_upsert(client, &join_url_creator, &join_body_with_sources).await; - let group_id = member_1.multicast_group_id; + let group_id = member_creator.multicast_group_id; let group_name = format!("mcast-{}", ssm_ip.replace('.', "-")); - let join_body_no_sources = - InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; - // Join by ID without sources - should fail - let join_url_by_id = format!( - "/v1/instances/ssm-inst-joiner/multicast-groups/{group_id}?project={project_name}" - ); - let error_by_id = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url_by_id) - .body(Some(&join_body_no_sources)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Join by ID without sources should fail") - .parsed_body::() - .unwrap(); - assert_eq!( - error_by_id.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for join-by-ID, got: {:?}", - error_by_id.error_code - ); + // Join by ID without sources -> should fail + { + let join_url = format!( + "/v1/instances/ssm-inst-joiner/multicast-groups/{group_id}?project={project_name}" + ); + let error = NexusRequest::new( + RequestBuilder::new(client, Method::PUT, &join_url) + .body(Some(&join_body_no_sources)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("Join by ID without sources should fail") + .parsed_body::() + .unwrap(); + assert_eq!( + error.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for join-by-ID, got: {:?}", + error.error_code + ); + } - // Join by name without sources - should fail - let join_url_by_name = format!( - "/v1/instances/ssm-inst-joiner/multicast-groups/{group_name}?project={project_name}" - ); - let error_by_name = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url_by_name) - .body(Some(&join_body_no_sources)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Join by name without sources should fail") - .parsed_body::() - .unwrap(); - assert_eq!( - error_by_name.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for join-by-name, got: {:?}", - error_by_name.error_code - ); + // Join by name without sources -> should fail + { + let join_url = format!( + "/v1/instances/ssm-inst-joiner/multicast-groups/{group_name}?project={project_name}" + ); + let error = NexusRequest::new( + RequestBuilder::new(client, Method::PUT, &join_url) + .body(Some(&join_body_no_sources)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("Join by name without sources should fail") + .parsed_body::() + .unwrap(); + assert_eq!( + error.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for join-by-name, got: {:?}", + error.error_code + ); + } - // Join by IP without sources - should fail - let join_url_by_ip = format!( - "/v1/instances/ssm-inst-joiner/multicast-groups/{ssm_ip}?project={project_name}" - ); - let error_by_ip = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url_by_ip) - .body(Some(&join_body_no_sources)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Join by IP without sources should fail") - .parsed_body::() - .unwrap(); - assert_eq!( - error_by_ip.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for join-by-IP, got: {:?}", - error_by_ip.error_code - ); + // Join by IP without sources -> should fail + { + let join_url = format!( + "/v1/instances/ssm-inst-joiner/multicast-groups/{ssm_ip}?project={project_name}" + ); + let error = NexusRequest::new( + RequestBuilder::new(client, Method::PUT, &join_url) + .body(Some(&join_body_no_sources)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("Join by IP without sources should fail") + .parsed_body::() + .unwrap(); + assert_eq!( + error.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for join-by-IP, got: {:?}", + error.error_code + ); + } + + // Case: SSM with empty sources array fails (treated same as `None`) + { + let ssm_ip = "232.30.0.150"; + let join_url = format!( + "/v1/instances/ssm-inst-joiner/multicast-groups/{ssm_ip}?project={project_name}" + ); + let join_body_empty_sources = InstanceMulticastGroupJoin { + source_ips: Some(vec![]), + ip_version: None, + }; + + let response = NexusRequest::new( + RequestBuilder::new(client, Method::PUT, &join_url) + .body(Some(&join_body_empty_sources)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("SSM with empty sources array should fail"); + let error: dropshot::HttpErrorResponseBody = + response.parsed_body().unwrap(); + assert_eq!( + error.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for SSM with empty sources, got: {:?}", + error.error_code + ); + } + + // Cleanup cleanup_instances( cptestctx, client, @@ -644,66 +655,6 @@ async fn test_join_existing_ssm_group_without_sources_fails( wait_for_group_deleted(cptestctx, &group_name).await; } -/// Test that SSM join-by-IP with empty sources array fails. -/// -/// `source_ips: Some(vec![])` (empty array) is treated the same as -/// `source_ips: None` for SSM validation - both mean "no sources" and -/// should fail for SSM addresses. -#[nexus_test] -async fn test_ssm_with_empty_sources_array_fails( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "ssm-empty-sources-project"; - let instance_name = "ssm-empty-sources-inst"; - - // Setup - ops::join3( - create_project(client, project_name), - create_default_ip_pools(client), - create_multicast_ip_pool_with_range( - client, - "ssm-empty-sources-pool", - (232, 46, 0, 1), - (232, 46, 0, 100), - ), - ) - .await; - - create_instance(client, project_name, instance_name).await; - - // Try to join SSM IP with empty sources array (should fail) - let ssm_ip = "232.46.0.50"; - let join_url = format!( - "/v1/instances/{instance_name}/multicast-groups/{ssm_ip}?project={project_name}" - ); - let join_body = InstanceMulticastGroupJoin { - source_ips: Some(vec![]), - ip_version: None, - }; - - let error = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url) - .body(Some(&join_body)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("SSM with empty sources array should fail"); - - let error_body: dropshot::HttpErrorResponseBody = - error.parsed_body().unwrap(); - assert_eq!( - error_body.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for SSM with empty sources, got: {:?}", - error_body.error_code - ); - - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; -} - /// Test join-by-IP with IP not in any pool should fail. #[nexus_test] async fn test_join_by_ip_not_in_pool_fails( @@ -1042,8 +993,8 @@ async fn test_explicit_ip_bypasses_ssm_asm_selection( let project_name = "explicit-ip-bypass-project"; let instance_name = "explicit-ip-bypass-inst"; - // Setup: create BOTH SSM and ASM pools - ops::join3( + // Setup: create both SSM and ASM pools + ops::join4( create_project(client, project_name), create_default_ip_pools(client), create_multicast_ip_pool_with_range( @@ -1052,14 +1003,12 @@ async fn test_explicit_ip_bypasses_ssm_asm_selection( (232, 80, 0, 1), (232, 80, 0, 255), ), - ) - .await; - - create_multicast_ip_pool_with_range( - client, - "bypass-asm-pool", - (224, 80, 0, 1), - (224, 80, 0, 255), + create_multicast_ip_pool_with_range( + client, + "bypass-asm-pool", + (224, 80, 0, 1), + (224, 80, 0, 255), + ), ) .await; diff --git a/nexus/tests/integration_tests/multicast/authorization.rs b/nexus/tests/integration_tests/multicast/authorization.rs index 6d39bf55c52..b2e6c0a25aa 100644 --- a/nexus/tests/integration_tests/multicast/authorization.rs +++ b/nexus/tests/integration_tests/multicast/authorization.rs @@ -12,6 +12,7 @@ use http::StatusCode; +use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils::resource_helpers::test_params::UserPassword; use nexus_test_utils::resource_helpers::{ @@ -136,29 +137,39 @@ async fn create_group_via_instance_join( .unwrap() } -/// Test that silo users can attach their own instances to fleet-scoped -/// multicast groups (including groups created by other users or fleet admins). +/// Test that silo users with various permission levels can interact with +/// multicast groups appropriately. +/// +/// This consolidated test verifies: +/// - Silo users can attach their own instances to fleet-scoped multicast groups +/// - Authenticated users can read multicast groups (no Fleet::Viewer required) +/// - Project-only users can access multicast groups in their project #[nexus_test] -async fn test_silo_users_can_attach_instances_to_multicast_groups( +async fn test_silo_user_multicast_permissions( cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; - create_default_ip_pools(&client).await; - // Get current silo info - let silo_url = format!("/v1/system/silos/{}", cptestctx.silo_name); + // Common setup: create pools (mcast pool auto-links to DEFAULT_SILO) + ops::join2( + create_default_ip_pools(&client), + create_multicast_ip_pool(&client, "mcast-pool"), + ) + .await; + + // Get the DEFAULT silo (same silo as PrivilegedUser) + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.identity().name); let silo: Silo = object_get(client, &silo_url).await; - // Create multicast pool and link to silo - create_multicast_ip_pool(&client, "mcast-pool").await; - link_ip_pool(&client, "default-v4", &silo.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo.identity.id, false).await; + // Create users with different permission levels: + // - collaborator_user: Silo Collaborator (can create projects/instances) + // - reader_user: No special roles (only authenticated) + // - project_user: Only project-level roles (no silo-level roles) - // Create a regular silo user - let user = create_local_user( + let collaborator_user = create_local_user( client, &silo, - &"test-user".parse().unwrap(), + &"collaborator-user".parse().unwrap(), UserPassword::LoginDisallowed, ) .await; @@ -167,15 +178,44 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( client, &silo_url, SiloRole::Collaborator, - user.id, + collaborator_user.id, + AuthnMode::PrivilegedUser, + ) + .await; + + let reader_user = create_local_user( + client, + &silo, + &"reader-user".parse().unwrap(), + UserPassword::LoginDisallowed, + ) + .await; + + let project_user = create_local_user( + client, + &silo, + &"project-only-user".parse().unwrap(), + UserPassword::LoginDisallowed, + ) + .await; + + // Create a project for project_user (using PrivilegedUser, then grant access) + let project_only = create_project(client, "project-only").await; + let project_url = format!("/v1/projects/{}", project_only.identity.name); + grant_iam( + client, + &project_url, + ProjectRole::Collaborator, + project_user.id, AuthnMode::PrivilegedUser, ) .await; - // User creates group via instance join (implicitly creates the group with first instance) + // Case: Silo users can attach instances to multicast groups + // Collaborator creates group via instance join (implicitly creates group) let group = create_group_via_instance_join( client, - user.id, + collaborator_user.id, "shared-group", "mcast-pool", ) @@ -183,7 +223,7 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( wait_for_group_active(client, "shared-group").await; - // User creates a second instance in a new project to test adding to existing group + // Collaborator creates a second project and instance to test adding to existing group let project_params = ProjectCreate { identity: IdentityMetadataCreateParams { name: "second-project".parse().unwrap(), @@ -195,7 +235,7 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( .body(Some(&project_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user.id)) + .authn_as(AuthnMode::SiloUser(collaborator_user.id)) .execute() .await .unwrap(); @@ -221,7 +261,7 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( anti_affinity_groups: Vec::new(), }; - let instance: Instance = NexusRequest::new( + let instance2: Instance = NexusRequest::new( RequestBuilder::new( client, http::Method::POST, @@ -230,7 +270,7 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( .body(Some(&instance_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user.id)) + .authn_as(AuthnMode::SiloUser(collaborator_user.id)) .execute() .await .unwrap() @@ -238,10 +278,9 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( .unwrap(); // User can attach additional instance to existing multicast group - // Uses instance-centric API: PUT /v1/instances/{instance}/multicast-groups/{group} let join_url = format!( "/v1/instances/{}/multicast-groups/{}?project=second-project", - instance.identity.name, group.identity.name + instance2.identity.name, group.identity.name ); let join_params = InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; @@ -251,76 +290,19 @@ async fn test_silo_users_can_attach_instances_to_multicast_groups( .body(Some(&join_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user.id)) + .authn_as(AuthnMode::SiloUser(collaborator_user.id)) .execute() .await .expect("User should be able to attach their instance to the group"); -} - -/// Test that authenticated silo users can read multicast groups without -/// requiring Fleet::Viewer role (verifies the Polar policy for read permission). -#[nexus_test] -async fn test_authenticated_users_can_read_multicast_groups( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - create_default_ip_pools(&client).await; - - // Get current silo info - let silo_url = format!("/v1/system/silos/{}", cptestctx.silo_name); - let silo: Silo = object_get(client, &silo_url).await; - - // Create multicast pool and link to silo - create_multicast_ip_pool(&client, "mcast-pool").await; - link_ip_pool(&client, "default-v4", &silo.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo.identity.id, false).await; - - // Create a collaborator user who can create groups - let creator = create_local_user( - client, - &silo, - &"creator-user".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - grant_iam( - client, - &silo_url, - SiloRole::Collaborator, - creator.id, - AuthnMode::PrivilegedUser, - ) - .await; - // Create a regular silo user with NO special roles (not even viewer) - let reader = create_local_user( - client, - &silo, - &"regular-user".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - // Creator creates a multicast group via instance join - let group = create_group_via_instance_join( - client, - creator.id, - "readable-group", - "mcast-pool", - ) - .await; - - // Wait for group to become active - wait_for_group_active(client, "readable-group").await; - - // Regular silo user (with no Fleet roles) can GET the multicast group + // Case: Authenticated users can read multicast groups + // Regular silo user (no Fleet roles) can GET the multicast group let get_group_url = mcast_group_url(&group.identity.name.to_string()); let read_group: MulticastGroup = NexusRequest::new( RequestBuilder::new(client, http::Method::GET, &get_group_url) .expect_status(Some(StatusCode::OK)), ) - .authn_as(AuthnMode::SiloUser(reader.id)) + .authn_as(AuthnMode::SiloUser(reader_user.id)) .execute() .await .expect("Silo user should be able to read multicast group") @@ -330,88 +312,243 @@ async fn test_authenticated_users_can_read_multicast_groups( assert_eq!(read_group.identity.id, group.identity.id); assert_eq!(read_group.identity.name, group.identity.name); - // Regular silo user can also LIST multicast groups - let list_groups: Vec = NexusRequest::iter_collection_authn( - client, - "/v1/multicast-groups", - "", - None, - ) - .await - .expect("Silo user should be able to list multicast groups") - .all_items; + // Regular silo user can also list multicast groups + let list_response: dropshot::ResultsPage = + NexusRequest::new( + RequestBuilder::new( + client, + http::Method::GET, + "/v1/multicast-groups", + ) + .expect_status(Some(StatusCode::OK)), + ) + .authn_as(AuthnMode::SiloUser(reader_user.id)) + .execute() + .await + .expect("Silo user should be able to list multicast groups") + .parsed_body() + .unwrap(); assert!( - list_groups.iter().any(|g| g.identity.id == group.identity.id), + list_response.items.iter().any(|g| g.identity.id == group.identity.id), "Multicast group should appear in list for silo user" ); - // Regular silo user can also lookup group by IP address - // The main multicast-groups endpoint accepts Name, ID, or IP + // Regular silo user can also look up group by IP address let multicast_ip = group.multicast_ip; let ip_lookup_url = format!("/v1/multicast-groups/{multicast_ip}"); let ip_lookup_group: MulticastGroup = NexusRequest::new( RequestBuilder::new(client, http::Method::GET, &ip_lookup_url) .expect_status(Some(StatusCode::OK)), ) - .authn_as(AuthnMode::SiloUser(reader.id)) + .authn_as(AuthnMode::SiloUser(reader_user.id)) .execute() .await - .expect("Silo user should be able to lookup group by IP") + .expect("Silo user should be able to look up group by IP") .parsed_body() .unwrap(); assert_eq!(ip_lookup_group.identity.id, group.identity.id); assert_eq!(ip_lookup_group.multicast_ip, multicast_ip); -} -/// Test that instances from different projects can attach to the same -/// fleet-scoped multicast group (no cross-project isolation). -#[nexus_test] -async fn test_cross_project_instance_attachment_allowed( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; + // Case: Lookup group by nonexistent IP returns 404 + let nonexistent_ip = "224.99.99.99"; + let nonexistent_ip_url = format!("/v1/multicast-groups/{nonexistent_ip}"); + NexusRequest::new( + RequestBuilder::new(client, http::Method::GET, &nonexistent_ip_url) + .expect_status(Some(StatusCode::NOT_FOUND)), + ) + .authn_as(AuthnMode::SiloUser(reader_user.id)) + .execute() + .await + .expect("Lookup by nonexistent IP should return 404"); + + // Case: Project-only users can access multicast groups in their project + // + // We sanity-check the group exists using a privileged iterator helper + // (which always uses PrivilegedUser), then exercise the list endpoint using + // `project_user` with a single-page GET. + let list_response: Vec = + NexusRequest::iter_collection_authn( + client, + "/v1/multicast-groups", + "", + None, + ) + .await + .expect("Should be able to list multicast groups") + .all_items; - // Create pools and projects - ops::join4( - create_default_ip_pools(&client), - create_project(client, "project1"), - create_project(client, "project2"), - create_multicast_ip_pool(&client, "mcast-pool"), + // Verify the group exists in the full list first + assert!( + list_response.iter().any(|g| g.identity.id == group.identity.id), + "Multicast group should exist in the fleet-wide list" + ); + + // Now verify project_user specifically can access the list endpoint + let project_user_list: dropshot::ResultsPage = + NexusRequest::object_get(client, "/v1/multicast-groups") + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect("Project-only user should be able to list multicast groups") + .parsed_body() + .unwrap(); + + assert!( + project_user_list + .items + .iter() + .any(|g| g.identity.id == group.identity.id), + "Project-only user should see multicast groups in list" + ); + + // Project-only user can read individual multicast group + let read_group_by_project_user: MulticastGroup = NexusRequest::new( + RequestBuilder::new(client, http::Method::GET, &get_group_url) + .expect_status(Some(StatusCode::OK)), ) - .await; + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect("Project-only user should be able to read multicast group") + .parsed_body() + .unwrap(); - // Create instances in both projects - let instance1 = create_instance(client, "project1", "instance1").await; - let instance2 = create_instance(client, "project2", "instance2").await; + assert_eq!(read_group_by_project_user.identity.id, group.identity.id); - // First instance join implicitly creates the group - let join_url1 = "/v1/instances/instance1/multicast-groups/cross-project-group?project=project1"; - let join_params = - InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; - put_upsert::<_, MulticastGroupMember>(client, join_url1, &join_params) - .await; + // A project-only user can create a multicast group via instance join. + // They create an instance in their project, then add it as a member. + let project_instance_params = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: "project-user-instance".parse().unwrap(), + description: "Instance for testing project-only user".to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: "project-user-instance".parse::().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![], + multicast_groups: vec![], + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: false, + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + }; + + let project_instance: Instance = NexusRequest::new( + RequestBuilder::new( + client, + http::Method::POST, + "/v1/instances?project=project-only", + ) + .body(Some(&project_instance_params)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect( + "Project-only user should be able to create instance in their project", + ) + .parsed_body() + .unwrap(); + + // Join instance to implicitly create a new group + let project_join_url = format!( + "/v1/instances/{}/multicast-groups/created-by-project-user?project={}", + project_instance.identity.id, project_only.identity.id + ); + + NexusRequest::new( + RequestBuilder::new(client, http::Method::PUT, &project_join_url) + .body(Some(&join_params)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect( + "Project-only user should be able to join group (implicitly creates group)", + ); // Fetch the implicitly created group - let group: MulticastGroup = - object_get(client, &mcast_group_url("cross-project-group")).await; + let user_created_group: MulticastGroup = + object_get(client, &mcast_group_url("created-by-project-user")).await; + + assert_eq!( + user_created_group.identity.name.as_str(), + "created-by-project-user" + ); + + // A project-only user can create a second instance and attach to existing group + let instance_name2 = "project-user-instance-2"; + let instances_url = + format!("/v1/instances?project={}", project_only.identity.id); + let instance_params2 = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: instance_name2.parse().unwrap(), + description: "Second instance created by project-only user" + .to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: instance_name2.parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![], + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: false, + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + multicast_groups: Vec::new(), + }; + let instance2: Instance = NexusRequest::objects_post( + client, + &instances_url, + &instance_params2, + ) + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect( + "Project-only user should be able to create an instance in the project", + ) + .parsed_body() + .expect("Should parse created instance"); - // Attach instance from project2 to the same group + // Project-only user can attach the instance they own to a fleet-scoped group let join_url2 = format!( - "/v1/instances/instance2/multicast-groups/{}?project=project2", - group.identity.name + "/v1/instances/{}/multicast-groups/{}?project={}", + instance2.identity.id, group.identity.name, project_only.identity.id + ); + NexusRequest::new( + RequestBuilder::new(client, http::Method::PUT, &join_url2) + .body(Some(&join_params)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::SiloUser(project_user.id)) + .execute() + .await + .expect( + "Project-only user should be able to attach their instance to the group", ); - put_upsert::<_, MulticastGroupMember>(client, &join_url2, &join_params) - .await; - // Verify both instances are members of the same group + // Verify instance is now a member let members = - list_multicast_group_members(client, "cross-project-group").await; - assert_eq!(members.len(), 2, "Should have 2 members"); - let instance_ids: Vec<_> = members.iter().map(|m| m.instance_id).collect(); - assert!(instance_ids.contains(&instance1.identity.id)); - assert!(instance_ids.contains(&instance2.identity.id)); + list_multicast_group_members(client, &group.identity.name.to_string()) + .await; + assert!( + members.iter().any(|m| m.instance_id == instance2.identity.id), + "Instance2 should be a member of the group" + ); } /// Verify that unauthenticated users cannot access any multicast API endpoints. @@ -429,14 +566,12 @@ async fn test_unauthenticated_access_denied( let client = &cptestctx.external_client; create_default_ip_pools(&client).await; - // Get current silo info - let silo_url = format!("/v1/system/silos/{}", cptestctx.silo_name); + // Get DEFAULT_SILO info (pools are linked to DEFAULT_SILO) + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.identity().name); let silo: Silo = object_get(client, &silo_url).await; - // Create multicast pool and link to silo + // Create multicast pool (auto-links to DEFAULT_SILO) create_multicast_ip_pool(&client, "mcast-pool").await; - link_ip_pool(&client, "default-v4", &silo.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo.identity.id, false).await; // Create a collaborator user who can create groups let creator = create_local_user( @@ -533,14 +668,12 @@ async fn test_unprivileged_users_can_list_group_members( let client = &cptestctx.external_client; create_default_ip_pools(&client).await; - // Get current silo info - let silo_url = format!("/v1/system/silos/{}", cptestctx.silo_name); + // Get DEFAULT_SILO info (pools are linked to DEFAULT_SILO) + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.identity().name); let silo: Silo = object_get(client, &silo_url).await; - // Create multicast pool and link to silo + // Create multicast pool (auto-links to DEFAULT_SILO) create_multicast_ip_pool(&client, "mcast-pool").await; - link_ip_pool(&client, "default-v4", &silo.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo.identity.id, false).await; // Create two regular silo users let privileged_user = create_local_user( @@ -712,272 +845,26 @@ async fn test_unprivileged_users_can_list_group_members( ); } -/// Test that authenticated silo users with ONLY project-level roles (no -/// silo-level roles) can still access multicast groups. This verifies that -/// being an authenticated SiloUser is sufficient - multicast group access does -/// not depend on having any specific silo-level or project-level roles. +/// Consolidated test for cross-silo multicast isolation. +/// +/// This test verifies the cross-silo authorization boundaries for multicast: +/// - Silo admins can read but not modify other silos' groups +/// - Cross-silo instance attachment works when pools are linked +/// - Silos cannot use unlinked pools /// -/// This verifies that project-only users can: -/// - List and read multicast groups (fleet-scoped discovery) -/// - Implicitly create groups via instance join API (group owned by their silo) -/// - Create instances and attach them to groups +/// Setup: Creates 2 silos with users and pools with different linking configurations. #[nexus_test] -async fn test_project_only_users_can_access_multicast_groups( +async fn test_cross_silo_multicast_isolation( cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; - // create_default_ip_pools already links "default" pool to the DEFAULT_SILO create_default_ip_pools(&client).await; - // Create multicast pool (already linked to DEFAULT_SILO by helper) - create_multicast_ip_pool(&client, "mcast-pool").await; - - // Get the DEFAULT silo (same silo as the privileged test user) - // This ensures that when we create a project using AuthnMode::PrivilegedUser, - // it will be created in the same silo as our project_user - use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; - let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.identity().name); - let silo: Silo = object_get(client, &silo_url).await; - - // Create a user with NO silo-level roles (only project-level roles) - let project_user = create_local_user( - client, - &silo, - &"project-only-user".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - // Create a project using AuthnMode::PrivilegedUser, which creates it in DEFAULT_SILO - // (the same silo where we created project_user above) - let project = create_project(client, "project-only").await; - - // Grant ONLY project-level role (Project::Collaborator), NO silo roles - // Users with project-level roles can work within that project even without - // silo-level roles, as long as they reference the project by ID - let project_url = format!("/v1/projects/{}", project.identity.name); - grant_iam( - client, - &project_url, - ProjectRole::Collaborator, - project_user.id, - AuthnMode::PrivilegedUser, - ) - .await; - - // Create a silo collaborator who can create the first group - let creator = create_local_user( - client, - &silo, - &"creator-user".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - grant_iam( - client, - &silo_url, - SiloRole::Collaborator, - creator.id, - AuthnMode::PrivilegedUser, - ) - .await; - - // Creator creates a multicast group via instance join - let group = create_group_via_instance_join( - client, - creator.id, - "project-user-test", - "mcast-pool", - ) - .await; - - // Project-only user CAN LIST multicast groups (no silo roles needed) - let list_response: dropshot::ResultsPage = - NexusRequest::object_get(client, "/v1/multicast-groups") - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect("Project-only user should be able to list multicast groups") - .parsed_body() - .unwrap(); - - let list_groups = list_response.items; - - assert!( - list_groups.iter().any(|g| g.identity.id == group.identity.id), - "Project-only user should see multicast groups in list" - ); - - // Project-only user CAN READ individual multicast group - let get_group_url = mcast_group_url(&group.identity.name.to_string()); - let read_group: MulticastGroup = NexusRequest::new( - RequestBuilder::new(client, http::Method::GET, &get_group_url) - .expect_status(Some(StatusCode::OK)), - ) - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect("Project-only user should be able to read multicast group") - .parsed_body() - .unwrap(); - - assert_eq!(read_group.identity.id, group.identity.id); - - // Project-only user CAN CREATE a multicast group via instance join - // They create an instance in their project, then add it as a member - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: "project-user-instance".parse().unwrap(), - description: "Instance for testing project-only user".to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: "project-user-instance".parse::().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![], - multicast_groups: vec![], - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: false, - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - }; - - let instance: Instance = NexusRequest::new( - RequestBuilder::new( - client, - http::Method::POST, - "/v1/instances?project=project-only", - ) - .body(Some(&instance_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect( - "Project-only user should be able to create instance in their project", - ) - .parsed_body() - .unwrap(); - - // Join instance to implicitly create the group - // Uses instance-centric API: PUT /v1/instances/{instance}/multicast-groups/{group} - let join_url = format!( - "/v1/instances/{}/multicast-groups/created-by-project-user?project={}", - instance.identity.id, project.identity.id - ); - let join_params = - InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; - - NexusRequest::new( - RequestBuilder::new(client, http::Method::PUT, &join_url) - .body(Some(&join_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect("Project-only user should be able to join group (implicitly creates group)"); - - // Fetch the implicitly created group - let user_created_group: MulticastGroup = - object_get(client, &mcast_group_url("created-by-project-user")).await; - - assert_eq!( - user_created_group.identity.name.as_str(), - "created-by-project-user" - ); - - // Project-only user CAN CREATE a second instance in the project (Project::Collaborator) - // Must use project ID (not name) since user has no silo-level roles - let instance_name2 = "project-user-instance-2"; - let instances_url = - format!("/v1/instances?project={}", project.identity.id); - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: instance_name2.parse().unwrap(), - description: "Second instance created by project-only user" - .to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: instance_name2.parse().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![], - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: false, - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - multicast_groups: Vec::new(), - }; - let instance2: Instance = NexusRequest::objects_post( - client, - &instances_url, - &instance_params, - ) - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect( - "Project-only user should be able to create an instance in the project", - ) - .parsed_body() - .expect("Should parse created instance"); - - // Project-only user can attach the instance they own to a fleet-scoped group - // Uses instance-centric API: PUT /v1/instances/{instance}/multicast-groups/{group} - let join_url2 = format!( - "/v1/instances/{}/multicast-groups/{}?project={}", - instance2.identity.id, group.identity.name, project.identity.id - ); - NexusRequest::new( - RequestBuilder::new(client, http::Method::PUT, &join_url2) - .body(Some(&join_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(project_user.id)) - .execute() - .await - .expect("Project-only user should be able to attach their instance to the group"); - - // Verify both instances are now members - let members = - list_multicast_group_members(client, &group.identity.name.to_string()) - .await; - assert!( - members.iter().any(|m| m.instance_id == instance2.identity.id), - "Instance2 should be a member of the group" - ); -} - -/// Test that users from different silos can both read multicast groups -/// (fleet-scoped visibility). This validates the core cross-silo multicast use case: -/// multicast groups are discoverable across silo boundaries. -/// -/// This test verifies: -/// - Users in different silos can both discover and read the same multicast groups -/// - Groups created by Silo A are visible to Silo B users (and vice versa) -#[nexus_test] -async fn test_silo_admins_cannot_modify_other_silos_groups( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - create_default_ip_pools(&client).await; - - // Create multicast IP pool (fleet-scoped) + // Create multicast IP pool (fleet-scoped) create_multicast_ip_pool(&client, "mcast-pool").await; // Create Silo A (not using default test silo - it has Admin->FleetAdmin mapping) - // We explicitly create both silos with no fleet role mappings to test the + // We explicitly create silos with no fleet role mappings to test the // authorization boundary correctly. let silo_a_params = SiloCreate { identity: IdentityMetadataCreateParams { @@ -1005,7 +892,7 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( link_ip_pool(&client, "default-v4", &silo_a.identity.id, true).await; link_ip_pool(&client, "mcast-pool", &silo_a.identity.id, false).await; - // Create Silo B + // Create Silo B - linked to mcast-pool (for cross-silo attachment tests) let silo_b_params = SiloCreate { identity: IdentityMetadataCreateParams { name: "silo-b".parse().unwrap(), @@ -1030,9 +917,38 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( let silo_b_url = format!("/v1/system/silos/{}", silo_b.identity.name); link_ip_pool(&client, "default-v4", &silo_b.identity.id, true).await; + // Link mcast-pool to Silo B as well - cross-silo multicast works by + // linking the same pool to multiple silos (pool linking = access control) link_ip_pool(&client, "mcast-pool", &silo_b.identity.id, false).await; - // Create silo admin for Silo A + // Create Silo C - not linked to mcast-pool (for unlinked pool tests) + let silo_c_params = SiloCreate { + identity: IdentityMetadataCreateParams { + name: "silo-c-unlinked".parse().unwrap(), + description: "Silo without multicast pool linked".to_string(), + }, + quotas: SiloQuotasCreate::empty(), + discoverable: false, + identity_mode: SiloIdentityMode::LocalOnly, + admin_group_name: None, + tls_certificates: vec![], + mapped_fleet_roles: Default::default(), + }; + + let silo_c: Silo = + NexusRequest::objects_post(client, "/v1/system/silos", &silo_c_params) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + let silo_c_url = format!("/v1/system/silos/{}", silo_c.identity.name); + // Only link the default pool to Silo C (not the mcast-pool) + link_ip_pool(&client, "default-v4", &silo_c.identity.id, true).await; + + // Create admin for Silo A let admin_a = create_local_user( client, &silo_a, @@ -1050,7 +966,7 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( ) .await; - // Create silo admin for Silo B + // Create admin for Silo B let admin_b = create_local_user( client, &silo_b, @@ -1068,6 +984,26 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( ) .await; + // Create user for Silo C (unlinked pool tests) + let user_c = create_local_user( + client, + &silo_c, + &"user-c".parse().unwrap(), + UserPassword::LoginDisallowed, + ) + .await; + + grant_iam( + client, + &silo_c_url, + SiloRole::Collaborator, + user_c.id, + AuthnMode::PrivilegedUser, + ) + .await; + + // Case: Silo admin cannot modify other silo's groups + // // Admin A creates a multicast group via instance join (owned by Silo A) let group_a = create_group_via_instance_join( client, @@ -1086,7 +1022,7 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( ) .await; - // Both silo admins CAN READ each other's groups (fleet-scoped visibility) + // Both silo admins can read each other's groups (fleet-scoped visibility) let read_b_by_a: MulticastGroup = NexusRequest::new( RequestBuilder::new( client, @@ -1112,111 +1048,18 @@ async fn test_silo_admins_cannot_modify_other_silos_groups( ) .expect_status(Some(StatusCode::OK)), ) - .authn_as(AuthnMode::SiloUser(admin_b.id)) - .execute() - .await - .unwrap() - .parsed_body() - .unwrap(); - - assert_eq!(read_a_by_b.identity.id, group_a.identity.id); -} - -/// Test that instances from different silos can attach to the same multicast -/// group when both silos have the multicast pool linked. -/// -/// Cross-silo multicast works by linking the same pool to multiple silos. -/// Pool linking is the mechanism of access control: a silo can only use -/// pools that are linked to it. -/// -/// This test verifies: -/// - Users in different silos (both linked to the pool) can join the same group -/// - Instances from Silo A can attach to a group -/// - Instances from Silo B can attach to the SAME group -/// - Both members can be listed together in the group membership -#[nexus_test] -async fn test_cross_silo_instance_attachment( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - create_default_ip_pools(&client).await; - - // Create multicast IP pool (fleet-scoped) - create_multicast_ip_pool(&client, "mcast-pool").await; - - // Get Silo A (default test silo) - let silo_a_url = format!("/v1/system/silos/{}", cptestctx.silo_name); - let silo_a: Silo = object_get(client, &silo_a_url).await; - link_ip_pool(&client, "default-v4", &silo_a.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo_a.identity.id, false).await; - - // Create Silo B - let silo_b_params = SiloCreate { - identity: IdentityMetadataCreateParams { - name: "silo-b-cross".parse().unwrap(), - description: "Second silo for cross-silo instance attachment" - .to_string(), - }, - quotas: SiloQuotasCreate::empty(), - discoverable: false, - identity_mode: SiloIdentityMode::LocalOnly, - admin_group_name: None, - tls_certificates: vec![], - mapped_fleet_roles: Default::default(), - }; - - let silo_b: Silo = - NexusRequest::objects_post(client, "/v1/system/silos", &silo_b_params) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body() - .unwrap(); - - let silo_b_url = format!("/v1/system/silos/{}", silo_b.identity.name); - link_ip_pool(&client, "default-v4", &silo_b.identity.id, true).await; - // Link mcast-pool to Silo B as well - cross-silo multicast works by - // linking the same pool to multiple silos (pool linking = access control) - link_ip_pool(&client, "mcast-pool", &silo_b.identity.id, false).await; - - // Create user in Silo A - let user_a = create_local_user( - client, - &silo_a, - &"user-a".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - grant_iam( - client, - &silo_a_url, - SiloRole::Collaborator, - user_a.id, - AuthnMode::PrivilegedUser, - ) - .await; - - // Create user in Silo B - let user_b = create_local_user( - client, - &silo_b, - &"user-b".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - grant_iam( - client, - &silo_b_url, - SiloRole::Collaborator, - user_b.id, - AuthnMode::PrivilegedUser, - ) - .await; + .authn_as(AuthnMode::SiloUser(admin_b.id)) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + + assert_eq!(read_a_by_b.identity.id, group_a.identity.id); - // User A creates a project in Silo A + // Case: Cross-silo instance attachment behavior + // + // Admin A creates a project and instance in Silo A let project_a_params = ProjectCreate { identity: IdentityMetadataCreateParams { name: "project-silo-a".parse().unwrap(), @@ -1229,12 +1072,12 @@ async fn test_cross_silo_instance_attachment( .body(Some(&project_a_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_a.id)) + .authn_as(AuthnMode::SiloUser(admin_a.id)) .execute() .await .unwrap(); - // User B creates a project in Silo B + // Admin B creates a project in Silo B let project_b_params = ProjectCreate { identity: IdentityMetadataCreateParams { name: "project-silo-b".parse().unwrap(), @@ -1247,12 +1090,12 @@ async fn test_cross_silo_instance_attachment( .body(Some(&project_b_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_b.id)) + .authn_as(AuthnMode::SiloUser(admin_b.id)) .execute() .await .unwrap(); - // User A creates instance in Silo A's project + // Admin A creates instance in Silo A's project let instance_a_params = InstanceCreate { identity: IdentityMetadataCreateParams { name: "instance-silo-a".parse().unwrap(), @@ -1283,14 +1126,14 @@ async fn test_cross_silo_instance_attachment( .body(Some(&instance_a_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_a.id)) + .authn_as(AuthnMode::SiloUser(admin_a.id)) .execute() .await .unwrap() .parsed_body() .unwrap(); - // User B creates instance in Silo B's project + // Admin B creates instance in Silo B's project let instance_b_params = InstanceCreate { identity: IdentityMetadataCreateParams { name: "instance-silo-b".parse().unwrap(), @@ -1321,15 +1164,14 @@ async fn test_cross_silo_instance_attachment( .body(Some(&instance_b_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_b.id)) + .authn_as(AuthnMode::SiloUser(admin_b.id)) .execute() .await .unwrap() .parsed_body() .unwrap(); - // User A joins their instance (from Silo A) to a new group (implicitly creates it) - // Uses instance-centric API: PUT /v1/instances/{instance}/multicast-groups/{group} + // Admin A joins their instance (from Silo A) to a new group (implicitly creates it) let group_name = "cross-silo-group"; let join_url_a = format!( "/v1/instances/{}/multicast-groups/{}?project=project-silo-a", @@ -1343,16 +1185,16 @@ async fn test_cross_silo_instance_attachment( .body(Some(&join_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_a.id)) + .authn_as(AuthnMode::SiloUser(admin_a.id)) .execute() .await - .expect("User A should be able to join instance to group"); + .expect("Admin A should be able to join instance to group"); // Fetch the implicitly created group let group: MulticastGroup = object_get(client, &mcast_group_url(group_name)).await; - // User B joins their instance (from Silo B) to the same fleet-scoped group + // Admin B joins their instance (from Silo B) to the same fleet-scoped group // This is the key test: cross-silo instance attachment should succeed let join_url_b = format!( "/v1/instances/{}/multicast-groups/{}?project=project-silo-b", @@ -1364,10 +1206,10 @@ async fn test_cross_silo_instance_attachment( .body(Some(&join_params)) .expect_status(Some(StatusCode::CREATED)), ) - .authn_as(AuthnMode::SiloUser(user_b.id)) + .authn_as(AuthnMode::SiloUser(admin_b.id)) .execute() .await - .expect("User B should be able to join instance to the same group"); + .expect("Admin B should be able to join instance to the same group"); // Both instances should be visible in the group's member list let members_url = mcast_group_members_url(&group.identity.name.to_string()); @@ -1398,107 +1240,27 @@ async fn test_cross_silo_instance_attachment( // Both users should be able to see the complete member list (both silos linked) let members_by_a: dropshot::ResultsPage = NexusRequest::object_get(client, &members_url) - .authn_as(AuthnMode::SiloUser(user_a.id)) + .authn_as(AuthnMode::SiloUser(admin_a.id)) .execute() .await .unwrap() .parsed_body() .unwrap(); - assert_eq!(members_by_a.items.len(), 2, "User A should see both members"); + assert_eq!(members_by_a.items.len(), 2, "Admin A should see both members"); let members_by_b: dropshot::ResultsPage = NexusRequest::object_get(client, &members_url) - .authn_as(AuthnMode::SiloUser(user_b.id)) + .authn_as(AuthnMode::SiloUser(admin_b.id)) .execute() .await .unwrap() .parsed_body() .unwrap(); - assert_eq!(members_by_b.items.len(), 2, "User B should see both members"); - - // Case: Cross-silo IP lookup - // User B (from Silo B, which has mcast-pool linked) can lookup the group - // by its IP address. Cross-silo access works because both silos are - // linked to the same pool. - let multicast_ip = group.multicast_ip; - let ip_lookup_url = format!("/v1/multicast-groups/{multicast_ip}"); - let ip_lookup_result: MulticastGroup = NexusRequest::new( - RequestBuilder::new(client, http::Method::GET, &ip_lookup_url) - .expect_status(Some(StatusCode::OK)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should be able to lookup group by IP (pool linked to silo)") - .parsed_body() - .unwrap(); - - assert_eq!( - ip_lookup_result.identity.id, group.identity.id, - "IP lookup should return the correct group" - ); - assert_eq!( - ip_lookup_result.multicast_ip, multicast_ip, - "IP lookup result should have matching multicast_ip" - ); - - // Case: Cross-silo new group creation - // User B (from Silo B, which has mcast-pool linked) can create a new - // multicast group. Pool linking is the mechanism of access control. - let new_group_name = "user-b-created-group"; - let join_new_group_url = format!( - "/v1/instances/{}/multicast-groups/{}?project=project-silo-b", - instance_b.identity.id, new_group_name - ); - - // This should succeed because mcast-pool is linked to Silo B - NexusRequest::new( - RequestBuilder::new(client, http::Method::PUT, &join_new_group_url) - .body(Some(&join_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should create new group (pool linked to silo)"); - - // Verify the new group was created and is accessible - let new_group: MulticastGroup = - object_get(client, &mcast_group_url(new_group_name)).await; - assert_eq!( - new_group.identity.name.as_str(), - new_group_name, - "New group should have correct name" - ); - - // Leave the new group (triggers implicit deletion) - NexusRequest::new( - RequestBuilder::new(client, http::Method::DELETE, &join_new_group_url) - .expect_status(Some(StatusCode::NO_CONTENT)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("Should clean up new group member"); - - // Rejoin User B's instance to the original group for subsequent tests - NexusRequest::new( - RequestBuilder::new(client, http::Method::PUT, &join_url_b) - .body(Some(&join_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should rejoin original group"); - - // Case: Cross-silo detach - // Using instance-centric API: DELETE /v1/instances/{instance}/multicast-groups/{group} + assert_eq!(members_by_b.items.len(), 2, "Admin B should see both members"); - // User A cannot detach User B's instance (404 - can't see Silo B's instance) - // Using instance ID since we're crossing silo boundaries + // Admin A cannot detach Admin B's instance (404 - can't see Silo B's instance) let member_delete_b_by_a_url = format!( "/v1/instances/{}/multicast-groups/{}", instance_b.identity.id, group.identity.name, @@ -1512,12 +1274,12 @@ async fn test_cross_silo_instance_attachment( ) .expect_status(Some(StatusCode::NOT_FOUND)), ) - .authn_as(AuthnMode::SiloUser(user_a.id)) + .authn_as(AuthnMode::SiloUser(admin_a.id)) .execute() .await - .expect("User A should get 404 when trying to detach Silo B's instance"); + .expect("Admin A should get 404 when trying to detach Silo B's instance"); - // User BcanN detach their own instance from the group (even though owned by different silo) + // Admin B can detach their own instance from the group let member_delete_b_url = format!( "/v1/instances/{}/multicast-groups/{}", instance_b.identity.id, group.identity.name, @@ -1527,12 +1289,12 @@ async fn test_cross_silo_instance_attachment( RequestBuilder::new(client, http::Method::DELETE, &member_delete_b_url) .expect_status(Some(StatusCode::NO_CONTENT)), ) - .authn_as(AuthnMode::SiloUser(user_b.id)) + .authn_as(AuthnMode::SiloUser(admin_b.id)) .execute() .await - .expect("User B should be able to detach their own instance"); + .expect("Admin B should be able to detach their own instance"); - // Verify only User A's instance remains + // Verify only Admin A's instance remains let members_after_detach: dropshot::ResultsPage = NexusRequest::object_get(client, &members_url) .authn_as(AuthnMode::PrivilegedUser) @@ -1551,6 +1313,85 @@ async fn test_cross_silo_instance_attachment( members_after_detach.items[0].instance_id, instance_a.identity.id, "Remaining member should be Silo A's instance" ); + + // Case: Silo cannot use unlinked pool + // + // User C creates a project and instance in Silo C (which has no mcast-pool link) + let project_c_params = ProjectCreate { + identity: IdentityMetadataCreateParams { + name: "project-silo-c".parse().unwrap(), + description: "Project in Silo C".to_string(), + }, + }; + + NexusRequest::new( + RequestBuilder::new(client, http::Method::POST, "/v1/projects") + .body(Some(&project_c_params)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::SiloUser(user_c.id)) + .execute() + .await + .expect("User C should create project in Silo C"); + + let instance_c_params = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: "instance-silo-c".parse().unwrap(), + description: "Instance in Silo C".to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: "instance-silo-c".parse::().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![], + multicast_groups: vec![], + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: false, + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + }; + + NexusRequest::new( + RequestBuilder::new( + client, + http::Method::POST, + "/v1/instances?project=project-silo-c", + ) + .body(Some(&instance_c_params)) + .expect_status(Some(StatusCode::CREATED)), + ) + .authn_as(AuthnMode::SiloUser(user_c.id)) + .execute() + .await + .expect("User C should create instance in Silo C"); + + // User C tries to join a multicast group - should fail because mcast-pool + // is not linked to Silo C + let join_url_c = "/v1/instances/instance-silo-c/multicast-groups/test-group?project=project-silo-c"; + + let error = NexusRequest::new( + RequestBuilder::new(client, http::Method::PUT, join_url_c) + .body(Some(&join_params)) + .expect_status(Some(StatusCode::BAD_REQUEST)), + ) + .authn_as(AuthnMode::SiloUser(user_c.id)) + .execute() + .await + .expect("User C should get 400 when no pool is linked to their silo"); + + // Verify the error indicates no pool was found + let error_body: dropshot::HttpErrorResponseBody = + error.parsed_body().unwrap(); + assert_eq!( + error_body.error_code, + Some("InvalidRequest".to_string()), + "Expected InvalidRequest for no pool available, got: {:?}", + error_body.error_code + ); } /// Test that both instance join endpoints have identical permission behavior @@ -1577,7 +1418,6 @@ async fn test_both_member_endpoints_have_same_permissions( // Get the DEFAULT silo (same silo as PrivilegedUser) // This ensures that when we create a project using AuthnMode::PrivilegedUser, // it will be created in the same silo as our users - use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.identity().name); let silo: Silo = object_get(client, &silo_url).await; @@ -1600,7 +1440,7 @@ async fn test_both_member_endpoints_have_same_permissions( .await; // Create User B (unprivileged) in the same silo - // User B intentionally has NO silo-level roles - they're just a regular user + // User B intentionally has no silo-level roles - they're just a regular user let user_b = create_local_user( client, &silo, @@ -1714,7 +1554,8 @@ async fn test_both_member_endpoints_have_same_permissions( ) .await; - // Create a second instance in the project (User A still owns it, but User B now has access) + // Create a second instance in the project + // (User A still owns it, but User B now has access) let instance_b = create_instance( client, project_a.identity.name.as_str(), @@ -1722,7 +1563,8 @@ async fn test_both_member_endpoints_have_same_permissions( ) .await; - // User B should now succeed via the instance-centric endpoint (has Instance::Modify permission) + // User B should now succeed via the instance-centric endpoint + // (has Instance::Modify permission) let join_url_b = format!( "/v1/instances/{}/multicast-groups/{}?project={}", instance_b.identity.id, group_a.identity.name, project_a.identity.name @@ -1752,7 +1594,8 @@ async fn test_both_member_endpoints_have_same_permissions( ) .await; - // User B should ALSO succeed via the instance-centric endpoint (same permission check) + // User B should also succeed via the instance-centric endpoint + // (same permission check) let instance_centric_url_c = format!( "/v1/instances/{}/multicast-groups/{}", instance_c.identity.id, group_a.identity.id @@ -1770,157 +1613,4 @@ async fn test_both_member_endpoints_have_same_permissions( .expect( "User B should succeed via instance-centric endpoint with permission", ); - - // This verifies both endpoints have identical permission behavior: - // - Without permission: both return 404 - // - With project-level access granted: both succeed with 201 Created -} - -/// Test that a silo cannot use a multicast pool that is not linked to it. -/// -/// Pool linking is the access control mechanism for multicast. A silo can only -/// use multicast pools that are explicitly linked to it. This test verifies -/// that a user in Silo B cannot join a multicast group when the pool is only -/// linked to Silo A. -#[nexus_test] -async fn test_silo_cannot_use_unlinked_pool( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - create_default_ip_pools(&client).await; - - // Create multicast IP pool (fleet-scoped) - create_multicast_ip_pool(&client, "mcast-pool").await; - - // Get Silo A (default test silo) and link pools to it - let silo_a_url = format!("/v1/system/silos/{}", cptestctx.silo_name); - let silo_a: Silo = object_get(client, &silo_a_url).await; - link_ip_pool(&client, "default-v4", &silo_a.identity.id, true).await; - link_ip_pool(&client, "mcast-pool", &silo_a.identity.id, false).await; - - // Create Silo B (but do not link mcast-pool to it) - let silo_b_params = SiloCreate { - identity: IdentityMetadataCreateParams { - name: "silo-b-unlinked".parse().unwrap(), - description: "Silo without multicast pool linked".to_string(), - }, - quotas: SiloQuotasCreate::empty(), - discoverable: false, - identity_mode: SiloIdentityMode::LocalOnly, - admin_group_name: None, - tls_certificates: vec![], - mapped_fleet_roles: Default::default(), - }; - - let silo_b: Silo = - NexusRequest::objects_post(client, "/v1/system/silos", &silo_b_params) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body() - .unwrap(); - - let silo_b_url = format!("/v1/system/silos/{}", silo_b.identity.name); - - // Link only the default pool to Silo B (not the mcast-pool) - link_ip_pool(&client, "default-v4", &silo_b.identity.id, true).await; - - // Create user in Silo B - let user_b = create_local_user( - client, - &silo_b, - &"user-b-unlinked".parse().unwrap(), - UserPassword::LoginDisallowed, - ) - .await; - - grant_iam( - client, - &silo_b_url, - SiloRole::Collaborator, - user_b.id, - AuthnMode::PrivilegedUser, - ) - .await; - - // User B creates a project and instance in Silo B - let project_params = ProjectCreate { - identity: IdentityMetadataCreateParams { - name: "project-silo-b".parse().unwrap(), - description: "Project in Silo B".to_string(), - }, - }; - - NexusRequest::new( - RequestBuilder::new(client, http::Method::POST, "/v1/projects") - .body(Some(&project_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should create project in Silo B"); - - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: "instance-silo-b".parse().unwrap(), - description: "Instance in Silo B".to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: "instance-silo-b".parse::().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![], - multicast_groups: vec![], - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: false, - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - }; - - NexusRequest::new( - RequestBuilder::new( - client, - http::Method::POST, - "/v1/instances?project=project-silo-b", - ) - .body(Some(&instance_params)) - .expect_status(Some(StatusCode::CREATED)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should create instance in Silo B"); - - // User B tries to join a multicast group - should fail because mcast-pool - // is not linked to Silo B - // Uses instance-centric API: PUT /v1/instances/{instance}/multicast-groups/{group} - let join_url = "/v1/instances/instance-silo-b/multicast-groups/test-group?project=project-silo-b"; - let join_params = - InstanceMulticastGroupJoin { source_ips: None, ip_version: None }; - - let error = NexusRequest::new( - RequestBuilder::new(client, http::Method::PUT, join_url) - .body(Some(&join_params)) - .expect_status(Some(StatusCode::BAD_REQUEST)), - ) - .authn_as(AuthnMode::SiloUser(user_b.id)) - .execute() - .await - .expect("User B should get 400 when no pool is linked to their silo"); - - // Verify the error indicates no pool was found - let error_body: dropshot::HttpErrorResponseBody = - error.parsed_body().unwrap(); - assert_eq!( - error_body.error_code, - Some("InvalidRequest".to_string()), - "Expected InvalidRequest for no pool available, got: {:?}", - error_body.error_code - ); } diff --git a/nexus/tests/integration_tests/multicast/failures.rs b/nexus/tests/integration_tests/multicast/failures.rs index 4f8f32ccd30..d48a07123fd 100644 --- a/nexus/tests/integration_tests/multicast/failures.rs +++ b/nexus/tests/integration_tests/multicast/failures.rs @@ -46,20 +46,24 @@ use omicron_common::api::external::{InstanceState, SwitchLocation}; use omicron_uuid_kinds::{InstanceUuid, MulticastGroupUuid}; use super::*; -use crate::integration_tests::instances::{ - instance_simulate, instance_wait_for_state, -}; +use crate::integration_tests::instances as instance_helpers; +/// Test DPD failure during group "Creating" state. +/// +/// When DPD is unavailable during group activation: +/// - Group stays in Creating state +/// - Member stays in Joining state +/// - After DPD recovery, group becomes Active and member becomes Joined #[nexus_test] -async fn test_multicast_group_dpd_communication_failure_recovery( +async fn test_dpd_failure_during_creating_state( cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; let project_name = "test-project"; - let group_name = "dpd-failure-group"; - let instance_name = "dpd-failure-instance"; + let group_name = "creating-dpd-fail-group"; + let instance_name = "creating-fail-instance"; - // Setup: project, pools - parallelize creation + // Setup: project, pools ops::join3( create_project(&client, project_name), create_default_ip_pools(&client), @@ -70,15 +74,17 @@ async fn test_multicast_group_dpd_communication_failure_recovery( // Create instance first create_instance(client, project_name, instance_name).await; - // Stop DPD before implicit creation to test failure recovery + // Stop DPD before implicit creation cptestctx.stop_dendrite(SwitchLocation::Switch0).await; // Add instance to multicast group via instance-centric API multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Verify group was implicitly created and is in "Creating" state since DPD is unavailable - // The reconciler can't progress the group to Active without DPD communication + // Wait for reconciler to process + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; + + // Check group state - should remain in "Creating" since DPD is down let group_get_url = mcast_group_url(group_name); let fetched_group: MulticastGroup = object_get(client, &group_get_url).await; @@ -88,53 +94,23 @@ async fn test_multicast_group_dpd_communication_failure_recovery( "Group should remain in Creating state when DPD is unavailable, found: {}", fetched_group.state ); - - // Verify group properties are maintained despite DPD issues assert_eq!(fetched_group.identity.name.as_str(), group_name); - // Case: Verify member state during DPD failure - // Instance is running, so member has sled_id, - // but DPD is unavailable so it can't be programmed. + // Verify member is in Joining state let members = list_multicast_group_members(client, group_name).await; assert_eq!(members.len(), 1, "Should have exactly one member"); assert_eq!( members[0].state, "Joining", - "Member should be Joining when DPD unavailable (waiting to be programmed)" + "Member should be Joining when DPD unavailable" ); - // Start instance so it has a valid VMM state for recovery - let instance: Instance = object_get( - client, - &format!("/v1/instances/{instance_name}?project={project_name}"), - ) - .await; - let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - let nexus = &cptestctx.server.server_context().nexus; - - let start_url = - format!("/v1/instances/{instance_name}/start?project={project_name}"); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &start_url) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Should start instance"); - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; - - // Restart DPD and verify member recovers to "Joined" + // Recovery: restart DPD and verify group/member recover cptestctx.restart_dendrite(SwitchLocation::Switch0).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Group should become Active and member should recover to Joined wait_for_group_active(client, group_name).await; wait_for_member_state( cptestctx, group_name, - instance.identity.id, + members[0].instance_id, nexus_db_model::MulticastGroupMemberState::Joined, ) .await; @@ -146,172 +122,17 @@ async fn test_multicast_group_dpd_communication_failure_recovery( "Member should recover to Joined after DPD restart" ); + // Cleanup cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; wait_for_group_deleted(cptestctx, group_name).await; } -#[nexus_test] -async fn test_multicast_reconciler_state_consistency_validation( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "test-project"; - - // Setup: project and pools - ops::join3( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool(&client, "mcast-pool"), - ) - .await; - - // Group names for implicit groups (implicitly created when first member joins) - let group_names = - ["consistency-group-1", "consistency-group-2", "consistency-group-3"]; - - // Create instances first (groups will be implicitly created when members attach) - let instance_names: Vec<_> = group_names - .iter() - .map(|&group_name| format!("instance-{group_name}")) - .collect(); - - // Create all instances in parallel - let create_futures = instance_names.iter().map(|instance_name| { - create_instance(client, project_name, instance_name) - }); - ops::join_all(create_futures).await; - - // Stop DPD before attaching members to test state consistency during failure - // Groups will be implicitly created but stay in "Creating" state - cptestctx.stop_dendrite(SwitchLocation::Switch0).await; - - // Attach instances to their respective groups (triggers implicit creation for each group) - // Since DPD is down, groups will remain in "Creating" state - for (instance_name, &group_name) in instance_names.iter().zip(&group_names) - { - multicast_group_attach( - cptestctx, - project_name, - instance_name, - group_name, - ) - .await; - } - - // Wait for reconciler to attempt processing (will fail due to DPD being down) - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify each group is in a consistent state (DPD failure prevents reconciliation) - for group_name in group_names.iter() { - let group_get_url = mcast_group_url(group_name); - let fetched_group: MulticastGroup = - object_get(client, &group_get_url).await; - - // State should be "Creating" since DPD is down - assert_eq!( - fetched_group.state, "Creating", - "Group {group_name} should remain in Creating state when DPD is unavailable, found: {}", - fetched_group.state - ); - - // Case: Verify member state during DPD failure - // Instance is running, so member has sled_id, - // but DPD is unavailable so it can't be programmed. - let members = list_multicast_group_members(client, group_name).await; - assert_eq!( - members.len(), - 1, - "Group {group_name} should have exactly one member" - ); - assert_eq!( - members[0].state, "Joining", - "Member in group {group_name} should be Joining when DPD unavailable" - ); - } - - // Verify groups are still stuck in expected states before restarting DPD - // This explicitly validates that without DPD, groups cannot transition - for group_name in group_names.iter() { - verify_group_deleted_or_in_states(client, group_name, &["Creating"]) - .await; - } - - // Restart DPD before cleanup so instances can stop properly - cptestctx.restart_dendrite(SwitchLocation::Switch0).await; - - let instance_name_refs: Vec<&str> = - instance_names.iter().map(|s| s.as_str()).collect(); - cleanup_instances(cptestctx, client, project_name, &instance_name_refs) - .await; - - // Verify groups are deleted (implicit deletion completes with DPD available) - for group_name in group_names.iter() { - wait_for_group_deleted(cptestctx, group_name).await; - } -} - -#[nexus_test] -async fn test_dpd_failure_during_creating_state( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "test-project"; - let group_name = "creating-dpd-fail-group"; - let instance_name = "creating-fail-instance"; - - // Setup: project, pools - ops::join3( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool(&client, "mcast-pool"), - ) - .await; - - // Create instance first - create_instance(client, project_name, instance_name).await; - - // Stop DPD before implicit creation - cptestctx.stop_dendrite(SwitchLocation::Switch0).await; - - // Add instance to multicast group via instance-centric API - multicast_group_attach(cptestctx, project_name, instance_name, group_name) - .await; - - // Wait for reconciler to process - tests DPD communication handling during "Creating" state - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Check group state after reconciler processes with DPD unavailable - let group_get_url = mcast_group_url(group_name); - let fetched_group: MulticastGroup = - object_get(client, &group_get_url).await; - - // Group should remain in "Creating" state since DPD is down - assert_eq!( - fetched_group.state, "Creating", - "Group should remain in Creating state when DPD is unavailable during activation, found: {}", - fetched_group.state - ); - - // Verify group properties are maintained - assert_eq!(fetched_group.identity.name.as_str(), group_name); - - // Case: Verify member state during DPD failure - // Instance is running, so member has sled_id, - // but DPD is unavailable so it can't be programmed. - let members = list_multicast_group_members(client, group_name).await; - assert_eq!(members.len(), 1, "Should have exactly one member"); - assert_eq!( - members[0].state, "Joining", - "Member should be Joining when DPD unavailable (waiting to be programmed)" - ); - - // Test cleanup - remove member, which triggers implicit deletion - multicast_group_detach(client, project_name, instance_name, group_name) - .await; - - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; -} - +/// Test DPD failure during group "Active" state. +/// +/// When DPD becomes unavailable after group is already Active: +/// - Group remains in Active state +/// - Member remains in Joined state +/// - Existing state is preserved despite DPD communication failure #[nexus_test] async fn test_dpd_failure_during_active_state( cptestctx: &ControlPlaneTestContext, @@ -321,7 +142,7 @@ async fn test_dpd_failure_during_active_state( let group_name = "active-dpd-fail-group"; let instance_name = "active-fail-instance"; - // Create project and pools in parallel + // Setup: project, pools ops::join3( create_project(&client, project_name), create_default_ip_pools(&client), @@ -331,11 +152,11 @@ async fn test_dpd_failure_during_active_state( let instance = create_instance(client, project_name, instance_name).await; - // Add instance to multicast group via instance-centric API + // Add instance to multicast group multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Wait for group to become "Active" and member to reach "Joined" state + // Wait for group to become Active and member to reach Joined wait_for_group_active(client, group_name).await; wait_for_member_state( cptestctx, @@ -345,49 +166,49 @@ async fn test_dpd_failure_during_active_state( ) .await; - // Verify group is now "Active" + // Verify group is Active let group_get_url = mcast_group_url(group_name); let active_group: MulticastGroup = object_get(client, &group_get_url).await; assert_eq!(active_group.state, "Active"); - // Now stop DPD while group is "Active" and member is "Joined" + // Now stop DPD while group is Active cptestctx.stop_dendrite(SwitchLocation::Switch0).await; - // Wait for reconciler to process - tests DPD communication handling during "Active" state + // Wait for reconciler to process wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Check group state after reconciler processes with DPD unavailable + // Group should remain Active despite DPD failure let fetched_group: MulticastGroup = object_get(client, &group_get_url).await; - - // Group should remain "Active" - existing "Active" groups shouldn't change state due to DPD failures assert_eq!( fetched_group.state, "Active", - "Active group should remain Active despite DPD communication failure, found: {}", + "Active group should remain Active despite DPD failure, found: {}", fetched_group.state ); - - // Verify group properties are maintained assert_eq!(fetched_group.identity.name.as_str(), group_name); - // Case: Verify member state persists during DPD failure for Active groups - // Members that were already "Joined" should remain "Joined" even when DPD is unavailable + // Member should remain Joined let members = list_multicast_group_members(client, group_name).await; assert_eq!(members.len(), 1, "Should have exactly one member"); assert_eq!( members[0].state, "Joined", - "Member should remain Joined when DPD fails after group reached Active state, got: {}", + "Member should remain Joined when DPD fails, got: {}", members[0].state ); - // Test cleanup - remove member, which triggers implicit deletion + // Cleanup multicast_group_detach(client, project_name, instance_name, group_name) .await; - - // Wait for reconciler to process the deletion wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; + cptestctx.restart_dendrite(SwitchLocation::Switch0).await; + cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; } +/// Test DPD failure during group "Deleting" state. +/// +/// When DPD is unavailable during implicit group deletion: +/// - Group stays in Deleting state (cannot complete cleanup) +/// - After DPD recovery, deletion completes #[nexus_test] async fn test_dpd_failure_during_deleting_state( cptestctx: &ControlPlaneTestContext, @@ -397,7 +218,7 @@ async fn test_dpd_failure_during_deleting_state( let group_name = "deleting-dpd-fail-group"; let instance_name = "deleting-fail-instance"; - // Create project and pools in parallel + // Setup: project, pools ops::join3( create_project(&client, project_name), create_default_ip_pools(&client), @@ -405,28 +226,22 @@ async fn test_dpd_failure_during_deleting_state( ) .await; - // Create instance first + // Create instance and add to group create_instance(client, project_name, instance_name).await; - - // Add instance to multicast group via instance-centric API multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - // Wait for group to reach "Active" state before testing deletion + // Wait for group to reach Active state wait_for_group_active(client, group_name).await; - // Stop DPD before triggering deletion (by removing member) + // Stop DPD before triggering deletion cptestctx.stop_dendrite(SwitchLocation::Switch0).await; - // Remove the member to trigger implicit deletion (group should go to "Deleting" state) + // Remove member to trigger implicit deletion multicast_group_detach(client, project_name, instance_name, group_name) .await; - // The group should now be in "Deleting" state and DPD is down - // Let's check the state before reconciler runs - // Group should be accessible via GET request - - // Try to get group - should be accessible in "Deleting" state + // Check group is in Deleting state let get_result = objects_list_page_authz::( client, "/v1/multicast-groups", @@ -443,21 +258,20 @@ async fn test_dpd_failure_during_deleting_state( let group = &remaining_groups[0]; assert_eq!( group.state, "Deleting", - "Group should be in Deleting state after deletion request, found: {}", + "Group should be in Deleting state, found: {}", group.state ); } - // Wait for reconciler to attempt deletion with DPD down + // Wait for reconciler to attempt deletion wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Check final state - group should remain in "Deleting" state since DPD is unavailable - // The reconciler cannot complete deletion without DPD communication - let final_result = - nexus_test_utils::resource_helpers::objects_list_page_authz::< - MulticastGroup, - >(client, "/v1/multicast-groups") - .await; + // Group should remain in Deleting since DPD is unavailable + let final_result = objects_list_page_authz::( + client, + "/v1/multicast-groups", + ) + .await; let final_groups: Vec<_> = final_result .items @@ -469,13 +283,16 @@ async fn test_dpd_failure_during_deleting_state( let group = &final_groups[0]; assert_eq!( group.state, "Deleting", - "Group should remain in Deleting state when DPD is unavailable during deletion, found: {}", + "Group should remain in Deleting when DPD unavailable, found: {}", group.state ); - - // Verify group properties are maintained during failed deletion assert_eq!(group.identity.name.as_str(), group_name); } + + // Restart DPD and cleanup + cptestctx.restart_dendrite(SwitchLocation::Switch0).await; + cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; + wait_for_group_deleted(cptestctx, group_name).await; } #[nexus_test] @@ -542,9 +359,9 @@ async fn test_multicast_group_members_during_dpd_failure( created_group.identity.id ); - // Case: Verify member state during DPD failure - // Instance is running, so member has sled_id, - // but DPD is unavailable so it can't be programmed. + // Verify member state during DPD failure + // Instance is running, so member has sled_id, but DPD is unavailable so it + // can't be programmed against. assert_eq!( members_during_failure[0].state, "Joining", "Member should be Joining when DPD unavailable (waiting to be programmed)" @@ -567,80 +384,6 @@ async fn test_multicast_group_members_during_dpd_failure( wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; } -/// Test that implicit creation works correctly when DPD is unavailable. -/// -/// When a member is added to a non-existent group (by name), the system should: -/// 1. Implicitly create the group in "Creating" state -/// 2. Create the member in "Left" state (since instance is stopped) -/// 3. The group should remain in "Creating" state until DPD is available -/// -/// This tests the implicit group lifecycle: groups are implicitly created -/// when the first instance joins. -#[nexus_test] -async fn test_implicit_creation_with_dpd_failure( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "implicit-create-dpd-fail-project"; - let group_name = "implicit-created-dpd-fail-group"; - let instance_name = "implicit-create-instance"; - - // Create project and pools in parallel - ops::join3( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool(&client, "implicit-create-pool"), - ) - .await; - - // Create instance first - let instance = create_instance(client, project_name, instance_name).await; - - // Stop DPD before implicit creation - cptestctx.stop_dendrite(SwitchLocation::Switch0).await; - - // Add the instance to a non-existent group (triggers implicit creation) - multicast_group_attach(cptestctx, project_name, instance_name, group_name) - .await; - - // Verify the implicitly created group exists and is in "Creating" state - let group_url = mcast_group_url(group_name); - let implicitly_created_group: MulticastGroup = - object_get(client, &group_url).await; - - assert_eq!( - implicitly_created_group.state, "Creating", - "Implicitly created group should start in Creating state" - ); - assert_eq!(implicitly_created_group.identity.name.as_str(), group_name); - - // Wait for reconciler - group should remain in "Creating" since DPD is down - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify group is still in "Creating" state - let fetched_group: MulticastGroup = object_get(client, &group_url).await; - assert_eq!( - fetched_group.state, "Creating", - "Implicitly created group should remain in Creating state when DPD is unavailable" - ); - - // Verify member is still attached - let members = list_multicast_group_members(client, group_name).await; - assert_eq!(members.len(), 1, "Should have one member"); - assert_eq!(members[0].instance_id, instance.identity.id); - - // Case: Verify member state during DPD failure for implicit creation - // Instance is running, so member has sled_id, - // but DPD is unavailable so it can't be programmed. - assert_eq!( - members[0].state, "Joining", - "Member should be Joining when DPD unavailable (waiting to be programmed)" - ); - - multicast_group_detach(client, project_name, instance_name, group_name) - .await; -} - /// Test that implicit deletion works correctly when DPD is unavailable. /// /// When the last member leaves an implicit group, the system should: @@ -967,8 +710,11 @@ async fn test_multicast_join_deleted_instance( .await; // Create two instances - create_instance(client, project_name, instance_to_delete).await; - create_instance(client, project_name, remaining_instance).await; + ops::join2( + create_instance(client, project_name, instance_to_delete), + create_instance(client, project_name, remaining_instance), + ) + .await; // Create group with the remaining instance (so the group stays alive) multicast_group_attach( @@ -1142,7 +888,6 @@ async fn test_member_joining_to_left_on_instance_stop( ); // Stop the instance while member is in "Joining" state - let nexus = &cptestctx.server.server_context().nexus; let stop_url = format!("/v1/instances/{instance_name}/stop?project={project_name}"); NexusRequest::new( @@ -1154,8 +899,18 @@ async fn test_member_joining_to_left_on_instance_stop( .execute() .await .unwrap(); - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(&client, instance_id, InstanceState::Stopped).await; + // For stopping, simulate once then wait (don't repeatedly simulate as VMM gets removed) + instance_helpers::instance_simulate( + &cptestctx.server.server_context().nexus, + &instance_id, + ) + .await; + instance_helpers::instance_wait_for_state( + client, + instance_id, + InstanceState::Stopped, + ) + .await; // Run reconciler - should detect invalid instance and transition to "Left" wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; @@ -1214,7 +969,6 @@ async fn test_left_member_waits_for_group_active( ) .await; let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - let nexus = &cptestctx.server.server_context().nexus; // Stop DPD to keep group in "Creating" state cptestctx.stop_dendrite(SwitchLocation::Switch0).await; @@ -1258,8 +1012,7 @@ async fn test_left_member_waits_for_group_active( .execute() .await .unwrap(); - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(&client, instance_id, InstanceState::Running).await; + instance_wait_for_running_with_simulation(cptestctx, instance_id).await; // Run reconciler - member should stay in Left because group is not Active wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; diff --git a/nexus/tests/integration_tests/multicast/groups.rs b/nexus/tests/integration_tests/multicast/groups.rs index 183197d95f5..738aee6d5b4 100644 --- a/nexus/tests/integration_tests/multicast/groups.rs +++ b/nexus/tests/integration_tests/multicast/groups.rs @@ -602,59 +602,6 @@ async fn test_multicast_group_member_errors( wait_for_group_deleted(cptestctx, group_name).await; } -#[nexus_test] -async fn test_lookup_multicast_group_by_ip( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "test-project"; - let group_name = "test-lookup-group"; - - // Create project and IP pools in parallel - ops::join3( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool_with_range( - &client, - "mcast-pool", - (224, 7, 0, 10), - (224, 7, 0, 255), - ), - ) - .await; - - // Implicitly create multicast group by adding an instance as first member - let instance_name = "lookup-test-instance"; - create_instance(client, project_name, instance_name).await; - - // Use instance-centric API to join group (implicit group creation) - multicast_group_attach(cptestctx, project_name, instance_name, group_name) - .await; - - // Wait for group to become active - wait_for_group_active(&client, group_name).await; - - // Get the group to find its auto-allocated IP address - let created_group: MulticastGroup = - object_get(client, &mcast_group_url(group_name)).await; - let multicast_ip = created_group.multicast_ip; - - // Test lookup by IP (using the auto-allocated IP) via the main endpoint - // The main multicast-groups endpoint now accepts Name, ID, or IP - let lookup_url = format!("/v1/multicast-groups/{multicast_ip}"); - let found_group: MulticastGroup = object_get(client, &lookup_url).await; - assert_groups_eq(&created_group, &found_group); - - // Test lookup with nonexistent IP - let nonexistent_ip = IpAddr::V4(Ipv4Addr::new(224, 0, 1, 200)); - let lookup_bad_url = format!("/v1/multicast-groups/{nonexistent_ip}"); - - object_get_error(client, &lookup_bad_url, StatusCode::NOT_FOUND).await; - - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; - wait_for_group_deleted(cptestctx, group_name).await; -} - #[nexus_test] async fn test_instance_deletion_removes_multicast_memberships( cptestctx: &ControlPlaneTestContext, @@ -944,62 +891,62 @@ async fn test_cannot_delete_multicast_pool_with_groups( .expect("Should be able to delete pool after ranges are deleted"); } -/// Assert that two multicast groups are equal in all fields. -fn assert_groups_eq(left: &MulticastGroup, right: &MulticastGroup) { - assert_eq!(left.identity.id, right.identity.id); - assert_eq!(left.identity.name, right.identity.name); - assert_eq!(left.identity.description, right.identity.description); - assert_eq!(left.multicast_ip, right.multicast_ip); - assert_eq!(left.source_ips, right.source_ips); - assert_eq!(left.mvlan, right.mvlan); - assert_eq!(left.ip_pool_id, right.ip_pool_id); -} - -/// Test that source IPs are validated when joining a multicast group. +/// Consolidated test for SSM (Source-Specific Multicast) source IP behavior. /// -/// Source IPs enable Source-Specific Multicast (SSM) where traffic is filtered -/// by both destination (multicast IP) and source addresses. +/// This test covers: +/// - Source IP union on group view (multiple members with different sources) +/// - IPv4/IPv6 source address family validation +/// - Multiple SSM groups from same pool with different sources /// -/// Also verifies that the group view's `source_ips` field contains the -/// union of all member source IPs. +/// SSM groups (232.0.0.0/8 for IPv4, ff3x::/32 for IPv6) require source IPs +/// to be specified on join. The group's `source_ips` field shows the union +/// of all member sources. #[nexus_test] -async fn test_source_ip_validation_on_join( - cptestctx: &ControlPlaneTestContext, -) { +async fn test_ssm_source_ip_behavior(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - let project_name = "source-ip-validation-project"; - let instance_name = "source-ip-test-instance"; - let instance2_name = "source-ip-test-instance-2"; - let instance3_name = "source-ip-test-instance-3"; - - // SSM IP address (232.x.x.x range) - let ssm_ip = "232.1.0.100"; + let project_name = "ssm-source-ip-test"; // Create project and IP pools in parallel - ops::join3( + // SSM pool uses 232.x.x.x range with enough IPs for multiple groups + let (_, _, ssm_pool) = ops::join3( create_project(&client, project_name), create_default_ip_pools(&client), create_multicast_ip_pool_with_range( &client, - "source-ip-mcast-pool", + "ssm-test-pool", (232, 1, 0, 1), - (232, 1, 0, 255), + (232, 1, 0, 100), ), ) .await; - // Create instances - create_instance(client, project_name, instance_name).await; - create_instance(client, project_name, instance2_name).await; - create_instance(client, project_name, instance3_name).await; + // Also create IPv6 pool for address family validation tests + create_multicast_ip_pool_v6(&client, "ssm-test-pool-v6").await; + + // Create instances for all test cases + let instance_names = [ + "ssm-inst-1", + "ssm-inst-2", + "ssm-inst-3", + "ssm-inst-4", + "ssm-inst-5", + "ssm-inst-6", + ]; + for name in &instance_names { + create_instance(client, project_name, name).await; + } + // Case: Source IP union on group view + // Multiple members join with different sources, group view shows union + let ssm_union_ip = "232.1.0.10"; let source1 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); let source2 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)); let source3 = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 3)); // First instance joins with source1 - creates SSM group let join_url1 = format!( - "/v1/instances/{instance_name}/multicast-groups/{ssm_ip}?project={project_name}" + "/v1/instances/{}/multicast-groups/{ssm_union_ip}?project={project_name}", + instance_names[0] ); let join_body1 = InstanceMulticastGroupJoin { source_ips: Some(vec![source1]), @@ -1010,12 +957,14 @@ async fn test_source_ip_validation_on_join( // Verify group source_ips shows source1 let group: MulticastGroup = - object_get(client, &format!("/v1/multicast-groups/{ssm_ip}")).await; + object_get(client, &format!("/v1/multicast-groups/{ssm_union_ip}")) + .await; assert_eq!(group.source_ips, vec![source1], "Group should show source1"); // Second instance joins with source1 and source2 let join_url2 = format!( - "/v1/instances/{instance2_name}/multicast-groups/{ssm_ip}?project={project_name}" + "/v1/instances/{}/multicast-groups/{ssm_union_ip}?project={project_name}", + instance_names[1] ); let join_body2 = InstanceMulticastGroupJoin { source_ips: Some(vec![source1, source2]), @@ -1026,7 +975,8 @@ async fn test_source_ip_validation_on_join( // Verify group source_ips is union of member sources (sorted for comparison) let group: MulticastGroup = - object_get(client, &format!("/v1/multicast-groups/{ssm_ip}")).await; + object_get(client, &format!("/v1/multicast-groups/{ssm_union_ip}")) + .await; let mut actual_sources = group.source_ips.clone(); actual_sources.sort(); let mut expected_sources = vec![source1, source2]; @@ -1038,7 +988,8 @@ async fn test_source_ip_validation_on_join( // Third instance joins with source3 - union should now include all three let join_url3 = format!( - "/v1/instances/{instance3_name}/multicast-groups/{ssm_ip}?project={project_name}" + "/v1/instances/{}/multicast-groups/{ssm_union_ip}?project={project_name}", + instance_names[2] ); let join_body3 = InstanceMulticastGroupJoin { source_ips: Some(vec![source3]), @@ -1049,13 +1000,14 @@ async fn test_source_ip_validation_on_join( // Verify group source_ips is union of all three members' sources let group: MulticastGroup = - object_get(client, &format!("/v1/multicast-groups/{ssm_ip}")).await; + object_get(client, &format!("/v1/multicast-groups/{ssm_union_ip}")) + .await; let mut actual_sources = group.source_ips.clone(); actual_sources.sort(); - let mut expected_sources = vec![source1, source2, source3]; - expected_sources.sort(); + let mut expected_union_sources = vec![source1, source2, source3]; + expected_union_sources.sort(); assert_eq!( - actual_sources, expected_sources, + actual_sources, expected_union_sources, "Group source_ips should be union of all member sources" ); @@ -1073,12 +1025,12 @@ async fn test_source_ip_validation_on_join( let listed_group = groups .all_items .iter() - .find(|g| g.multicast_ip.to_string() == ssm_ip) + .find(|g| g.multicast_ip.to_string() == ssm_union_ip) .expect("SSM group should appear in list"); let mut listed_sources = listed_group.source_ips.clone(); listed_sources.sort(); assert_eq!( - listed_sources, expected_sources, + listed_sources, expected_union_sources, "List endpoint should also return source_ips as union" ); @@ -1086,7 +1038,7 @@ async fn test_source_ip_validation_on_join( wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; // Verify DPD external group contains the union of sources - let multicast_ip: IpAddr = ssm_ip.parse().unwrap(); + let multicast_ip: IpAddr = ssm_union_ip.parse().unwrap(); let dpd_response = dpd_client(cptestctx) .multicast_group_get(&multicast_ip) .await @@ -1112,67 +1064,25 @@ async fn test_source_ip_validation_on_join( dpd_source_ips.sort(); assert_eq!( - dpd_source_ips, expected_sources, + dpd_source_ips, expected_union_sources, "DPD external group sources should be union of all member sources" ); - // Cleanup - cleanup_instances( - cptestctx, - client, - project_name, - &[instance_name, instance2_name, instance3_name], - ) - .await; - wait_for_group_deleted(cptestctx, ssm_ip).await; -} - -/// Test that source IP address family must match multicast group address family. -/// -/// IPv4 multicast groups can only have IPv4 source IPs, and vice versa. -/// This test covers both directions: -/// - IPv6 sources with IPv4 group (fails) -/// - IPv4 sources with IPv6 group (fails) -#[nexus_test] -async fn test_source_ip_address_family_validation( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "addr-family-validation-project"; - let instance_name = "addr-family-test-instance"; - - // IPv4 SSM address - let ipv4_ssm_ip = "232.5.0.100"; - - // Create project and both IPv4 and IPv6 multicast pools - ops::join4( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool_with_range( - &client, - "addr-family-mcast-pool-v4", - (232, 5, 0, 1), - (232, 5, 0, 255), - ), - create_multicast_ip_pool_v6(&client, "addr-family-mcast-pool-v6"), - ) - .await; - - create_instance(client, project_name, instance_name).await; - - // Try to join IPv4 group with IPv6 source - should fail + // Case: IPv6 source with IPv4 group should fail + let ipv4_ssm_ip = "232.1.0.20"; let ipv6_source: IpAddr = "2001:db8::1".parse().unwrap(); - let join_url = format!( - "/v1/instances/{instance_name}/multicast-groups/{ipv4_ssm_ip}?project={project_name}" + let join_url_v4_with_v6 = format!( + "/v1/instances/{}/multicast-groups/{ipv4_ssm_ip}?project={project_name}", + instance_names[3] ); - let join_body = InstanceMulticastGroupJoin { + let join_body_v4_with_v6 = InstanceMulticastGroupJoin { source_ips: Some(vec![ipv6_source]), ip_version: None, }; let error = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url) - .body(Some(&join_body)) + RequestBuilder::new(client, Method::PUT, &join_url_v4_with_v6) + .body(Some(&join_body_v4_with_v6)) .expect_status(Some(StatusCode::BAD_REQUEST)), ) .authn_as(AuthnMode::PrivilegedUser) @@ -1189,19 +1099,20 @@ async fn test_source_ip_address_family_validation( error.error_code ); - // Try to join IPv6 group with IPv4 source - should fail + // Case: IPv4 source with IPv6 group should fail let ipv4_source: IpAddr = "10.0.0.1".parse().unwrap(); - let join_url_v6 = format!( - "/v1/instances/{instance_name}/multicast-groups/ipv6-mismatch-group?project={project_name}" + let join_url_v6_with_v4 = format!( + "/v1/instances/{}/multicast-groups/ipv6-mismatch-group?project={project_name}", + instance_names[3] ); - let join_body_v6 = InstanceMulticastGroupJoin { + let join_body_v6_with_v4 = InstanceMulticastGroupJoin { source_ips: Some(vec![ipv4_source]), ip_version: Some(IpVersion::V6), }; let error_v6 = NexusRequest::new( - RequestBuilder::new(client, Method::PUT, &join_url_v6) - .body(Some(&join_body_v6)) + RequestBuilder::new(client, Method::PUT, &join_url_v6_with_v4) + .body(Some(&join_body_v6_with_v4)) .expect_status(Some(StatusCode::BAD_REQUEST)), ) .authn_as(AuthnMode::PrivilegedUser) @@ -1218,7 +1129,102 @@ async fn test_source_ip_address_family_validation( error_v6.error_code ); - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; + // Case: Multiple SSM groups from same pool with different sources + // Each instance joins a different SSM group with unique sources + let group_configs = [ + ("ssm-multi-group-1", "10.1.1.1"), + ("ssm-multi-group-2", "10.2.2.2"), + ("ssm-multi-group-3", "10.3.3.3"), + ]; + + // Create 3 different SSM groups from the same pool + for (i, (group_name, source_ip)) in group_configs.iter().enumerate() { + let instance_name = instance_names[i + 3]; // Use instances 3, 4, 5 + let join_url = format!( + "/v1/instances/{instance_name}/multicast-groups/{group_name}?project={project_name}" + ); + let join_params = InstanceMulticastGroupJoin { + source_ips: Some(vec![source_ip.parse().unwrap()]), + ip_version: None, + }; + put_upsert::<_, MulticastGroupMember>(client, &join_url, &join_params) + .await; + + // Wait for group to become active + wait_for_group_active(client, group_name).await; + } + + // Verify all groups exist with correct properties + let mut allocated_ips = Vec::new(); + for (group_name, expected_source) in &group_configs { + let group: MulticastGroup = + object_get(client, &mcast_group_url(group_name)).await; + + // Verify pool reference + assert_eq!( + group.ip_pool_id, ssm_pool.identity.id, + "Group {group_name} should reference the shared SSM pool" + ); + + // Verify SSM range (232.x.x.x) + if let IpAddr::V4(ip) = group.multicast_ip { + assert_eq!( + ip.octets()[0], + 232, + "Group {group_name} should have IP in SSM range (232.x.x.x)" + ); + assert_eq!( + ip.octets()[1], + 1, + "Group {group_name} should have IP from pool range (232.1.x.x)" + ); + } else { + panic!("Expected IPv4 multicast address for group {group_name}"); + } + + // Verify source IPs + assert_eq!( + group.source_ips.len(), + 1, + "Group {group_name} should have exactly 1 source IP" + ); + assert_eq!( + group.source_ips[0].to_string(), + *expected_source, + "Group {group_name} should have correct source IP" + ); + + // Collect allocated IP for uniqueness check + allocated_ips.push(group.multicast_ip); + } + + // Verify all allocated IPs are unique + let unique_ips: std::collections::HashSet<_> = + allocated_ips.iter().collect(); + assert_eq!( + unique_ips.len(), + allocated_ips.len(), + "All SSM groups should have unique multicast IPs from the pool" + ); + + // Verify we can list all members for each group + for (group_name, _) in &group_configs { + let members = list_multicast_group_members(&client, group_name).await; + assert_eq!( + members.len(), + 1, + "Group {group_name} should have exactly 1 member" + ); + } + + // Cleanup all instances and wait for groups to be deleted + cleanup_instances(cptestctx, client, project_name, &instance_names).await; + + // Wait for all groups to be deleted + wait_for_group_deleted(cptestctx, ssm_union_ip).await; + for (group_name, _) in &group_configs { + wait_for_group_deleted(cptestctx, group_name).await; + } } /// Test default pool behavior when no pool is specified on member join. @@ -1369,171 +1375,6 @@ async fn test_pool_range_allocation(cptestctx: &ControlPlaneTestContext) { .await; } -/// Test multiple instances joining different SSM groups from the same SSM pool. -/// -/// Verifies: -/// - Pool allocates unique multicast IPs to each SSM group -/// - Different SSM groups can coexist with different source IP requirements -/// - Proper isolation between SSM groups on the same pool -#[nexus_test] -async fn test_multiple_ssm_groups_same_pool( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "multiple-ssm-test"; - - // Create project and IP pools in parallel - let (_, _, ssm_pool) = ops::join3( - create_project(&client, project_name), - create_default_ip_pools(&client), - create_multicast_ip_pool_with_range( - &client, - "ssm-shared-pool", - (232, 50, 0, 10), - (232, 50, 0, 50), - ), - ) - .await; - - // Create 3 instances - let instance_names = ["ssm-inst-1", "ssm-inst-2", "ssm-inst-3"]; - for name in &instance_names { - create_instance(client, project_name, name).await; - } - - // Each instance joins a different SSM group with different sources - let group_configs = [ - ("ssm-group-1", "10.1.1.1"), - ("ssm-group-2", "10.2.2.2"), - ("ssm-group-3", "10.3.3.3"), - ]; - - // Create all 3 SSM groups from the same pool - for (i, (group_name, source_ip)) in group_configs.iter().enumerate() { - let instance_name = instance_names[i]; - let join_url = format!( - "/v1/instances/{instance_name}/multicast-groups/{group_name}?project={project_name}" - ); - let join_params = InstanceMulticastGroupJoin { - source_ips: Some(vec![source_ip.parse().unwrap()]), - ip_version: None, - }; - put_upsert::<_, MulticastGroupMember>(client, &join_url, &join_params) - .await; - - // Wait for group to become active - wait_for_group_active(client, group_name).await; - } - - // Verify all groups exist with correct properties - let mut allocated_ips = Vec::new(); - for (group_name, expected_source) in &group_configs { - let group: MulticastGroup = - object_get(client, &mcast_group_url(group_name)).await; - - // Verify pool reference - assert_eq!( - group.ip_pool_id, ssm_pool.identity.id, - "Group {} should reference the shared SSM pool", - group_name - ); - - // Verify SSM range (232.x.x.x) - if let IpAddr::V4(ip) = group.multicast_ip { - assert_eq!( - ip.octets()[0], - 232, - "Group {} should have IP in SSM range (232.x.x.x)", - group_name - ); - assert_eq!( - ip.octets()[1], - 50, - "Group {} should have IP from pool range (232.50.x.x)", - group_name - ); - } else { - panic!("Expected IPv4 multicast address for group {}", group_name); - } - - // Verify source IPs - assert_eq!( - group.source_ips.len(), - 1, - "Group {} should have exactly 1 source IP", - group_name - ); - assert_eq!( - group.source_ips[0].to_string(), - *expected_source, - "Group {} should have correct source IP", - group_name - ); - - // Collect allocated IP for uniqueness check - allocated_ips.push(group.multicast_ip); - } - - // Verify all allocated IPs are unique - let unique_ips: std::collections::HashSet<_> = - allocated_ips.iter().collect(); - assert_eq!( - unique_ips.len(), - allocated_ips.len(), - "All SSM groups should have unique multicast IPs from the pool" - ); - - // Verify we can list all members for each group - for (group_name, _) in &group_configs { - let members = list_multicast_group_members(&client, group_name).await; - assert_eq!( - members.len(), - 1, - "Group {} should have exactly 1 member", - group_name - ); - } - - // Test that instances can join groups with different source IPs (sources are per-member) - let instance4_name = "ssm-inst-4"; - create_instance(client, project_name, instance4_name).await; - - let different_source: IpAddr = "10.99.99.99".parse().unwrap(); - let join_url_diff_source = format!( - "/v1/instances/{instance4_name}/multicast-groups/ssm-group-1?project={project_name}" - ); - let join_params_diff_source = InstanceMulticastGroupJoin { - source_ips: Some(vec![different_source]), - ip_version: None, - }; - put_upsert::<_, MulticastGroupMember>( - client, - &join_url_diff_source, - &join_params_diff_source, - ) - .await; - - // Verify group source_ips now includes both the original and new source - let group: MulticastGroup = - object_get(client, "/v1/multicast-groups/ssm-group-1").await; - assert!( - group.source_ips.contains(&different_source), - "Group source_ips should include new member's source" - ); - - let all_instances: Vec<_> = instance_names - .iter() - .chain(std::iter::once(&instance4_name)) - .map(|s| *s) - .collect(); - cleanup_instances(cptestctx, client, project_name, &all_instances).await; - - // Verify all groups are deleted - for (group_name, _) in &group_configs { - wait_for_group_deleted(cptestctx, group_name).await; - } -} - /// Test multicast pool selection with custom ASM range. /// /// Verifies that joining a multicast group by name correctly allocates diff --git a/nexus/tests/integration_tests/multicast/instances.rs b/nexus/tests/integration_tests/multicast/instances.rs index ff3650ac967..75be25d09b8 100644 --- a/nexus/tests/integration_tests/multicast/instances.rs +++ b/nexus/tests/integration_tests/multicast/instances.rs @@ -9,7 +9,7 @@ //! Instance lifecycle tests: //! //! - Full lifecycle: Create, attach, start, stop, delete flows -//! - Attach conflicts: Cannot attach same instance twice to same group +//! - Idempotency: Duplicate attach operations succeed without creating duplicates //! - Attach limits: Validates per-instance multicast group limits //! - State transitions: Member states change with instance state //! - Persistence: Memberships survive instance stop/start cycles @@ -42,7 +42,7 @@ use nexus_types::internal_api::params::InstanceMigrateRequest; use omicron_common::api::external::{ ByteCount, IdentityMetadataCreateParams, Instance, InstanceCpuCount, - InstanceState, + InstanceState, Nullable, }; use omicron_nexus::TestInterfaces; use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; @@ -468,311 +468,6 @@ async fn test_multicast_group_attach_limits( .await; } -#[nexus_test] -async fn test_multicast_group_instance_state_transitions( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - - // Create project and pools in parallel - ops::join3( - create_default_ip_pools(&client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(&client, "mcast-pool"), - ) - .await; - - // Create stopped instance (no multicast groups at creation) - let stopped_instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - "state-test-instance", - false, - &[], - ) - .await; - - // Add instance to group (group implicitly creates if it doesn't exist) - multicast_group_attach( - cptestctx, - PROJECT_NAME, - "state-test-instance", - "state-test-group", - ) - .await; - - // Wait for group to become Active before proceeding - wait_for_group_active(client, "state-test-group").await; - - // Verify instance is stopped and in multicast group - assert_eq!(stopped_instance.runtime.run_state, InstanceState::Stopped); - - // Wait for member to reach "Left" state (stopped instance members start in "Left" state) - wait_for_member_state( - cptestctx, - "state-test-group", - stopped_instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Left, - ) - .await; - - // Start the instance and verify multicast behavior - let instance_id = - InstanceUuid::from_untyped_uuid(stopped_instance.identity.id); - let nexus = &cptestctx.server.server_context().nexus; - - // Start the instance using direct POST request (not PUT) - let start_url = format!( - "/v1/instances/state-test-instance/start?project={PROJECT_NAME}" - ); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &start_url) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body::() - .unwrap(); - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(&client, instance_id, InstanceState::Running).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Stop the instance and verify multicast behavior persists - let stop_url = format!( - "/v1/instances/state-test-instance/stop?project={PROJECT_NAME}" - ); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &stop_url) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap() - .parsed_body::() - .unwrap(); - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(&client, instance_id, InstanceState::Stopped).await; - - // Verify control plane still shows membership regardless of instance state - let members_url = mcast_group_members_url("state-test-group"); - let final_members: Vec = - nexus_test_utils::http_testing::NexusRequest::iter_collection_authn( - client, - &members_url, - "", - None, - ) - .await - .unwrap() - .all_items; - - assert_eq!( - final_members.len(), - 1, - "Control plane should maintain multicast membership across instance state changes" - ); - assert_eq!(final_members[0].instance_id, stopped_instance.identity.id); - - object_delete( - client, - &format!("/v1/instances/state-test-instance?project={PROJECT_NAME}"), - ) - .await; - - wait_for_group_deleted(cptestctx, "state-test-group").await; -} - -/// Test that multicast group membership persists through instance stop/start cycles -/// (parallel to external IP persistence behavior) -#[nexus_test] -async fn test_multicast_group_persistence_through_stop_start( - cptestctx: &ControlPlaneTestContext, -) { - // Ensure inventory and DPD are ready before creating instances with multicast groups - ensure_multicast_test_ready(cptestctx).await; - - let client = &cptestctx.external_client; - - // Create project and pools in parallel - ops::join3( - create_default_ip_pools(&client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(&client, "mcast-pool"), - ) - .await; - - // Create instance and start it (no multicast groups at creation) - let instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - "persist-test-instance", - true, - &[], - ) - .await; - - // Add instance to group (group implicitly creates if it doesn't exist) - multicast_group_attach( - cptestctx, - PROJECT_NAME, - "persist-test-instance", - "persist-test-group", - ) - .await; - - // Wait for group to become Active - wait_for_group_active(client, "persist-test-group").await; - - let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - - // Simulate the instance transitioning to Running state - let nexus = &cptestctx.server.server_context().nexus; - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; - - // Wait for member to be joined (reconciler will process the sled_id set by instance start) - wait_for_member_state( - cptestctx, - "persist-test-group", - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Verify instance is in the group - let members_url = mcast_group_members_url("persist-test-group"); - let members_before_stop = - nexus_test_utils::http_testing::NexusRequest::iter_collection_authn::< - MulticastGroupMember, - >(client, &members_url, "", None) - .await - .expect("Should list group members before stop") - .all_items; - - assert_eq!( - members_before_stop.len(), - 1, - "Group should have 1 member before stop" - ); - assert_eq!(members_before_stop[0].instance_id, instance.identity.id); - - // Stop the instance - let instance_stop_url = format!( - "/v1/instances/persist-test-instance/stop?project={PROJECT_NAME}" - ); - nexus_test_utils::http_testing::NexusRequest::new( - nexus_test_utils::http_testing::RequestBuilder::new( - client, - http::Method::POST, - &instance_stop_url, - ) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(http::StatusCode::ACCEPTED)), - ) - .authn_as(nexus_test_utils::http_testing::AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Should stop instance"); - - // Simulate the stop transition - let nexus = &cptestctx.server.server_context().nexus; - instance_simulate(nexus, &instance_id).await; - - // Wait for instance to be stopped - instance_wait_for_state( - client, - instance_id, - omicron_common::api::external::InstanceState::Stopped, - ) - .await; - - // Verify multicast group membership persists while stopped - let members_while_stopped = - nexus_test_utils::http_testing::NexusRequest::iter_collection_authn::< - MulticastGroupMember, - >(client, &members_url, "", None) - .await - .expect("Should list group members while stopped") - .all_items; - - assert_eq!( - members_while_stopped.len(), - 1, - "Group membership should persist while instance is stopped" - ); - assert_eq!(members_while_stopped[0].instance_id, instance.identity.id); - - // Start the instance again - let instance_start_url = format!( - "/v1/instances/persist-test-instance/start?project={PROJECT_NAME}" - ); - nexus_test_utils::http_testing::NexusRequest::new( - nexus_test_utils::http_testing::RequestBuilder::new( - client, - http::Method::POST, - &instance_start_url, - ) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(http::StatusCode::ACCEPTED)), - ) - .authn_as(nexus_test_utils::http_testing::AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Should start instance"); - - // Simulate the instance transitioning back to "Running" state - let nexus = &cptestctx.server.server_context().nexus; - instance_simulate(nexus, &instance_id).await; - - // Wait for instance to be running again - instance_wait_for_state( - client, - instance_id, - omicron_common::api::external::InstanceState::Running, - ) - .await; - - // Verify multicast group membership still exists after restart - let members_after_restart = - nexus_test_utils::http_testing::NexusRequest::iter_collection_authn::< - MulticastGroupMember, - >(client, &members_url, "", None) - .await - .expect("Should list group members after restart") - .all_items; - - assert_eq!( - members_after_restart.len(), - 1, - "Group membership should persist after instance restart" - ); - assert_eq!(members_after_restart[0].instance_id, instance.identity.id); - - // Wait for member to be joined again after restart - wait_for_member_state( - cptestctx, - "persist-test-group", - instance.identity.id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - cleanup_instances( - cptestctx, - client, - PROJECT_NAME, - &["persist-test-instance"], - ) - .await; - // Group is implicitly deleted when last member (instance) is removed - wait_for_group_deleted(cptestctx, "persist-test-group").await; -} - /// Verify concurrent multicast operations maintain correct member states. /// /// The system handles multiple instances joining simultaneously, rapid attach/detach @@ -1059,28 +754,25 @@ async fn test_multicast_member_cleanup_instance_never_started( wait_for_group_deleted_from_dpd(cptestctx, underlay_multicast_ip).await; } -/// Verify multicast group membership persists through instance migration. +/// Test multicast group membership during instance migration. +/// +/// This test verifies two migration scenarios: +/// 1. Single instance migration: membership persists, DPD is updated, port mapping works +/// 2. Concurrent migrations: multiple instances migrate simultaneously without interference /// -/// The RPW reconciler detects sled_id changes and updates DPD configuration on +/// The RPW reconciler detects `sled_id` changes and updates DPD configuration on /// both source and target switches to maintain uninterrupted multicast traffic. -/// Member state follows the expected lifecycle: Joined on source `sled` → `sled_id` -/// updated during migration → "Joined" again on target sled after reconciler -/// processes the change. #[nexus_test(extra_sled_agents = 1)] -async fn test_multicast_group_membership_during_migration( +async fn test_multicast_migration_scenarios( cptestctx: &ControlPlaneTestContext, ) { - // Ensure inventory and DPD are ready before creating instances with multicast groups ensure_multicast_test_ready(cptestctx).await; let client = &cptestctx.external_client; let lockstep_client = &cptestctx.lockstep_client; let nexus = &cptestctx.server.server_context().nexus; - let project_name = "migration-test-project"; - let group_name = "migration-test-group"; - let instance_name = "migration-test-instance"; + let project_name = "migration-project"; - // Create project and pools in parallel ops::join3( create_project(client, project_name), create_default_ip_pools(client), @@ -1093,263 +785,137 @@ async fn test_multicast_group_membership_during_migration( ) .await; - // Create and start instance first (no multicast groups at creation) - let instance = instance_for_multicast_groups( + let available_sleds = + [cptestctx.first_sled_id(), cptestctx.second_sled_id()]; + + // Case: Single instance migration with DPD verification + + let group1_name = "single-migration-group"; + let instance1 = instance_for_multicast_groups( cptestctx, project_name, - instance_name, + "single-migration-inst", true, &[], ) .await; + let instance1_id = InstanceUuid::from_untyped_uuid(instance1.identity.id); - // Add instance to group (group implicitly creates if it doesn't exist) - multicast_group_attach(cptestctx, project_name, instance_name, group_name) - .await; - wait_for_group_active(client, group_name).await; - - // Get the group's multicast IP for DPD verification later - let created_group = get_multicast_group(client, group_name).await; - let multicast_ip = created_group.multicast_ip; + multicast_group_attach( + cptestctx, + project_name, + "single-migration-inst", + group1_name, + ) + .await; + wait_for_group_active(client, group1_name).await; - let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); + let group1 = get_multicast_group(client, group1_name).await; + let multicast_ip = group1.multicast_ip; - // Simulate instance startup and wait for Running state - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; - - // Wait for instance to reach "Joined" state (member creation is processed by reconciler) + instance_simulate(nexus, &instance1_id).await; + instance_wait_for_state(client, instance1_id, InstanceState::Running).await; wait_for_member_state( cptestctx, - group_name, - instance.identity.id, + group1_name, + instance1.identity.id, nexus_db_model::MulticastGroupMemberState::Joined, ) .await; - let pre_migration_members = - list_multicast_group_members(client, group_name).await; - assert_eq!(pre_migration_members.len(), 1); - assert_eq!(pre_migration_members[0].instance_id, instance.identity.id); - assert_eq!(pre_migration_members[0].state, "Joined"); - - // Verify group exists in DPD before migration + // Verify DPD before migration let dpd_client = nexus_test_utils::dpd_client(cptestctx); dpd_client .multicast_group_get(&multicast_ip) .await - .expect("Multicast group should exist in DPD before migration"); + .expect("Group should exist in DPD before migration"); - // Get source and target sleds for migration - let source_sled_id = nexus - .active_instance_info(&instance_id, None) + // Migrate instance + let source_sled = nexus + .active_instance_info(&instance1_id, None) .await .unwrap() .expect("Running instance should be on a sled") .sled_id; + let target_sled = + *available_sleds.iter().find(|&&s| s != source_sled).unwrap(); - let target_sled_id = if source_sled_id == cptestctx.first_sled_id() { - cptestctx.second_sled_id() - } else { - cptestctx.first_sled_id() - }; - - // Initiate migration - let migrate_url = format!("/instances/{instance_id}/migrate"); - nexus_test_utils::http_testing::NexusRequest::new( - nexus_test_utils::http_testing::RequestBuilder::new( - lockstep_client, - Method::POST, - &migrate_url, - ) - .body(Some(&InstanceMigrateRequest { dst_sled_id: target_sled_id })) - .expect_status(Some(StatusCode::OK)), + let migrate_url = format!("/instances/{instance1_id}/migrate"); + NexusRequest::new( + RequestBuilder::new(lockstep_client, Method::POST, &migrate_url) + .body(Some(&InstanceMigrateRequest { dst_sled_id: target_sled })) + .expect_status(Some(StatusCode::OK)), ) - .authn_as(nexus_test_utils::http_testing::AuthnMode::PrivilegedUser) + .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .expect("Should initiate instance migration"); + .expect("Should initiate migration"); - // Get propolis IDs for source and target - let info = nexus - .active_instance_info(&instance_id, None) - .await - .unwrap() - .expect("Instance should be on a sled"); - let src_propolis_id = info.propolis_id; - let dst_propolis_id = - info.dst_propolis_id.expect("Instance should have a migration target"); + let info = + nexus.active_instance_info(&instance1_id, None).await.unwrap().unwrap(); + let src_propolis = info.propolis_id; + let dst_propolis = info.dst_propolis_id.unwrap(); - // Complete migration on source sled and wait for instance to enter "Migrating" - vmm_simulate_on_sled(cptestctx, nexus, source_sled_id, src_propolis_id) + vmm_simulate_on_sled(cptestctx, nexus, source_sled, src_propolis).await; + instance_wait_for_state(client, instance1_id, InstanceState::Migrating) .await; - // Instance should transition to "Migrating"; membership should remain "Joined" - instance_wait_for_state(client, instance_id, InstanceState::Migrating) - .await; + // Verify membership persists during migration let migrating_members = - list_multicast_group_members(client, group_name).await; - assert_eq!( - migrating_members.len(), - 1, - "Membership should remain during migration" - ); - assert_eq!(migrating_members[0].instance_id, instance.identity.id); - assert_eq!( - migrating_members[0].state, "Joined", - "Member should stay Joined while migrating" - ); - - // Complete migration on target sled - vmm_simulate_on_sled(cptestctx, nexus, target_sled_id, dst_propolis_id) - .await; + list_multicast_group_members(client, group1_name).await; + assert_eq!(migrating_members.len(), 1); + assert_eq!(migrating_members[0].state, "Joined"); - // Wait for migration to complete - instance_wait_for_state(client, instance_id, InstanceState::Running).await; + vmm_simulate_on_sled(cptestctx, nexus, target_sled, dst_propolis).await; + instance_wait_for_state(client, instance1_id, InstanceState::Running).await; - // Verify instance is now on the target sled - let post_migration_sled = nexus - .active_instance_info(&instance_id, None) + // Verify post-migration state + let post_sled = nexus + .active_instance_info(&instance1_id, None) .await .unwrap() - .expect("Migrated instance should still be on a sled") + .unwrap() .sled_id; + assert_eq!(post_sled, target_sled, "Instance should be on target sled"); - assert_eq!( - post_migration_sled, target_sled_id, - "Instance should be on target sled after migration" - ); - - // Wait for multicast reconciler to process the sled_id change - // The RPW reconciler should detect the sled_id change and re-apply DPD configuration wait_for_multicast_reconciler(lockstep_client).await; - - // Verify multicast membership persists after migration - let post_migration_members = - list_multicast_group_members(client, group_name).await; - - assert_eq!( - post_migration_members.len(), - 1, - "Multicast membership should persist through migration" - ); - assert_eq!(post_migration_members[0].instance_id, instance.identity.id); - - // Wait for member to reach "Joined" state on target sled - // The RPW reconciler should transition the member back to "Joined" after re-applying DPD configuration wait_for_member_state( cptestctx, - group_name, - instance.identity.id, + group1_name, + instance1.identity.id, nexus_db_model::MulticastGroupMemberState::Joined, ) .await; - let final_member_state = &post_migration_members[0]; - assert_eq!( - final_member_state.state, "Joined", - "Member should be in 'Joined' state after migration completes" - ); - - // Verify inventory-based port mapping updated correctly after migration - // This confirms the RPW reconciler correctly mapped the new sled to its rear port - verify_inventory_based_port_mapping(cptestctx, &instance_id) + verify_inventory_based_port_mapping(cptestctx, &instance1_id) .await - .expect("Port mapping should be updated after migration"); - - // Verify group still exists in DPD after migration + .expect("Port mapping should be updated"); dpd_client .multicast_group_get(&multicast_ip) .await - .expect("Multicast group should exist in DPD after migration"); - - // Cleanup: Stop and delete instance - let stop_url = - format!("/v1/instances/{instance_name}/stop?project={project_name}"); - nexus_test_utils::http_testing::NexusRequest::new( - nexus_test_utils::http_testing::RequestBuilder::new( - client, - Method::POST, - &stop_url, - ) - .body(None as Option<&serde_json::Value>) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(nexus_test_utils::http_testing::AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Should stop instance"); - - // Simulate stop and wait for stopped state - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Stopped).await; - - // Delete instance; group is implicitly deleted when last member removed - object_delete( - client, - &format!("/v1/instances/{instance_name}?project={project_name}"), - ) - .await; - - // Implicit model: group is implicitly deleted when last member (instance) is removed - wait_for_group_deleted(cptestctx, group_name).await; -} + .expect("Group should exist in DPD after migration"); -/// Verify the RPW reconciler handles concurrent instance migrations within the same multicast group. -/// -/// Multiple instances in the same multicast group can migrate simultaneously without -/// interfering with each other's membership states. The reconciler correctly processes -/// concurrent sled_id changes for all members, ensuring each reaches Joined state on -/// their respective target sleds. -#[nexus_test(extra_sled_agents = 1)] -async fn test_multicast_group_concurrent_member_migrations( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let lockstep_client = &cptestctx.lockstep_client; - let nexus = &cptestctx.server.server_context().nexus; - let project_name = "concurrent-migration-project"; - let group_name = "concurrent-migration-group"; + // Case: Concurrent migrations - // Create project and pools in parallel - ops::join3( - create_project(client, project_name), - create_default_ip_pools(client), - create_multicast_ip_pool_with_range( - client, - "concurrent-migration-pool", - (224, 62, 0, 1), - (224, 62, 0, 255), - ), - ) - .await; - - // Ensure inventory and DPD are ready before creating instances - ensure_multicast_test_ready(cptestctx).await; - - // Create multiple instances - let instance_names = ["concurrent-instance-1", "concurrent-instance-2"]; - let create_futures = instance_names - .iter() - .map(|name| create_instance(client, project_name, name)); + let group2_name = "concurrent-migration-group"; + let instance_names = ["concurrent-inst-1", "concurrent-inst-2"]; + let create_futures = + instance_names.iter().map(|n| create_instance(client, project_name, n)); let instances = ops::join_all(create_futures).await; - // First instance attach (implicitly creates the group) multicast_group_attach( cptestctx, project_name, instance_names[0], - group_name, + group2_name, ) .await; - wait_for_group_active(client, group_name).await; - - // Second instance attach (group already exists) + wait_for_group_active(client, group2_name).await; multicast_group_attach( cptestctx, project_name, instance_names[1], - group_name, + group2_name, ) .await; @@ -1358,37 +924,25 @@ async fn test_multicast_group_concurrent_member_migrations( .map(|i| InstanceUuid::from_untyped_uuid(i.identity.id)) .collect(); - // Simulate all instances to Running state in parallel - let simulate_futures = instance_ids.iter().map(|&instance_id| async move { + // Start all instances via simulation + for &instance_id in &instance_ids { instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running) .await; - }); - ops::join_all(simulate_futures).await; - - // Wait for all members to reach "Joined" state - for instance in &instances { + } + for inst in &instances { wait_for_member_state( cptestctx, - group_name, - instance.identity.id, + group2_name, + inst.identity.id, nexus_db_model::MulticastGroupMemberState::Joined, ) .await; } - // Verify we have 2 members initially - let pre_migration_members = - list_multicast_group_members(client, group_name).await; - assert_eq!(pre_migration_members.len(), 2); - - // Get current sleds for all instances + // Get source/target sleds for each instance let mut source_sleds = Vec::new(); let mut target_sleds = Vec::new(); - - let available_sleds = - [cptestctx.first_sled_id(), cptestctx.second_sled_id()]; - for &instance_id in &instance_ids { let current_sled = nexus .active_instance_info(&instance_id, None) @@ -1397,121 +951,101 @@ async fn test_multicast_group_concurrent_member_migrations( .expect("Running instance should be on a sled") .sled_id; source_sleds.push(current_sled); - - // Find a different sled for migration target - let target_sled = available_sleds - .iter() - .find(|&&sled| sled != current_sled) - .copied() - .expect("Should have available target sled"); - target_sleds.push(target_sled); + target_sleds.push( + *available_sleds.iter().find(|&&s| s != current_sled).unwrap(), + ); } - // Initiate both migrations concurrently - let migration_futures = instance_ids.iter().zip(target_sleds.iter()).map( - |(&instance_id, &target_sled)| { - let migrate_url = format!("/instances/{instance_id}/migrate"); - nexus_test_utils::http_testing::NexusRequest::new( - nexus_test_utils::http_testing::RequestBuilder::new( - lockstep_client, - Method::POST, - &migrate_url, - ) - .body(Some(&InstanceMigrateRequest { - dst_sled_id: target_sled, - })) - .expect_status(Some(StatusCode::OK)), + // Initiate concurrent migrations + let migration_futures = + instance_ids.iter().zip(target_sleds.iter()).map(|(&id, &target)| { + let url = format!("/instances/{id}/migrate"); + NexusRequest::new( + RequestBuilder::new(lockstep_client, Method::POST, &url) + .body(Some(&InstanceMigrateRequest { dst_sled_id: target })) + .expect_status(Some(StatusCode::OK)), ) - .authn_as(nexus_test_utils::http_testing::AuthnMode::PrivilegedUser) + .authn_as(AuthnMode::PrivilegedUser) .execute() - }, - ); - - // Execute both migrations concurrently - let migration_responses = ops::join_all(migration_futures).await; - - // Verify both migrations were initiated successfully - for response in migration_responses { - response.expect("Migration should initiate successfully"); + }); + let responses = ops::join_all(migration_futures).await; + for r in responses { + r.expect("Migration should initiate"); } - // Complete both migrations by simulating on both source and target sleds + // Complete all migrations for (i, &instance_id) in instance_ids.iter().enumerate() { - // Get propolis IDs for this instance let info = nexus .active_instance_info(&instance_id, None) .await .unwrap() - .expect("Instance should be on a sled"); - let src_propolis_id = info.propolis_id; - let dst_propolis_id = info - .dst_propolis_id - .expect("Instance should have a migration target"); - - // Complete migration on source and target + .unwrap(); vmm_simulate_on_sled( cptestctx, nexus, source_sleds[i], - src_propolis_id, + info.propolis_id, ) .await; vmm_simulate_on_sled( cptestctx, nexus, target_sleds[i], - dst_propolis_id, + info.dst_propolis_id.unwrap(), ) .await; - instance_wait_for_state(client, instance_id, InstanceState::Running) .await; } - // Verify all instances are on their target sleds + // Verify all on target sleds for (i, &instance_id) in instance_ids.iter().enumerate() { - let current_sled = nexus + let sled = nexus .active_instance_info(&instance_id, None) .await .unwrap() - .expect("Migrated instance should be on target sled") + .unwrap() .sled_id; - assert_eq!( - current_sled, + sled, target_sleds[i], - "Instance {} should be on target sled after migration", + "Instance {} should be on target sled", i + 1 ); } - // Wait for multicast reconciler to process all sled_id changes wait_for_multicast_reconciler(lockstep_client).await; - // Verify all members are still in the group and reach "Joined" state - let post_migration_members = - list_multicast_group_members(client, group_name).await; - + let post_members = list_multicast_group_members(client, group2_name).await; assert_eq!( - post_migration_members.len(), + post_members.len(), 2, - "Both instances should remain multicast group members after concurrent migration" + "Both members should persist after concurrent migration" ); - // Verify both members reach "Joined" state on their new sleds - for instance in &instances { + for inst in &instances { wait_for_member_state( cptestctx, - group_name, - instance.identity.id, + group2_name, + inst.identity.id, nexus_db_model::MulticastGroupMemberState::Joined, ) .await; } - // Cleanup and delete instances (group is automatically deleted when last member removed) - cleanup_instances(cptestctx, client, project_name, &instance_names).await; - wait_for_group_deleted(cptestctx, group_name).await; + // Cleanup + cleanup_instances( + cptestctx, + client, + project_name, + &["single-migration-inst", instance_names[0], instance_names[1]], + ) + .await; + ops::join2( + wait_for_group_deleted(cptestctx, group1_name), + wait_for_group_deleted(cptestctx, group2_name), + ) + .await; } /// Test that source_ips are preserved across instance stop/start. @@ -1560,8 +1094,7 @@ async fn test_source_ips_preserved_on_instance_restart( // Simulate and wait for running let nexus = &cptestctx.server.server_context().nexus; - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; + instance_wait_for_running_with_simulation(cptestctx, instance_id).await; // Join SSM multicast group with source_ips let ssm_ip = "232.50.0.100"; @@ -1617,10 +1150,9 @@ async fn test_source_ips_preserved_on_instance_restart( .expect("Should restart instance"); // Simulate and wait for running - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; + instance_wait_for_running_with_simulation(cptestctx, instance_id).await; - // Verify source_ips are PRESERVED after restart + // Verify source_ips are preserved after restart // Get the member via the group members list let expected_group_name = format!("mcast-{}", ssm_ip.replace('.', "-")); let members_url = @@ -1943,21 +1475,20 @@ async fn test_instance_create_with_ssm_multicast_groups( wait_for_group_deleted(cptestctx, &ssm_group_name).await; } -/// Test that creating an instance with SSM multicast group (by IP) without -/// sources fails validation. +/// Test that SSM multicast groups without sources fail validation on both +/// instance create and reconfigure paths. /// /// SSM addresses (232/8 for IPv4) require source IPs to be specified. This -/// test verifies the validation happens during instance creation and prevents -/// creating the instance without proper SSM sources. +/// test verifies the validation happens during both: +/// a). Instance creation (POST /v1/instances) with SSM group without sources +/// b). Instance reconfigure (PUT /v1/instances) adding new SSM group without sources #[nexus_test] -async fn test_instance_create_with_ssm_without_sources_fails( +async fn test_ssm_without_sources_fails_create_and_reconfigure( cptestctx: &ControlPlaneTestContext, ) { - use nexus_types::external_api::params::MulticastGroupJoinSpec; - let client = &cptestctx.external_client; - let project_name = "ssm-nosrc-create-project"; - let instance_name = "ssm-nosrc-create-instance"; + let project_name = "ssm-nosrc-project"; + let instance_name = "ssm-nosrc-instance"; // Setup: create pools and project ops::join3( @@ -1965,18 +1496,17 @@ async fn test_instance_create_with_ssm_without_sources_fails( create_project(client, project_name), create_multicast_ip_pool_with_range( client, - "ssm-nosrc-create-pool", + "ssm-nosrc-pool", (232, 80, 0, 1), (232, 80, 0, 100), ), ) .await; - // Try to create instance with SSM multicast group WITHOUT source_ips - // This should fail validation let ssm_ip: IpAddr = "232.80.0.10".parse().unwrap(); - let instance_params = InstanceCreate { + // Case: Instance creation with SSM group without sources should fail + let instance_params_with_ssm = InstanceCreate { identity: IdentityMetadataCreateParams { name: instance_name.parse().unwrap(), description: "Instance should fail with SSM without sources".into(), @@ -1994,7 +1524,6 @@ async fn test_instance_create_with_ssm_without_sources_fails( auto_restart_policy: None, anti_affinity_groups: Vec::new(), cpu_platform: None, - // SSM group by IP with NO source_ips - should fail multicast_groups: vec![MulticastGroupJoinSpec { group: ssm_ip.to_string().parse().unwrap(), source_ips: None, // Missing sources for SSM! @@ -2003,9 +1532,9 @@ async fn test_instance_create_with_ssm_without_sources_fails( }; let instance_url = format!("/v1/instances?project={project_name}"); - let error = NexusRequest::new( + let create_error = NexusRequest::new( RequestBuilder::new(client, Method::POST, &instance_url) - .body(Some(&instance_params)) + .body(Some(&instance_params_with_ssm)) .expect_status(Some(StatusCode::BAD_REQUEST)), ) .authn_as(AuthnMode::PrivilegedUser) @@ -2013,47 +1542,21 @@ async fn test_instance_create_with_ssm_without_sources_fails( .await .expect("Creating instance with SSM without sources should fail"); - // Verify the error message mentions SSM/source requirements - let error_body: serde_json::Value = - serde_json::from_slice(&error.body).unwrap(); - let error_message = error_body["message"].as_str().unwrap_or(""); + let create_error_body: serde_json::Value = + serde_json::from_slice(&create_error.body).unwrap(); + let create_error_message = + create_error_body["message"].as_str().unwrap_or(""); assert!( - error_message.contains("SSM") || error_message.contains("source"), - "Error should mention SSM or source IPs: {error_message}" + create_error_message.contains("SSM") + || create_error_message.contains("source"), + "Create error should mention SSM or source IPs: {create_error_message}" ); -} - -/// Test that instance reconfigure adding a new SSM group without sources fails. -/// -/// When reconfiguring an instance to add new multicast groups: -/// - For existing memberships: `source_ips = None` means "preserve existing" -/// - For new memberships: `source_ips = None` means "no sources" which is -/// invalid for SSM -#[nexus_test] -async fn test_instance_reconfigure_add_new_ssm_without_sources_fails( - cptestctx: &ControlPlaneTestContext, -) { - use omicron_common::api::external::Nullable; - let client = &cptestctx.external_client; - let project_name = "ssm-nosrc-reconfig-project"; - let instance_name = "ssm-nosrc-reconfig-instance"; - - // Setup: create pools and project - ops::join3( - create_default_ip_pools(&client), - create_project(client, project_name), - create_multicast_ip_pool_with_range( - client, - "ssm-nosrc-reconfig-pool", - (232, 81, 0, 1), - (232, 81, 0, 100), - ), - ) - .await; - - // First: create an instance WITHOUT any multicast groups - let instance_params = InstanceCreate { + // Case: Instance reconfiguration while adding SSM group without sources + // should fail + // + // We first create instance without multicast groups + let instance_params_no_mcast = InstanceCreate { identity: IdentityMetadataCreateParams { name: instance_name.parse().unwrap(), description: "Instance for SSM reconfigure test".into(), @@ -2071,29 +1574,24 @@ async fn test_instance_reconfigure_add_new_ssm_without_sources_fails( auto_restart_policy: None, anti_affinity_groups: Vec::new(), cpu_platform: None, - multicast_groups: vec![], // No multicast groups initially + multicast_groups: vec![], // No multicast groups init }; - let instance_url = format!("/v1/instances?project={project_name}"); let instance: Instance = - object_create(client, &instance_url, &instance_params).await; + object_create(client, &instance_url, &instance_params_no_mcast).await; let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - // Simulate instance to running state let nexus = &cptestctx.server.server_context().nexus; instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running).await; - // Now try to reconfigure to add a new SSM group without sources - let ssm_ip: IpAddr = "232.81.0.10".parse().unwrap(); - + // Try to reconfigure to add SSM group without sources let update_params = InstanceUpdate { ncpus: InstanceCpuCount(2), memory: ByteCount::from_gibibytes_u32(4), boot_disk: Nullable(None), auto_restart_policy: Nullable(None), cpu_platform: Nullable(None), - // Try to add new SSM group without sources - should fail multicast_groups: Some(vec![MulticastGroupJoinSpec { group: ssm_ip.to_string().parse().unwrap(), source_ips: None, // Missing sources for new SSM group! @@ -2103,7 +1601,7 @@ async fn test_instance_reconfigure_add_new_ssm_without_sources_fails( let update_url = format!("/v1/instances/{instance_name}?project={project_name}"); - let error = NexusRequest::new( + let reconfig_error = NexusRequest::new( RequestBuilder::new(client, Method::PUT, &update_url) .body(Some(&update_params)) .expect_status(Some(StatusCode::BAD_REQUEST)), @@ -2111,167 +1609,19 @@ async fn test_instance_reconfigure_add_new_ssm_without_sources_fails( .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .expect("Reconfigure adding new SSM group without sources should fail"); + .expect("Reconfigure adding SSM group without sources should fail"); - // Verify the error message mentions SSM/source requirements - let error_body: serde_json::Value = - serde_json::from_slice(&error.body).unwrap(); - let error_message = error_body["message"].as_str().unwrap_or(""); + let reconfig_error_body: serde_json::Value = + serde_json::from_slice(&reconfig_error.body).unwrap(); + let reconfig_error_message = + reconfig_error_body["message"].as_str().unwrap_or(""); assert!( - error_message.contains("SSM") || error_message.contains("source"), - "Error should mention SSM or source IPs: {error_message}" + reconfig_error_message.contains("SSM") + || reconfig_error_message.contains("source"), + "Reconfigure error should mention SSM or source IPs: {reconfig_error_message}" ); - // Cleanup - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; -} - -/// Test explicit member state transitions during reactivation (Left → Joining → Joined). -/// -/// Verifies the 3-state lifecycle: -/// - Create instance with multicast group → member in "Left" state (stopped) -/// - Start instance → RPW transitions to "Joined" -/// - Stop instance → RPW transitions to "Left" -/// - Start instance again → RPW transitions Left → Joining → Joined (reactivation) -#[nexus_test] -async fn test_member_state_transitions_on_reactivation( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - let project_name = "state-transition-project"; - let instance_name = "state-transition-inst"; - - // Setup - ops::join2( - create_project(client, project_name), - create_default_ip_pools(client), - ) - .await; - create_multicast_ip_pool(client, "state-pool").await; - - // Create instance (stopped) - let instance: Instance = object_create( - client, - &format!("/v1/instances?project={project_name}"), - &InstanceCreate { - identity: IdentityMetadataCreateParams { - name: instance_name.parse().unwrap(), - description: "test state transitions".into(), - }, - ncpus: InstanceCpuCount(2), - memory: ByteCount::from_gibibytes_u32(4), - hostname: instance_name.parse().unwrap(), - user_data: Vec::new(), - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: Vec::new(), - disks: Vec::new(), - boot_disk: None, - start: false, - auto_restart_policy: None, - anti_affinity_groups: Vec::new(), - cpu_platform: None, - multicast_groups: Vec::new(), - }, - ) - .await; - let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - let nexus = &cptestctx.server.server_context().nexus; - - // Join multicast group while stopped - // Use IP within the default pool's range (224.2.0.0 - 224.2.255.255) - let multicast_ip: IpAddr = "224.2.90.1".parse().unwrap(); - let expected_group_name = - format!("mcast-{}", multicast_ip.to_string().replace('.', "-")); - let join_url = format!( - "/v1/instances/{instance_name}/multicast-groups/{multicast_ip}?project={project_name}" - ); - let member: MulticastGroupMember = - put_upsert(client, &join_url, &InstanceMulticastGroupJoin::default()) - .await; - - // Case: Stopped instance -> member in "Left" state - wait_for_member_state( - cptestctx, - &expected_group_name, - member.instance_id, - nexus_db_model::MulticastGroupMemberState::Left, - ) - .await; - - // Start instance - let start_url = - format!("/v1/instances/{instance_name}/start?project={project_name}"); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &start_url) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Start should succeed"); - - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; - - // Case: Running instance -> member now in "Joined" state - wait_for_member_state( - cptestctx, - &expected_group_name, - member.instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Stop instance - let stop_url = - format!("/v1/instances/{instance_name}/stop?project={project_name}"); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &stop_url) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Stop should succeed"); - - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Stopped).await; - - // Case: Stopped after running -> member goes back to "Left" state - wait_for_member_state( - cptestctx, - &expected_group_name, - member.instance_id, - nexus_db_model::MulticastGroupMemberState::Left, - ) - .await; - - // Start instance again (reactivation) - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &start_url) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("Restart should succeed"); - - instance_simulate(nexus, &instance_id).await; - instance_wait_for_state(client, instance_id, InstanceState::Running).await; - - // Case: Reactivation complete -> member goes back to "Joined" state - wait_for_member_state( - cptestctx, - &expected_group_name, - member.instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Cleanup cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; - wait_for_group_deleted(cptestctx, &expected_group_name).await; } /// Test that instance deletion only removes that instance's membership, @@ -2537,7 +1887,7 @@ async fn test_group_with_all_members_left(cptestctx: &ControlPlaneTestContext) { ) .await; - // Now add a second instance to the SAME group + // Now add a second instance to the same group let instance2 = instance_for_multicast_groups( cptestctx, project_name, diff --git a/nexus/tests/integration_tests/multicast/mod.rs b/nexus/tests/integration_tests/multicast/mod.rs index b9bfe574893..0e9e7e14cfc 100644 --- a/nexus/tests/integration_tests/multicast/mod.rs +++ b/nexus/tests/integration_tests/multicast/mod.rs @@ -729,6 +729,66 @@ pub(crate) async fn wait_for_instance_sled_assignment( } } +/// Wait for an instance to reach Running state, driving simulation on each poll. +/// +/// More robust than passively waiting, as it actively drives instance +/// simulation while polling for the Running state. +/// +/// Only use for Running transitions. For Stopped state, use `instance_simulate` +/// once followed by `instance_wait_for_state`, as the VMM gets removed during +/// stop and repeated simulation will fail. +pub(crate) async fn instance_wait_for_running_with_simulation( + cptestctx: &ControlPlaneTestContext, + instance_id: InstanceUuid, +) -> Instance { + let expected_state = InstanceState::Running; + let client = &cptestctx.external_client; + let nexus = &cptestctx.server.server_context().nexus; + let url = format!("/v1/instances/{instance_id}"); + + match wait_for_condition( + || async { + instance_helpers::instance_simulate(nexus, &instance_id).await; + + let response = NexusRequest::object_get(client, &url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .map_err(|e| { + CondCheckError::::Failed(format!( + "request failed: {e}" + )) + })?; + + let instance: Instance = response.parsed_body().map_err(|e| { + CondCheckError::::Failed(format!("parse failed: {e}")) + })?; + + if instance.runtime.run_state == expected_state { + Ok(instance) + } else { + Err(CondCheckError::::NotYet) + } + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + { + Ok(instance) => instance, + Err(poll::Error::TimedOut(elapsed)) => { + panic!( + "instance {instance_id} did not reach {expected_state:?} within {elapsed:?}" + ); + } + Err(poll::Error::PermanentError(err)) => { + panic!( + "failed waiting for instance {instance_id} to reach {expected_state:?}: {err}" + ); + } + } +} + /// Verify that inventory-based sled-to-switch-port mapping is correct. /// /// This validates the entire flow: @@ -943,38 +1003,6 @@ pub(crate) async fn wait_for_group_deleted( } } -/// Verify a group is either deleted or in one of the expected states. -/// -/// Useful when DPD is unavailable and groups can't complete state transitions. -/// For example, when DPD is down during deletion, groups may be stuck in -/// "Creating" or "Deleting" state rather than being fully deleted. -pub(crate) async fn verify_group_deleted_or_in_states( - client: &ClientTestContext, - group_name: &str, - expected_states: &[&str], -) { - let groups_result = - nexus_test_utils::resource_helpers::objects_list_page_authz::< - MulticastGroup, - >(client, "/v1/multicast-groups") - .await; - - let matching_groups: Vec<_> = groups_result - .items - .into_iter() - .filter(|g| g.identity.name == group_name) - .collect(); - - if !matching_groups.is_empty() { - // Group still exists - should be in one of the expected states - let actual_state = &matching_groups[0].state; - assert!( - expected_states.contains(&actual_state.as_str()), - "Group {group_name} should be in one of {expected_states:?} states, found: {actual_state}" - ); - } -} - /// Wait for a multicast group to be deleted from DPD (dataplane) with reconciler activation. /// /// This function waits for the DPD to report that the multicast group no longer exists diff --git a/nexus/tests/integration_tests/multicast/networking_integration.rs b/nexus/tests/integration_tests/multicast/networking_integration.rs index 9027fda51d8..6508f9b0a84 100644 --- a/nexus/tests/integration_tests/multicast/networking_integration.rs +++ b/nexus/tests/integration_tests/multicast/networking_integration.rs @@ -8,7 +8,8 @@ //! //! - External IPs: Instances with ephemeral/floating IPs can join multicast groups //! - Floating IP attach/detach: Multicast membership unaffected by IP changes -//! - Complex network configs: Multiple NICs, VPCs, subnets with multicast + +use std::time::Duration; use http::{Method, StatusCode}; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; @@ -25,271 +26,113 @@ use nexus_types::external_api::views::FloatingIp; use omicron_common::api::external::IpVersion; use omicron_common::api::external::{ ByteCount, IdentityMetadataCreateParams, Instance, InstanceCpuCount, - InstanceState, NameOrId, + NameOrId, }; +use omicron_test_utils::dev::poll::{CondCheckError, wait_for_condition}; use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use super::*; -use crate::integration_tests::instances::{ - fetch_instance_external_ips, instance_simulate, instance_wait_for_state, -}; +use crate::integration_tests::instances::fetch_instance_external_ips; -/// Verify instances can have both external IPs and multicast group membership. +/// Consolidated test for external IP scenarios with multicast group membership. /// -/// External IP allocation works for multicast group members, multicast state persists -/// through external IP operations, and no conflicts occur between external IP and multicast -/// DPD configuration. +/// This test covers three scenarios with shared setup: +/// - Basic external IP attach/detach with multicast +/// - Lifecycle with 1-2 attach/detach cycles +/// - External IP at instance creation #[nexus_test] -async fn test_multicast_with_external_ip_basic( +async fn test_multicast_external_ip_scenarios( cptestctx: &nexus_test_utils::ControlPlaneTestContext< omicron_nexus::Server, >, ) { let client = &cptestctx.external_client; - let project_name = "external-ip-mcast-project"; - let group_name = "external-ip-mcast-group"; - let instance_name = "external-ip-mcast-instance"; + let project_name = "external-ip-scenarios-project"; - // Setup: project and IP pools in parallel + // Shared setup: project and IP pools ops::join3( create_project(client, project_name), create_default_ip_pools(client), // For external IPs create_multicast_ip_pool_with_range( client, - "external-ip-mcast-pool", + "external-ip-scenarios-pool", (224, 100, 0, 1), (224, 100, 0, 255), ), ) .await; - // Create instance (will start by default) - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: instance_name.parse().unwrap(), - description: "Instance with external IP and multicast".to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: instance_name.parse().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![], // Start without external IP - multicast_groups: vec![], - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: true, // Start the instance - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - }; - - let instance_url = format!("/v1/instances?project={project_name}"); - let instance: Instance = - object_create(client, &instance_url, &instance_params).await; - let instance_id = instance.identity.id; - - // Transition instance to Running state - let nexus = &cptestctx.server.server_context().nexus; - let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - instance_simulate(nexus, &instance_uuid).await; - instance_wait_for_state(client, instance_uuid, InstanceState::Running) - .await; - // Ensure multicast test prerequisites (inventory + DPD) are ready ensure_multicast_test_ready(cptestctx).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Add instance to multicast group via instance-centric API - multicast_group_attach(cptestctx, project_name, instance_name, group_name) - .await; - wait_for_group_active(client, group_name).await; - - // Wait for multicast member to reach "Joined" state - wait_for_member_state( - cptestctx, - group_name, - instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Verify member count - let members = list_multicast_group_members(client, group_name).await; - assert_eq!(members.len(), 1, "Should have one multicast member"); - - // Allocate ephemeral external IP to the same instance - let ephemeral_ip_url = format!( - "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" - ); - NexusRequest::new( - RequestBuilder::new(client, Method::POST, &ephemeral_ip_url) - .body(Some(&EphemeralIpCreate { - pool_selector: PoolSelector::Auto { - ip_version: Some(IpVersion::V4), - }, - })) - .expect_status(Some(StatusCode::ACCEPTED)), - ) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .unwrap(); - - // Verify both multicast and external IP work together - - // Check that multicast membership is preserved - let members_after_ip = - list_multicast_group_members(client, group_name).await; - assert_eq!( - members_after_ip.len(), - 1, - "Multicast member should still exist after external IP allocation" - ); - assert_eq!(members_after_ip[0].instance_id, instance_id); - assert_eq!( - members_after_ip[0].state, "Joined", - "Member state should remain Joined" - ); - - // Check that external IP is properly attached - let external_ips_after_attach = - fetch_instance_external_ips(client, instance_name, project_name).await; - assert!( - !external_ips_after_attach.is_empty(), - "Instance should have external IP" - ); - // Note: external_ip.ip() from the response may differ from what's actually attached, - // so we just verify that an external IP exists - // Remove ephemeral external IP and verify multicast is unaffected - let external_ip_detach_url = format!( - "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" - ); - object_delete(client, &external_ip_detach_url).await; - - // Wait for operations to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify multicast membership is still intact after external IP removal - let members_after_detach = - list_multicast_group_members(client, group_name).await; - assert_eq!( - members_after_detach.len(), - 1, - "Multicast member should persist after external IP removal" - ); - assert_eq!(members_after_detach[0].instance_id, instance_id); - assert_eq!( - members_after_detach[0].state, "Joined", - "Member should remain Joined" - ); - - // Verify ephemeral external IP is removed (SNAT IP may still be present) - let external_ips_after_detach = - fetch_instance_external_ips(client, instance_name, project_name).await; - // Instance should have at most 1 IP left (the SNAT IP), not the ephemeral IP we attached - assert!( - external_ips_after_detach.len() <= 1, - "Instance should have at most SNAT IP remaining" - ); - - // Cleanup - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; - wait_for_group_deleted(cptestctx, group_name).await; -} - -/// Verify external IP allocation/deallocation lifecycle for multicast group members. -/// -/// Multiple external IP attach/detach cycles don't affect multicast state, concurrent -/// operations don't cause race conditions, and dataplane configuration remains consistent -/// throughout the lifecycle. -#[nexus_test] -async fn test_multicast_external_ip_lifecycle( - cptestctx: &nexus_test_utils::ControlPlaneTestContext< - omicron_nexus::Server, - >, -) { - let client = &cptestctx.external_client; - let project_name = "external-ip-lifecycle-project"; - let group_name = "external-ip-lifecycle-group"; - let instance_name = "external-ip-lifecycle-instance"; - - // Setup in parallel - ops::join3( - create_project(client, project_name), - create_default_ip_pools(client), - create_multicast_ip_pool_with_range( - client, - "external-ip-lifecycle-pool", - (224, 101, 0, 1), - (224, 101, 0, 255), - ), - ) - .await; - - // Create instance - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: instance_name.parse().unwrap(), - description: "Instance for external IP lifecycle test".to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: instance_name.parse().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![], - multicast_groups: vec![], - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: true, - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - }; + // Case: Basic external IP attach/detach with multicast + // + // Verify instances can have both external IPs and multicast group membership. + // External IP allocation works for multicast group members, multicast state + // persists through external IP operations. + { + let instance_name = "basic-attach-detach-instance"; + let group_name = "basic-attach-detach-group"; + + // Create instance (will start by default) + let instance_params = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: instance_name.parse().unwrap(), + description: "Instance with external IP and multicast" + .to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: instance_name.parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![], // Start without external IP + multicast_groups: vec![], + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: true, // Start the instance + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + }; + + let instance_url = format!("/v1/instances?project={project_name}"); + let instance: Instance = + object_create(client, &instance_url, &instance_params).await; + let instance_id = instance.identity.id; + + // Transition instance to Running state + let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); + instance_wait_for_running_with_simulation(cptestctx, instance_uuid) + .await; - let instance_url = format!("/v1/instances?project={project_name}"); - let instance: Instance = - object_create(client, &instance_url, &instance_params).await; - let instance_id = instance.identity.id; + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Start instance and add to multicast group - let nexus = &cptestctx.server.server_context().nexus; - let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - instance_simulate(nexus, &instance_uuid).await; - instance_wait_for_state(client, instance_uuid, InstanceState::Running) + // Add instance to multicast group via instance-centric API + multicast_group_attach( + cptestctx, + project_name, + instance_name, + group_name, + ) .await; - - // Ensure multicast test prerequisites (inventory + DPD) are ready - ensure_multicast_test_ready(cptestctx).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Add instance to multicast group via instance-centric API - multicast_group_attach(cptestctx, project_name, instance_name, group_name) + wait_for_group_active(client, group_name).await; + + // Wait for multicast member to reach "Joined" state + wait_for_member_state( + cptestctx, + group_name, + instance_id, + nexus_db_model::MulticastGroupMemberState::Joined, + ) .await; - wait_for_group_active(client, group_name).await; - // Wait for member to transition from "Joining"->"Joined" - wait_for_member_state( - cptestctx, - group_name, - instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - - // Verify initial multicast state - let initial_members = - list_multicast_group_members(client, group_name).await; - assert_eq!(initial_members.len(), 1); - assert_eq!(initial_members[0].state, "Joined"); + // Verify member count + let members = list_multicast_group_members(client, group_name).await; + assert_eq!(members.len(), 1, "Should have one multicast member"); - // Test multiple external IP allocation/deallocation cycles - for cycle in 1..=3 { - // Allocate ephemeral external IP + // Allocate ephemeral external IP to the same instance let ephemeral_ip_url = format!( "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" ); @@ -307,32 +150,30 @@ async fn test_multicast_external_ip_lifecycle( .await .unwrap(); - // Wait for dataplane configuration to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify multicast state is preserved - let members_with_ip = + // Check that multicast membership is preserved + let members_after_ip = list_multicast_group_members(client, group_name).await; assert_eq!( - members_with_ip.len(), + members_after_ip.len(), 1, - "Cycle {cycle}: Multicast member should persist during external IP allocation" + "Multicast member should still exist after external IP allocation" ); + assert_eq!(members_after_ip[0].instance_id, instance_id); assert_eq!( - members_with_ip[0].state, "Joined", - "Cycle {cycle}: Member should remain Joined" + members_after_ip[0].state, "Joined", + "Member state should remain Joined" ); - // Verify external IP is attached - let external_ips_with_ip = + // Check that external IP is properly attached + let external_ips_after_attach = fetch_instance_external_ips(client, instance_name, project_name) .await; assert!( - !external_ips_with_ip.is_empty(), - "Cycle {cycle}: Instance should have external IP" + !external_ips_after_attach.is_empty(), + "Instance should have external IP" ); - // Deallocate ephemeral external IP + // Remove ephemeral external IP and verify multicast is unaffected let external_ip_detach_url = format!( "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" ); @@ -341,138 +182,286 @@ async fn test_multicast_external_ip_lifecycle( // Wait for operations to settle wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify multicast state is still preserved - let members_without_ip = + // Verify multicast membership is still intact after external IP removal + let members_after_detach = list_multicast_group_members(client, group_name).await; assert_eq!( - members_without_ip.len(), + members_after_detach.len(), 1, - "Cycle {cycle}: Multicast member should persist after external IP removal" + "Multicast member should persist after external IP removal" ); + assert_eq!(members_after_detach[0].instance_id, instance_id); assert_eq!( - members_without_ip[0].state, "Joined", - "Cycle {cycle}: Member should remain Joined after IP removal" + members_after_detach[0].state, "Joined", + "Member should remain Joined" ); // Verify ephemeral external IP is removed (SNAT IP may still be present) - let external_ips_without_ip = + let external_ips_after_detach = fetch_instance_external_ips(client, instance_name, project_name) .await; assert!( - external_ips_without_ip.len() <= 1, - "Cycle {cycle}: Instance should have at most SNAT IP remaining" + external_ips_after_detach.len() <= 1, + "Instance should have at most SNAT IP remaining" ); + + // Cleanup + cleanup_instances(cptestctx, client, project_name, &[instance_name]) + .await; + wait_for_group_deleted(cptestctx, group_name).await; } - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; - wait_for_group_deleted(cptestctx, group_name).await; -} + // Case: Lifecycle with 1-2 attach/detach cycles + // + // Verify external IP allocation/deallocation lifecycle for multicast group + // members. Multiple external IP attach/detach cycles don't affect multicast + // state and dataplane configuration remains consistent throughout. + { + let instance_name = "lifecycle-instance"; + let group_name = "lifecycle-group"; + + // Create instance + let instance_params = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: instance_name.parse().unwrap(), + description: "Instance for external IP lifecycle test" + .to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: instance_name.parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![], + multicast_groups: vec![], + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: true, + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + }; + + let instance_url = format!("/v1/instances?project={project_name}"); + let instance: Instance = + object_create(client, &instance_url, &instance_params).await; + let instance_id = instance.identity.id; + + // Start instance and add to multicast group + let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); + instance_wait_for_running_with_simulation(cptestctx, instance_uuid) + .await; -/// Verify instances can be created with both external IP and multicast group -/// simultaneously. -/// -/// Instance creation with both features works without conflicts during initial setup, -/// and both features are properly configured from creation. -#[nexus_test] -async fn test_multicast_with_external_ip_at_creation( - cptestctx: &nexus_test_utils::ControlPlaneTestContext< - omicron_nexus::Server, - >, -) { - let client = &cptestctx.external_client; - let project_name = "creation-mixed-project"; - let group_name = "creation-mixed-group"; - let instance_name = "creation-mixed-instance"; + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Setup - parallelize project and pool creation - ops::join3( - create_project(client, project_name), - create_default_ip_pools(client), - create_multicast_ip_pool_with_range( - client, - "creation-mixed-pool", - (224, 102, 0, 1), - (224, 102, 0, 255), - ), - ) - .await; + // Add instance to multicast group via instance-centric API + multicast_group_attach( + cptestctx, + project_name, + instance_name, + group_name, + ) + .await; + wait_for_group_active(client, group_name).await; + + // Wait for member to transition from "Joining"->"Joined" + wait_for_member_state( + cptestctx, + group_name, + instance_id, + nexus_db_model::MulticastGroupMemberState::Joined, + ) + .await; - // Create instance with external IP specified at creation - let external_ip_param = ExternalIpCreate::Ephemeral { - pool_selector: PoolSelector::Auto { ip_version: Some(IpVersion::V4) }, - }; - let instance_params = InstanceCreate { - identity: IdentityMetadataCreateParams { - name: instance_name.parse().unwrap(), - description: "Instance created with external IP and multicast" - .to_string(), - }, - ncpus: InstanceCpuCount::try_from(1).unwrap(), - memory: ByteCount::from_gibibytes_u32(1), - hostname: instance_name.parse().unwrap(), - user_data: vec![], - ssh_public_keys: None, - network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, - external_ips: vec![external_ip_param], // External IP at creation - multicast_groups: vec![], // Will add to multicast group after creation - disks: vec![], - boot_disk: None, - cpu_platform: None, - start: true, - auto_restart_policy: Default::default(), - anti_affinity_groups: Vec::new(), - }; + // Verify initial multicast state + let initial_members = + list_multicast_group_members(client, group_name).await; + assert_eq!(initial_members.len(), 1); + assert_eq!(initial_members[0].state, "Joined"); + + // Test 2 external IP allocation/deallocation cycles + for cycle in 1..=2 { + // Allocate ephemeral external IP + let ephemeral_ip_url = format!( + "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" + ); + NexusRequest::new( + RequestBuilder::new(client, Method::POST, &ephemeral_ip_url) + .body(Some(&EphemeralIpCreate { + pool_selector: PoolSelector::Auto { + ip_version: Some(IpVersion::V4), + }, + })) + .expect_status(Some(StatusCode::ACCEPTED)), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); + + // Wait for dataplane configuration to settle + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; + + // Verify multicast state is preserved + let members_with_ip = + list_multicast_group_members(client, group_name).await; + assert_eq!( + members_with_ip.len(), + 1, + "Cycle {cycle}: Multicast member should persist during external IP allocation" + ); + assert_eq!( + members_with_ip[0].state, "Joined", + "Cycle {cycle}: Member should remain Joined" + ); + + // Verify external IP is attached + let external_ips_with_ip = fetch_instance_external_ips( + client, + instance_name, + project_name, + ) + .await; + assert!( + !external_ips_with_ip.is_empty(), + "Cycle {cycle}: Instance should have external IP" + ); + + // Deallocate ephemeral external IP + let external_ip_detach_url = format!( + "/v1/instances/{instance_name}/external-ips/ephemeral?project={project_name}" + ); + object_delete(client, &external_ip_detach_url).await; + + // Wait for operations to settle + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; + + // Verify multicast state is still preserved + let members_without_ip = + list_multicast_group_members(client, group_name).await; + assert_eq!( + members_without_ip.len(), + 1, + "Cycle {cycle}: Multicast member should persist after external IP removal" + ); + assert_eq!( + members_without_ip[0].state, "Joined", + "Cycle {cycle}: Member should remain Joined after IP removal" + ); + + // Verify ephemeral external IP is removed (SNAT IP may still be present) + let external_ips_without_ip = fetch_instance_external_ips( + client, + instance_name, + project_name, + ) + .await; + assert!( + external_ips_without_ip.len() <= 1, + "Cycle {cycle}: Instance should have at most SNAT IP remaining" + ); + } + + // Cleanup + cleanup_instances(cptestctx, client, project_name, &[instance_name]) + .await; + wait_for_group_deleted(cptestctx, group_name).await; + } - let instance_url = format!("/v1/instances?project={project_name}"); - let instance: Instance = - object_create(client, &instance_url, &instance_params).await; - let instance_id = instance.identity.id; + // Case: External IP at instance creation + // + // Verify instances can be created with both external IP and multicast group + // simultaneously. Instance creation with both features works without + // conflicts during initial setup. + { + let instance_name = "creation-with-ip-instance"; + let group_name = "creation-with-ip-group"; + + // Create instance with external IP specified at creation + let external_ip_param = ExternalIpCreate::Ephemeral { + pool_selector: PoolSelector::Auto { + ip_version: Some(IpVersion::V4), + }, + }; + let instance_params = InstanceCreate { + identity: IdentityMetadataCreateParams { + name: instance_name.parse().unwrap(), + description: "Instance created with external IP and multicast" + .to_string(), + }, + ncpus: InstanceCpuCount::try_from(1).unwrap(), + memory: ByteCount::from_gibibytes_u32(1), + hostname: instance_name.parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: InstanceNetworkInterfaceAttachment::DefaultIpv4, + external_ips: vec![external_ip_param], // External IP at creation + multicast_groups: vec![], // Will add to multicast group after creation + disks: vec![], + boot_disk: None, + cpu_platform: None, + start: true, + auto_restart_policy: Default::default(), + anti_affinity_groups: Vec::new(), + }; + + let instance_url = format!("/v1/instances?project={project_name}"); + let instance: Instance = + object_create(client, &instance_url, &instance_params).await; + let instance_id = instance.identity.id; + + // Transition to running + let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); + instance_wait_for_running_with_simulation(cptestctx, instance_uuid) + .await; - // Transition to running - let nexus = &cptestctx.server.server_context().nexus; - let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - instance_simulate(nexus, &instance_uuid).await; - instance_wait_for_state(client, instance_uuid, InstanceState::Running) - .await; + wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Ensure multicast test prerequisites (inventory + DPD) are ready - ensure_multicast_test_ready(cptestctx).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - - // Verify external IP was allocated at creation - let external_ips_after_start = - fetch_instance_external_ips(client, instance_name, project_name).await; - assert!( - !external_ips_after_start.is_empty(), - "Instance should have external IP from creation" - ); + // Verify external IP was allocated at creation + let external_ips_after_start = + fetch_instance_external_ips(client, instance_name, project_name) + .await; + assert!( + !external_ips_after_start.is_empty(), + "Instance should have external IP from creation" + ); - // Add to multicast group via instance-centric API - multicast_group_attach(cptestctx, project_name, instance_name, group_name) + // Add to multicast group via instance-centric API + multicast_group_attach( + cptestctx, + project_name, + instance_name, + group_name, + ) + .await; + wait_for_group_active(client, group_name).await; + + // Verify both features work together - wait for member to reach Joined state + wait_for_member_state( + cptestctx, + group_name, + instance_id, + nexus_db_model::MulticastGroupMemberState::Joined, + ) .await; - wait_for_group_active(client, group_name).await; - - // Verify both features work together - wait for member to reach Joined state - wait_for_member_state( - cptestctx, - group_name, - instance_id, - nexus_db_model::MulticastGroupMemberState::Joined, - ) - .await; - let members = list_multicast_group_members(client, group_name).await; - assert_eq!(members.len(), 1, "Should have multicast member"); + let members = list_multicast_group_members(client, group_name).await; + assert_eq!(members.len(), 1, "Should have multicast member"); - let external_ips_final = - fetch_instance_external_ips(client, instance_name, project_name).await; - assert!( - !external_ips_final.is_empty(), - "Instance should retain external IP" - ); + let external_ips_final = + fetch_instance_external_ips(client, instance_name, project_name) + .await; + assert!( + !external_ips_final.is_empty(), + "Instance should retain external IP" + ); - cleanup_instances(cptestctx, client, project_name, &[instance_name]).await; - wait_for_group_deleted(cptestctx, group_name).await; + // Cleanup + cleanup_instances(cptestctx, client, project_name, &[instance_name]) + .await; + wait_for_group_deleted(cptestctx, group_name).await; + } } /// Verify instances can have both floating IPs and multicast group membership. @@ -542,21 +531,34 @@ async fn test_multicast_with_floating_ip_basic( object_create(client, &instance_url, &instance_params).await; let instance_id = instance.identity.id; - // Transition instance to Running state - let nexus = &cptestctx.server.server_context().nexus; let instance_uuid = InstanceUuid::from_untyped_uuid(instance_id); - instance_simulate(nexus, &instance_uuid).await; - instance_wait_for_state(client, instance_uuid, InstanceState::Running) - .await; + wait_for_instance_sled_assignment(cptestctx, &instance_uuid).await; + instance_wait_for_running_with_simulation(cptestctx, instance_uuid).await; // Ensure multicast test prerequisites (inventory + DPD) are ready ensure_multicast_test_ready(cptestctx).await; - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; // Add instance to multicast group via instance-centric API multicast_group_attach(cptestctx, project_name, instance_name, group_name) .await; - wait_for_group_active(client, group_name).await; + // Group activation is reconciler-driven; explicitly drive it to avoid flakes. + wait_for_condition_with_reconciler( + &cptestctx.lockstep_client, + || async { + let group = get_multicast_group(client, group_name).await; + if group.state == "Active" { + Ok(()) + } else { + Err(CondCheckError::::NotYet) + } + }, + &POLL_INTERVAL, + &MULTICAST_OPERATION_TIMEOUT, + ) + .await + .unwrap_or_else(|e| { + panic!("group {group_name} did not reach Active state in time: {e:?}") + }); // Wait for multicast member to reach "Joined" state wait_for_member_state( @@ -614,16 +616,32 @@ async fn test_multicast_with_floating_ip_basic( ); // Check that floating IP is properly attached - let external_ips_after_attach = - fetch_instance_external_ips(client, instance_name, project_name).await; - assert!( - !external_ips_after_attach.is_empty(), - "Instance should have external IP" - ); - // Find the floating IP among the external IPs (there may also be SNAT IP) - let has_floating_ip = - external_ips_after_attach.iter().any(|ip| ip.ip() == floating_ip.ip); - assert!(has_floating_ip, "Instance should have the floating IP attached"); + wait_for_condition( + || async { + let external_ips = fetch_instance_external_ips( + client, + instance_name, + project_name, + ) + .await; + let has_floating_ip = + external_ips.iter().any(|ip| ip.ip() == floating_ip.ip); + if has_floating_ip { + Ok(()) + } else { + Err(CondCheckError::::NotYet) + } + }, + &Duration::from_millis(200), + &Duration::from_secs(30), + ) + .await + .unwrap_or_else(|e| { + panic!( + "instance did not show floating IP {} as attached within 30s: {e:?}", + floating_ip.ip + ) + }); // Detach floating IP and verify multicast is unaffected let detach_url = format!( @@ -640,9 +658,6 @@ async fn test_multicast_with_floating_ip_basic( .parsed_body::() .unwrap(); - // Wait for operations to settle - wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; - // Verify multicast membership is still intact after floating IP removal let members_after_detach = list_multicast_group_members(client, group_name).await; @@ -658,14 +673,32 @@ async fn test_multicast_with_floating_ip_basic( ); // Verify floating IP is detached (SNAT IP may still be present) - let external_ips_after_detach = - fetch_instance_external_ips(client, instance_name, project_name).await; - let still_has_floating_ip = - external_ips_after_detach.iter().any(|ip| ip.ip() == floating_ip.ip); - assert!( - !still_has_floating_ip, - "Instance should not have the floating IP attached anymore" - ); + wait_for_condition( + || async { + let external_ips = fetch_instance_external_ips( + client, + instance_name, + project_name, + ) + .await; + let still_has_floating_ip = + external_ips.iter().any(|ip| ip.ip() == floating_ip.ip); + if !still_has_floating_ip { + Ok(()) + } else { + Err(CondCheckError::::NotYet) + } + }, + &Duration::from_millis(200), + &Duration::from_secs(30), + ) + .await + .unwrap_or_else(|e| { + panic!( + "instance still showed floating IP {} as attached after 30s: {e:?}", + floating_ip.ip + ) + }); // Cleanup floating IP let fip_delete_url = diff --git a/nexus/tests/integration_tests/multicast/pool_selection.rs b/nexus/tests/integration_tests/multicast/pool_selection.rs index eac99fc5b3d..22831173b42 100644 --- a/nexus/tests/integration_tests/multicast/pool_selection.rs +++ b/nexus/tests/integration_tests/multicast/pool_selection.rs @@ -5,7 +5,7 @@ //! Integration tests for multicast IP pool selection. //! //! These tests verify pool selection behavior when joining multicast groups: -//! - SSM→ASM fallback when `has_sources=true` but no SSM pool is available +//! - SSM/ASM pool selection based on source IP presence //! - IP version disambiguation when both IPv4 and IPv6 pools exist use http::StatusCode; @@ -23,19 +23,17 @@ use super::*; const PROJECT_NAME: &str = "pool-selection-project"; -/// Test SSM→ASM fallback when joining with sources but only ASM pool exists. +/// Test SSM/ASM pool selection behavior. /// -/// When `has_sources=true` and no SSM pool is linked to the silo, the system -/// should fall back to using an ASM pool. This is valid because source filtering -/// still works on ASM addresses via IGMPv3/MLDv2, just without SSM's network-level -/// source guarantees. +/// This test validates three pool selection scenarios: +/// - SSM->ASM fallback when only ASM pool exists (with sources) +/// - SSM pool preferred when both pools exist (with sources) +/// - ASM pool used directly when no sources provided #[nexus_test] -async fn test_ssm_to_asm_fallback_with_sources( - cptestctx: &ControlPlaneTestContext, -) { +async fn test_pool_selection_ssm_asm(cptestctx: &ControlPlaneTestContext) { let client = &cptestctx.external_client; - // Setup: create only ASM pool (no SSM pool) + // Setup: create only ASM pool (no SSM pool) and a single instance for all cases ops::join3( create_default_ip_pools(client), create_project(client, PROJECT_NAME), @@ -44,29 +42,35 @@ async fn test_ssm_to_asm_fallback_with_sources( .await; ensure_multicast_test_ready(cptestctx).await; - // Create an instance + // Create a single instance to reuse for all cases let instance = instance_for_multicast_groups( cptestctx, PROJECT_NAME, - "fallback-instance", + "pool-test-instance", false, // don't start &[], ) .await; - // Join a group BY NAME with sources: - // - has_sources=true (sources provided) - // - Pool selection will try SSM first - // - No SSM pool → should fall back to ASM pool + // Case: SSM->ASM fallback when only ASM pool exists + // + // When sources are provided and no SSM pool is linked to the silo, the system + // should fall back to using an ASM pool. This is valid because source filtering + // still works on ASM addresses via IGMPv3/MLDv2, just without SSM's network-level + // source guarantees. + + // Join a group by name with sources: + // - Sources provided, so pool selection will try SSM first + // - No SSM pool -> should fall back to ASM pool // - Group will be created with ASM IP (224.x.x.x) - let join_url = format!( + let join_url1 = format!( "/v1/instances/{}/multicast-groups/asm-fallback-group?project={PROJECT_NAME}", instance.identity.id ); - let member: MulticastGroupMember = put_upsert( + let member1: MulticastGroupMember = put_upsert( client, - &join_url, + &join_url1, &InstanceMulticastGroupJoin { source_ips: Some(vec![ "10.0.0.1".parse::().unwrap(), @@ -78,73 +82,54 @@ async fn test_ssm_to_asm_fallback_with_sources( .await; // Verify member was created - assert_eq!(member.instance_id, instance.identity.id); + assert_eq!(member1.instance_id, instance.identity.id); - // Activate reconciler to process the new group ("Creating" → "Active") + // Activate reconciler to process the new group ("Creating" -> "Active") wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; // Wait for group to become "Active" (reconciler runs DPD ensure saga) - let group = wait_for_group_active(client, "asm-fallback-group").await; + let group1 = wait_for_group_active(client, "asm-fallback-group").await; // Verify the group got an ASM IP (224.x.x.x) from the fallback - let ip = group.multicast_ip; - match ip { + let ip1 = group1.multicast_ip; + match ip1 { IpAddr::V4(v4) => { assert!( v4.octets()[0] == 224, - "Expected ASM IP (224.x.x.x) from fallback, got {ip}" + "Expected ASM IP (224.x.x.x) from fallback, got {ip1}" ); } IpAddr::V6(_) => { - panic!("Expected IPv4 ASM address, got IPv6: {ip}"); + panic!("Expected IPv4 ASM address, got IPv6: {ip1}"); } } // Verify group is Active - assert_eq!(group.state, "Active"); -} + assert_eq!(group1.state, "Active"); -/// Test that SSM pool is preferred when both ASM and SSM pools exist. -#[nexus_test] -async fn test_ssm_pool_preferred_with_sources( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; + // Case: SSM pool preferred when both pools exist (with sources) + // + // When both ASM and SSM pools exist and sources are provided, the system + // should prefer the SSM pool for source-specific multicast. - // Setup: create both ASM and SSM pools - ops::join4( - create_default_ip_pools(client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(client, "asm-pool"), - create_multicast_ip_pool_with_range( - client, - "ssm-pool", - (232, 1, 0, 0), - (232, 1, 0, 255), - ), - ) - .await; - ensure_multicast_test_ready(cptestctx).await; - - // Create an instance - let instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - "ssm-prefer-instance", - false, - &[], + // Setup: add SSM pool (ASM pool already exists from previous setup) + create_multicast_ip_pool_with_range( + client, + "ssm-pool", + (232, 1, 0, 0), + (232, 1, 0, 255), ) .await; // Join with sources - should use SSM pool (not ASM) - let join_url = format!( + let join_url2 = format!( "/v1/instances/{}/multicast-groups/ssm-preferred-group?project={PROJECT_NAME}", instance.identity.id ); put_upsert::<_, MulticastGroupMember>( client, - &join_url, + &join_url2, &InstanceMulticastGroupJoin { source_ips: Some(vec!["10.0.0.1".parse::().unwrap()]), ip_version: None, @@ -152,95 +137,67 @@ async fn test_ssm_pool_preferred_with_sources( ) .await; - // Activate reconciler to process the new group ("Creating" → "Active") + // Activate reconciler to process the new group ("Creating" -> "Active") wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; // Wait for group to become Active - let group = wait_for_group_active(client, "ssm-preferred-group").await; + let group2 = wait_for_group_active(client, "ssm-preferred-group").await; // Verify the group got an SSM IP (232.x.x.x) - let ip = group.multicast_ip; - match ip { + let ip2 = group2.multicast_ip; + match ip2 { IpAddr::V4(v4) => { assert!( v4.octets()[0] == 232, - "Expected SSM IP (232.x.x.x), got {ip}" + "Expected SSM IP (232.x.x.x), got {ip2}" ); } IpAddr::V6(_) => { - panic!("Expected IPv4 SSM address, got IPv6: {ip}"); + panic!("Expected IPv4 SSM address, got IPv6: {ip2}"); } } - assert_eq!(group.state, "Active"); -} - -/// Test that ASM pool is used directly when no sources provided. -#[nexus_test] -async fn test_asm_pool_used_without_sources( - cptestctx: &ControlPlaneTestContext, -) { - let client = &cptestctx.external_client; - - // Setup: create both ASM and SSM pools - ops::join4( - create_default_ip_pools(client), - create_project(client, PROJECT_NAME), - create_multicast_ip_pool(client, "asm-pool"), - create_multicast_ip_pool_with_range( - client, - "ssm-pool", - (232, 1, 0, 0), - (232, 1, 0, 255), - ), - ) - .await; - ensure_multicast_test_ready(cptestctx).await; + assert_eq!(group2.state, "Active"); - // Create an instance - let instance = instance_for_multicast_groups( - cptestctx, - PROJECT_NAME, - "asm-direct-instance", - false, - &[], - ) - .await; + // Case: ASM pool used directly when no sources provided + // + // When no sources are provided (ASM mode), the system should use an ASM pool + // directly, even if an SSM pool is also available. // Join without sources - should use ASM pool directly (skip SSM) - let join_url = format!( + let join_url3 = format!( "/v1/instances/{}/multicast-groups/asm-direct-group?project={PROJECT_NAME}", instance.identity.id ); put_upsert::<_, MulticastGroupMember>( client, - &join_url, + &join_url3, &InstanceMulticastGroupJoin { source_ips: None, ip_version: None }, ) .await; - // Activate reconciler to process the new group ("Creating" → "Active") + // Activate reconciler to process the new group ("Creating" -> "Active") wait_for_multicast_reconciler(&cptestctx.lockstep_client).await; // Wait for group to become Active - let group = wait_for_group_active(client, "asm-direct-group").await; + let group3 = wait_for_group_active(client, "asm-direct-group").await; // Verify the group got an ASM IP (224.x.x.x) - let ip = group.multicast_ip; - match ip { + let ip3 = group3.multicast_ip; + match ip3 { IpAddr::V4(v4) => { assert!( v4.octets()[0] == 224, - "Expected ASM IP (224.x.x.x), got {ip}" + "Expected ASM IP (224.x.x.x), got {ip3}" ); } IpAddr::V6(_) => { - panic!("Expected IPv4 ASM address, got IPv6: {ip}"); + panic!("Expected IPv4 ASM address, got IPv6: {ip3}"); } } - assert_eq!(group.state, "Active"); + assert_eq!(group3.state, "Active"); } /// Test IP version disambiguation when both IPv4 and IPv6 multicast pools exist.