From 042f398c27e01ae1170fc20e9a4cab0e02414066 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 5 May 2026 07:29:07 +0000 Subject: [PATCH 1/3] [internal-dns] register and publish ddmd in the switch zone DDMD has always run in the switch zone alongside Dendrite, MGS, and MGD, but it was never registered in internal DNS, leaving no path for a cross-host consumer to discover it. This adds `ServiceName::Ddm`, plumbs `ddm_port` through the host-zone switch (RSS plan + reconfigurator DNS execution), threads an `Overridables::ddm_ports` map for the test suite, and lands a `DdmInstance` dropshot sim in test utils so that the test harness registers a real DDM port in DNS the same way it does for the other switch-zone services. We also drop the duplicate DDMD_PORT const in `ddm-admin-client` in favor of the canonical `omicron_common::address::DDMD_PORT`. Same-host callers continue to use `Client::localhost()`. This was extracted from the multicast PR (zl/multicast-mgd-ddm), which uses ddmd cross-host as the first DNS-resolved consumer, as Nexus is the consumer. --- clients/ddm-admin-client/src/lib.rs | 4 +- internal-dns/types/src/config.rs | 71 +++++++++++++++- internal-dns/types/src/names.rs | 5 +- nexus/reconfigurator/execution/src/dns.rs | 7 +- .../execution/src/test_utils.rs | 2 + nexus/reconfigurator/planning/src/example.rs | 3 +- nexus/test-utils/src/nexus_test.rs | 4 + nexus/test-utils/src/starter.rs | 30 +++++++ .../tests/integration_tests/initialization.rs | 7 ++ nexus/types/src/deployment/execution/dns.rs | 1 + .../src/deployment/execution/overridables.rs | 13 +++ sled-agent/rack-setup/src/plan/service.rs | 9 ++- test-utils/src/dev/maghemite.rs | 80 +++++++++++++++++++ 13 files changed, 222 insertions(+), 14 deletions(-) diff --git a/clients/ddm-admin-client/src/lib.rs b/clients/ddm-admin-client/src/lib.rs index 7a8b56d499d..466a8883918 100644 --- a/clients/ddm-admin-client/src/lib.rs +++ b/clients/ddm-admin-client/src/lib.rs @@ -13,6 +13,7 @@ pub use ddm_admin_client::types; use ddm_admin_client::Client as InnerClient; use either::Either; +use omicron_common::address::DDMD_PORT; use oxnet::Ipv6Net; use sled_hardware_types::underlay::BOOTSTRAP_MASK; use sled_hardware_types::underlay::BOOTSTRAP_PREFIX; @@ -26,9 +27,6 @@ use thiserror::Error; use crate::types::EnableStatsRequest; -// TODO-cleanup Is it okay to hardcode this port number here? -const DDMD_PORT: u16 = 8000; - #[derive(Debug, Error, SlogInlineError)] pub enum DdmError { #[error("Failed to construct an HTTP client:")] diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index d5bef144343..f6b04753a77 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -399,6 +399,7 @@ impl DnsConfigBuilder { dendrite_port: u16, mgs_port: u16, mgd_port: u16, + ddm_port: u16, ) -> anyhow::Result<()> { let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; @@ -407,7 +408,8 @@ impl DnsConfigBuilder { &zone, mgs_port, )?; - self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port) + self.service_backend_zone(ServiceName::Mgd, &zone, mgd_port)?; + self.service_backend_zone(ServiceName::Ddm, &zone, ddm_port) } /// Higher-level shorthand for adding a Nexus zone with both its internal @@ -731,7 +733,7 @@ impl DnsConfigBuilder { #[cfg(test)] mod test { - use super::{DnsConfigBuilder, Host, ServiceName}; + use super::{DnsConfigBuilder, DnsRecord, Host, ServiceName}; use crate::{config::Zone, names::DNS_ZONE}; use omicron_common::api::external::Generation; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; @@ -779,6 +781,8 @@ mod test { "_oximeter-reader._tcp", ); assert_eq!(ServiceName::Dendrite.dns_name(), "_dendrite._tcp",); + assert_eq!(ServiceName::Mgd.dns_name(), "_mgd._tcp",); + assert_eq!(ServiceName::Ddm.dns_name(), "_ddm._tcp",); assert_eq!( ServiceName::CruciblePantry.dns_name(), "_crucible-pantry._tcp", @@ -796,6 +800,69 @@ mod test { ); } + #[test] + fn host_zone_switch_publishes_all_services() { + let sled_uuid: SledUuid = + "001de000-51ed-4000-8000-000000000001".parse().unwrap(); + let switch_zone_ip = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1); + + // Use distinct port numbers so an arg-order swap in `host_zone_switch` + // surfaces as a port mismatch on the affected service. + let dendrite_port = 11; + let mgs_port = 13; + let mgd_port = 17; + let ddm_port = 19; + + let mut builder = DnsConfigBuilder::new(); + builder + .host_zone_switch( + sled_uuid, + switch_zone_ip, + dendrite_port, + mgs_port, + mgd_port, + ddm_port, + ) + .unwrap(); + + let config = builder.build_full_config_for_initial_generation(); + + let mut by_name: BTreeMap<&str, &[DnsRecord]> = BTreeMap::new(); + for zone in &config.zones { + for (name, records) in &zone.records { + by_name.insert(name.as_str(), records.as_slice()); + } + } + + for (expected_name, expected_port) in [ + ("_dendrite._tcp", dendrite_port), + ("_mgs._tcp", mgs_port), + ("_mgd._tcp", mgd_port), + ("_ddm._tcp", ddm_port), + ] { + let records = by_name.get(expected_name).unwrap_or_else(|| { + panic!( + "expected {expected_name} in published switch-zone \ + services; got {by_name:?}" + ) + }); + let srv_port = records + .iter() + .find_map(|r| match r { + DnsRecord::Srv(s) => Some(s.port), + _ => None, + }) + .unwrap_or_else(|| { + panic!("no SRV record for {expected_name}: {records:?}") + }); + + assert_eq!( + srv_port, expected_port, + "wrong SRV port for {expected_name}" + ); + } + } + #[test] fn display_hosts() { let sled_uuid = SledUuid::nil(); diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index 73b2439e48e..105d0222f3c 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -75,6 +75,7 @@ pub enum ServiceName { BoundaryNtp, InternalNtp, Mgd, + Ddm, } impl ServiceName { @@ -116,6 +117,7 @@ impl ServiceName { ServiceName::BoundaryNtp => "boundary-ntp", ServiceName::InternalNtp => "internal-ntp", ServiceName::Mgd => "mgd", + ServiceName::Ddm => "ddm", } } @@ -144,7 +146,8 @@ impl ServiceName { | ServiceName::CruciblePantry | ServiceName::BoundaryNtp | ServiceName::InternalNtp - | ServiceName::Mgd => { + | ServiceName::Mgd + | ServiceName::Ddm => { format!("_{}._tcp", self.service_kind()) } ServiceName::SledAgent(id) => { diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 685c7c85e6f..0a85c4dd114 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -988,9 +988,8 @@ mod test { // the previous pass (i.e., that corresponds to an Omicron zone). // // There are some ServiceNames missing here because they are not part of - // our representative config (e.g., ClickhouseKeeper) or they don't - // currently have DNS record at all (e.g., SledAgent, Maghemite, Mgd, - // Tfport). + // our representative config (e.g., ClickhouseKeeper) or because they + // do not currently have a DNS record at all (e.g., SledAgent). let mut srv_kinds_expected = BTreeSet::from([ ServiceName::Clickhouse, ServiceName::ClickhouseNative, @@ -1001,6 +1000,8 @@ mod test { ServiceName::NexusLockstep, ServiceName::Oximeter, ServiceName::Dendrite, + ServiceName::Mgd, + ServiceName::Ddm, ServiceName::CruciblePantry, ServiceName::BoundaryNtp, ServiceName::InternalNtp, diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index cd46adacd0b..fdb17289225 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -113,10 +113,12 @@ pub fn overridables_for_test( let dendrite_port = cptestctx.dendrite.read().unwrap().get(&switch_slot).unwrap().port; let mgd_port = cptestctx.mgd.get(&switch_slot).unwrap().port; + let ddm_port = cptestctx.ddm.get(&switch_slot).unwrap().port; overrides.override_switch_zone_ip(sled_id, ip); overrides.override_dendrite_port(sled_id, dendrite_port); overrides.override_mgs_port(sled_id, mgs_port); overrides.override_mgd_port(sled_id, mgd_port); + overrides.override_ddm_port(sled_id, ddm_port); } overrides } diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index a1f865e2934..7dbbf3640dc 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -1854,7 +1854,8 @@ mod tests { | ServiceName::RepoDepot | ServiceName::ManagementGatewayService | ServiceName::Dendrite - | ServiceName::Mgd => { + | ServiceName::Mgd + | ServiceName::Ddm => { out.insert(service, Ok(())); } // InternalNtp is too large to fit in a single DNS packet and diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 693aea88732..329f6f37d29 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -117,6 +117,7 @@ pub struct ControlPlaneTestContext { /// Ports of stopped dendrite instances (for use by start_dendrite) pub stopped_dendrite_ports: RwLock>, pub mgd: HashMap, + pub ddm: HashMap, pub external_dns_zone_name: String, pub external_dns: TransientDnsServer, pub internal_dns: TransientDnsServer, @@ -320,6 +321,9 @@ impl ControlPlaneTestContext { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut ddm) in self.ddm { + ddm.cleanup().await; + } self.logctx.cleanup_successful(); } } diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index aa9c5cbd268..8a646afea12 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -146,6 +146,7 @@ pub struct ControlPlaneStarter<'a, N: NexusServer> { pub gateway: BTreeMap, pub dendrite: RwLock>, pub mgd: HashMap, + pub ddm: HashMap, // NOTE: Only exists after starting Nexus, until external Nexus is // initialized. @@ -203,6 +204,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { gateway: BTreeMap::new(), dendrite: RwLock::new(HashMap::new()), mgd: HashMap::new(), + ddm: HashMap::new(), nexus_internal: None, nexus_internal_addr: None, external_dns_zone_name: None, @@ -461,6 +463,17 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.config.pkg.mgd.insert(switch_slot, config); } + pub async fn start_ddm(&mut self, switch_slot: SwitchSlot) { + let log = &self.logctx.log; + debug!(log, "Starting DDM sim"; "switch_slot" => ?switch_slot); + + let ddm = dev::maghemite::DdmInstance::start().await.unwrap(); + let port = ddm.port; + self.ddm.insert(switch_slot, ddm); + + debug!(log, "DDM sim started"; "port" => port); + } + pub async fn record_switch_dns( &mut self, sled_id: SledUuid, @@ -482,6 +495,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { self.dendrite.read().unwrap().get(&switch_slot).unwrap().port, self.gateway.get(&switch_slot).unwrap().port, self.mgd.get(&switch_slot).unwrap().port, + self.ddm.get(&switch_slot).unwrap().port, ) .unwrap() } @@ -1250,6 +1264,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { dendrite: RwLock::new(self.dendrite.into_inner().unwrap()), stopped_dendrite_ports: RwLock::new(HashMap::new()), mgd: self.mgd, + ddm: self.ddm, external_dns_zone_name: self.external_dns_zone_name.unwrap(), external_dns: self.external_dns.unwrap(), internal_dns: self.internal_dns.unwrap(), @@ -1291,6 +1306,9 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { for (_, mut mgd) in self.mgd { mgd.cleanup().await.unwrap(); } + for (_, mut ddm) in self.ddm { + ddm.cleanup().await; + } self.logctx.cleanup_successful(); } @@ -1631,6 +1649,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_mgd(SwitchSlot::Switch0).boxed() }), ), + ( + "start_ddm_switch0", + Box::new(|builder| { + builder.start_ddm(SwitchSlot::Switch0).boxed() + }), + ), ( "record_switch_dns", Box::new(|builder| { @@ -1675,6 +1699,12 @@ pub(crate) async fn setup_with_config_impl( builder.start_mgd(SwitchSlot::Switch1).boxed() }), ), + ( + "start_ddm_switch1", + Box::new(|builder| { + builder.start_ddm(SwitchSlot::Switch1).boxed() + }), + ), ( "record_switch_dns", Box::new(|builder| { diff --git a/nexus/tests/integration_tests/initialization.rs b/nexus/tests/integration_tests/initialization.rs index 350757cf1de..714880feb37 100644 --- a/nexus/tests/integration_tests/initialization.rs +++ b/nexus/tests/integration_tests/initialization.rs @@ -158,6 +158,11 @@ async fn test_nexus_boots_before_dendrite() { starter.start_mgd(SwitchSlot::Switch1).await; info!(log, "Started mgd"); + info!(log, "Starting ddm"); + starter.start_ddm(SwitchSlot::Switch0).await; + starter.start_ddm(SwitchSlot::Switch1).await; + info!(log, "Started ddm"); + info!(log, "Populating internal DNS records"); starter .record_switch_dns( @@ -197,6 +202,8 @@ async fn nexus_schema_test_setup( starter.start_dendrite(SwitchSlot::Switch1).await; starter.start_mgd(SwitchSlot::Switch0).await; starter.start_mgd(SwitchSlot::Switch1).await; + starter.start_ddm(SwitchSlot::Switch0).await; + starter.start_ddm(SwitchSlot::Switch1).await; starter.populate_internal_dns().await; } diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index 009377fd8d9..3730576eda2 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -158,6 +158,7 @@ pub fn blueprint_internal_dns_config( overrides.dendrite_port(scrimlet.id()), overrides.mgs_port(scrimlet.id()), overrides.mgd_port(scrimlet.id()), + overrides.ddm_port(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 881a7c49bdd..7dc3ae0bf4d 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use omicron_common::address::DDMD_PORT; use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; use omicron_common::address::MGD_PORT; @@ -29,6 +30,8 @@ pub struct Overridables { pub mgs_ports: BTreeMap, /// map: sled id -> TCP port on which that sled's MGD is listening pub mgd_ports: BTreeMap, + /// map: sled id -> TCP port on which that sled's DDM is listening + pub ddm_ports: BTreeMap, /// map: sled id -> IP address of the sled's switch zone pub switch_zone_ips: BTreeMap, } @@ -67,6 +70,16 @@ impl Overridables { self.mgd_ports.get(&sled_id).copied().unwrap_or(MGD_PORT) } + /// Specify the TCP port on which this sled's DDM is listening + pub fn override_ddm_port(&mut self, sled_id: SledUuid, port: u16) { + self.ddm_ports.insert(sled_id, port); + } + + /// Returns the TCP port on which this sled's DDM is listening + pub fn ddm_port(&self, sled_id: SledUuid) -> u16 { + self.ddm_ports.get(&sled_id).copied().unwrap_or(DDMD_PORT) + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/sled-agent/rack-setup/src/plan/service.rs b/sled-agent/rack-setup/src/plan/service.rs index 2ef6d79489a..59ffc7eb64e 100644 --- a/sled-agent/rack-setup/src/plan/service.rs +++ b/sled-agent/rack-setup/src/plan/service.rs @@ -29,10 +29,10 @@ use nexus_types::deployment::{ }; use nexus_types::external_api::sled::SledState; use omicron_common::address::{ - CP_SERVICES_RESERVED_ADDRESSES, DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, - Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, - NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, ReservedRackSubnet, - SLED_PREFIX, SLED_RESERVED_ADDRESSES, get_sled_address, + CP_SERVICES_RESERVED_ADDRESSES, DDMD_PORT, DENDRITE_PORT, DNS_HTTP_PORT, + DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, + NEXUS_LOCKSTEP_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, + ReservedRackSubnet, SLED_PREFIX, SLED_RESERVED_ADDRESSES, get_sled_address, get_switch_zone_address, }; use omicron_common::api::external::{Generation, MacAddr, Vni}; @@ -341,6 +341,7 @@ impl ServicePlan { DENDRITE_PORT, MGS_PORT, MGD_PORT, + DDMD_PORT, ) .unwrap(); } diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index 4c2d85df3ee..d3cf6524f1a 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,11 +4,13 @@ //! Tools for managing Maghemite during development +use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; use anyhow::Context; +use slog::{Discard, Logger, o}; use tempfile::TempDir; use tokio::{ fs::File, @@ -163,13 +165,91 @@ async fn find_mgd_port_in_log(logfile: String) -> Result { } } +/// In-process stand-in for the `ddmd` (Delay Driven Multipath daemon) +/// admin API. +/// +/// `ddmd` runs in sled global zones and switch zones in real deployments, +/// and depends on illumos networking facilities not available in a generic +/// dev test toolchain the way `mgd` is. This binds a dropshot server on an +/// auto-assigned port so the test suite has a real socket to publish in +/// internal DNS as `ServiceName::Ddm`. +/// +/// This currently has no registered routes. Any integration needing +/// concrete endpoints (e.g., peer lists) must extend the `ApiDescription`. +pub struct DdmInstance { + pub port: u16, + server: Option>, +} + +impl DdmInstance { + /// Start a DDM sim server bound to a random localhost port. + pub async fn start() -> Result { + let dropshot_config = dropshot::ConfigDropshot { + bind_address: SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + )), + ..Default::default() + }; + + let api: dropshot::ApiDescription<()> = dropshot::ApiDescription::new(); + let log = Logger::root(Discard, o!()); + + let server = dropshot::ServerBuilder::new(api, (), log) + .config(dropshot_config) + .start() + .context("failed to start DDM sim server")?; + + let port = server.local_addr().port(); + Ok(Self { port, server: Some(server) }) + } + + pub async fn cleanup(&mut self) { + if let Some(server) = self.server.take() { + server.close().await.expect("failed to close DDM sim server"); + } + } +} + +impl Drop for DdmInstance { + fn drop(&mut self) { + if self.server.is_some() { + eprintln!( + "WARN: dropped DdmInstance without cleaning it up first \ + (the dropshot server's tokio task may still be running)" + ); + } + } +} + #[cfg(test)] mod tests { + use super::DdmInstance; use super::find_mgd_port_in_log; use std::io::Write; use std::process::Stdio; use tempfile::NamedTempFile; + /// Smoke-test `DdmInstance`. We bind and serve a 404 for an unregistered + /// route, then shut down cleanly. + #[tokio::test] + async fn test_ddm_sim_binds_and_serves_404() { + let mut sim = DdmInstance::start().await.expect("DDM sim starts"); + assert!(sim.port > 0, "DDM sim should auto-assign a port"); + + let url = format!("http://[::1]:{}/peers", sim.port); + let resp = reqwest::get(&url).await.expect("server reachable"); + assert_eq!( + resp.status(), + reqwest::StatusCode::NOT_FOUND, + "no routes registered yet: expected 404" + ); + + sim.cleanup().await; + } + const EXPECTED_PORT: u16 = 4676; #[tokio::test] From 5ead40c5fcd99e53ac582e8c775359b1e5e5cc32 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Thu, 7 May 2026 03:03:12 +0000 Subject: [PATCH 2/3] [review, ddmd] Address review ~ HostSwitchZonePorts + real ddmd test fixture We address @jgallagher's review by: - Replacing the four positional `u16` arguments in `DnsConfigBuilder::host_zone_switch` with a `HostSwitchZonePorts` named-fields structure. - Replacing the dropshot-based stubbed `DdmInstance` in test-utils with a fixture that spawns and supervises a real `ddmd` subprocess running with `--no-state-machine`, analogous to `MgdInstance` and `mgd --no-bgp-dispatcher`. Only the switch-zone `ddmd` is registered in internal DNS, while sled-global-zone instances are accessed locally by their own host and don't need DNS registration. This **does** require maghemite changes, already PR'ed to oxidecomputer/maghemite#729. To make this all work, we wire `ddmd` into the developer xtask toolchain. `cargo xtask download maghemite-ddmd` reuses the existing `mg-ddm.tar.gz` illumos zone artifact (extracting `ddmd`/`ddmadm`). On Linux it overlays a raw `ddmd` binary, and on macOS it builds from source. Also, we had to bump `oxnet` from 0.1.4 to 0.1.5 to satisfy the new maghemite pin. --- .envrc | 1 + Cargo.lock | 16 +- Cargo.toml | 8 +- dev-tools/downloader/src/lib.rs | 82 ++++++++++ env.sh | 1 + internal-dns/types/src/config.rs | 39 +++-- nexus/test-utils/src/nexus_test.rs | 2 +- nexus/test-utils/src/starter.rs | 19 ++- nexus/types/src/deployment/execution/dns.rs | 5 +- .../src/deployment/execution/overridables.rs | 17 ++ package-manifest.toml | 12 +- sled-agent/rack-setup/src/plan/service.rs | 12 +- test-utils/src/dev/maghemite.rs | 148 +++++++++++------- tools/install_builder_prerequisites.sh | 1 + tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 6 +- tools/update_maghemite.sh | 8 +- 18 files changed, 276 insertions(+), 105 deletions(-) diff --git a/.envrc b/.envrc index 9731258f343..a6ab4f52174 100644 --- a/.envrc +++ b/.envrc @@ -5,6 +5,7 @@ PATH_add out/cockroachdb/bin PATH_add out/clickhouse PATH_add out/dendrite-stub/bin PATH_add out/mgd/root/opt/oxide/mgd/bin +PATH_add out/mg-ddm/root/opt/oxide/mg-ddm/bin if [ "$OMICRON_USE_FLAKE" = 1 ] && nix flake show &> /dev/null then diff --git a/Cargo.lock b/Cargo.lock index f61caae2532..bafba810445 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2525,10 +2525,10 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=7696ee48d5ee29a917dea459e281fe2e8ff20513#7696ee48d5ee29a917dea459e281fe2e8ff20513" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" dependencies = [ "oxnet", - "progenitor 0.13.0", + "progenitor 0.14.0", "reqwest 0.13.2", "serde", "slog", @@ -6509,11 +6509,11 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=7696ee48d5ee29a917dea459e281fe2e8ff20513#7696ee48d5ee29a917dea459e281fe2e8ff20513" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" dependencies = [ "chrono", "colored 3.1.1", - "progenitor 0.13.0", + "progenitor 0.14.0", "rdb-types", "reqwest 0.13.2", "schemars 0.8.22", @@ -10180,9 +10180,9 @@ dependencies = [ [[package]] name = "oxnet" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc6fb07ecd6d2a17ff1431bc5b3ce11036c0b6dd93a3c4904db5b910817b162" +checksum = "057865b45bb202b17ed475d8f22f0416412de2c317c168fefecf9d207faf048d" dependencies = [ "ipnetwork", "schemars 0.8.22", @@ -11845,7 +11845,7 @@ dependencies = [ [[package]] name = "rdb-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=7696ee48d5ee29a917dea459e281fe2e8ff20513#7696ee48d5ee29a917dea459e281fe2e8ff20513" +source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" dependencies = [ "oxnet", "schemars 0.8.22", @@ -14106,7 +14106,7 @@ version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.117", diff --git a/Cargo.toml b/Cargo.toml index 3e5adc43e60..9d27de57639 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -604,8 +604,8 @@ ntp-admin-api = { path = "ntp-admin/api" } ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-types = { path = "ntp-admin/types" } ntp-admin-types-versions = { path = "ntp-admin/types/versions" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -669,7 +669,7 @@ oxide-client = { path = "clients/oxide-client" } oxide-tokio-rt = "0.1.4" oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "bae0440c199b3908c12903a9532854936353433b", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } -oxnet = "0.1.4" +oxnet = "0.1.5" once_cell = "1.21.3" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapiv3 = "2.2.0" @@ -742,7 +742,7 @@ rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" } rayon = "1.10" rcgen = "0.12.1" -rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "7696ee48d5ee29a917dea459e281fe2e8ff20513" } +rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" } reedline = "0.40.0" ref-cast = "1.0" diff --git a/dev-tools/downloader/src/lib.rs b/dev-tools/downloader/src/lib.rs index 44fb340de28..a7e5a65683b 100644 --- a/dev-tools/downloader/src/lib.rs +++ b/dev-tools/downloader/src/lib.rs @@ -69,6 +69,9 @@ enum Target { /// Maghemite mgd binary MaghemiteMgd, + /// Maghemite ddmd binary + MaghemiteDdmd, + /// SoftNPU, an admin program (scadm) and a pre-compiled P4 program. Softnpu, @@ -137,6 +140,7 @@ pub async fn run_cmd(args: DownloadArgs) -> Result<()> { Target::Console => downloader.download_console().await, Target::DendriteStub => downloader.download_dendrite_stub().await, Target::MaghemiteMgd => downloader.download_maghemite_mgd().await, + Target::MaghemiteDdmd => downloader.download_maghemite_ddmd().await, Target::Softnpu => downloader.download_softnpu().await, Target::TransceiverControl => { downloader.download_transceiver_control().await @@ -946,6 +950,84 @@ impl Downloader<'_> { Ok(()) } + async fn download_maghemite_ddmd(&self) -> Result<()> { + let download_dir = self.output_dir.join("downloads"); + tokio::fs::create_dir_all(&download_dir).await?; + + let checksums_path = self.versions_dir.join("maghemite_mgd_checksums"); + let [mg_ddm_sha2, ddmd_linux_sha2] = get_values_from_file( + ["MG_DDM_SHA256", "DDMD_LINUX_SHA256"], + &checksums_path, + ) + .await?; + let commit_path = + self.versions_dir.join("maghemite_ddm_openapi_version"); + let [commit] = get_values_from_file(["COMMIT"], &commit_path).await?; + + let repo = "oxidecomputer/maghemite"; + let base_url = format!("{BUILDOMAT_URL}/{repo}/image/{commit}"); + + let filename = "mg-ddm.tar.gz"; + let tarball_path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &tarball_path, + &format!("{base_url}/{filename}"), + ChecksumAlgorithm::Sha2, + &mg_ddm_sha2, + ) + .await?; + unpack_tarball(&self.log, &tarball_path, &download_dir).await?; + + let destination_dir = self.output_dir.join("mg-ddm"); + let _ = tokio::fs::remove_dir_all(&destination_dir).await; + tokio::fs::create_dir_all(&destination_dir).await?; + copy_dir_all( + &download_dir.join("root"), + &destination_dir.join("root"), + )?; + + let binary_dir = destination_dir.join("root/opt/oxide/mg-ddm/bin"); + + match os_name()? { + Os::Linux => { + let filename = "ddmd"; + let path = download_dir.join(filename); + download_file_and_verify( + &self.log, + &path, + &format!( + "{BUILDOMAT_URL}/{repo}/linux/{commit}/{filename}" + ), + ChecksumAlgorithm::Sha2, + &ddmd_linux_sha2, + ) + .await?; + set_permissions(&path, 0o755).await?; + tokio::fs::copy(path, binary_dir.join(filename)).await?; + } + Os::Mac => { + info!( + self.log, + "Building maghemite ddmd from source for macOS" + ); + + let binaries = [("ddmd", &["--no-default-features"][..])]; + + let built_binaries = self + .build_from_git("maghemite", &commit, &binaries) + .await?; + + let dest = binary_dir.join("ddmd"); + tokio::fs::copy(&built_binaries[0], &dest).await?; + set_permissions(&dest, 0o755).await?; + } + Os::Illumos => (), + } + + Ok(()) + } + async fn download_softnpu(&self) -> Result<()> { let destination_dir = self.output_dir.join("npuzone"); tokio::fs::create_dir_all(&destination_dir).await?; diff --git a/env.sh b/env.sh index 6a84c35902a..114b53f07ed 100644 --- a/env.sh +++ b/env.sh @@ -12,6 +12,7 @@ export PATH="$OMICRON_WS/out/cockroachdb/bin:$PATH" export PATH="$OMICRON_WS/out/clickhouse:$PATH" export PATH="$OMICRON_WS/out/dendrite-stub/bin:$PATH" export PATH="$OMICRON_WS/out/mgd/root/opt/oxide/mgd/bin:$PATH" +export PATH="$OMICRON_WS/out/mg-ddm/root/opt/oxide/mg-ddm/bin:$PATH" # if xtrace was set previously, do not unset it case $OLD_SHELL_OPTS in diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index f6b04753a77..5b4f736e2c5 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -163,6 +163,20 @@ pub struct DnsConfigBuilder { service_instances_sleds: BTreeMap>, } +/// Ports for the per-switch services published in internal DNS by +/// [`DnsConfigBuilder::host_zone_switch`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct HostSwitchZonePorts { + /// Dendrite (`dpd`) admin API port. + pub dendrite: u16, + /// Management Gateway Service (`mgs`) port. + pub mgs: u16, + /// Maghemite `mgd` admin API port. + pub mgd: u16, + /// Maghemite `ddmd` admin API port. + pub ddm: u16, +} + /// Describes a host of type "sled" in the control plane DNS zone #[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] pub struct Sled(SledUuid); @@ -396,11 +410,14 @@ impl DnsConfigBuilder { &mut self, sled_id: SledUuid, switch_zone_ip: Ipv6Addr, - dendrite_port: u16, - mgs_port: u16, - mgd_port: u16, - ddm_port: u16, + ports: HostSwitchZonePorts, ) -> anyhow::Result<()> { + let HostSwitchZonePorts { + dendrite: dendrite_port, + mgs: mgs_port, + mgd: mgd_port, + ddm: ddm_port, + } = ports; let zone = self.host_dendrite(sled_id, switch_zone_ip)?; self.service_backend_zone(ServiceName::Dendrite, &zone, dendrite_port)?; self.service_backend_zone( @@ -733,7 +750,9 @@ impl DnsConfigBuilder { #[cfg(test)] mod test { - use super::{DnsConfigBuilder, DnsRecord, Host, ServiceName}; + use super::{ + DnsConfigBuilder, DnsRecord, Host, HostSwitchZonePorts, ServiceName, + }; use crate::{config::Zone, names::DNS_ZONE}; use omicron_common::api::external::Generation; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; @@ -818,10 +837,12 @@ mod test { .host_zone_switch( sled_uuid, switch_zone_ip, - dendrite_port, - mgs_port, - mgd_port, - ddm_port, + HostSwitchZonePorts { + dendrite: dendrite_port, + mgs: mgs_port, + mgd: mgd_port, + ddm: ddm_port, + }, ) .unwrap(); diff --git a/nexus/test-utils/src/nexus_test.rs b/nexus/test-utils/src/nexus_test.rs index 329f6f37d29..48c945e742b 100644 --- a/nexus/test-utils/src/nexus_test.rs +++ b/nexus/test-utils/src/nexus_test.rs @@ -322,7 +322,7 @@ impl ControlPlaneTestContext { mgd.cleanup().await.unwrap(); } for (_, mut ddm) in self.ddm { - ddm.cleanup().await; + ddm.cleanup().await.unwrap(); } self.logctx.cleanup_successful(); } diff --git a/nexus/test-utils/src/starter.rs b/nexus/test-utils/src/starter.rs index 8a646afea12..48257f8dab3 100644 --- a/nexus/test-utils/src/starter.rs +++ b/nexus/test-utils/src/starter.rs @@ -23,6 +23,7 @@ use futures::future::BoxFuture; use gateway_test_utils::setup::GatewayTestContext; use iddqd::IdOrdMap; use internal_dns_types::config::DnsConfigBuilder; +use internal_dns_types::config::HostSwitchZonePorts; use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use internal_dns_types::names::ServiceName; use nexus_config::Database; @@ -492,10 +493,18 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { .host_zone_switch( sled_id, Ipv6Addr::LOCALHOST, - self.dendrite.read().unwrap().get(&switch_slot).unwrap().port, - self.gateway.get(&switch_slot).unwrap().port, - self.mgd.get(&switch_slot).unwrap().port, - self.ddm.get(&switch_slot).unwrap().port, + HostSwitchZonePorts { + dendrite: self + .dendrite + .read() + .unwrap() + .get(&switch_slot) + .unwrap() + .port, + mgs: self.gateway.get(&switch_slot).unwrap().port, + mgd: self.mgd.get(&switch_slot).unwrap().port, + ddm: self.ddm.get(&switch_slot).unwrap().port, + }, ) .unwrap() } @@ -1307,7 +1316,7 @@ impl<'a, N: NexusServer> ControlPlaneStarter<'a, N> { mgd.cleanup().await.unwrap(); } for (_, mut ddm) in self.ddm { - ddm.cleanup().await; + ddm.cleanup().await.unwrap(); } self.logctx.cleanup_successful(); } diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index 3730576eda2..c901dcc92f7 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -155,10 +155,7 @@ pub fn blueprint_internal_dns_config( dns_builder.host_zone_switch( scrimlet.id(), switch_zone_ip, - overrides.dendrite_port(scrimlet.id()), - overrides.mgs_port(scrimlet.id()), - overrides.mgd_port(scrimlet.id()), - overrides.ddm_port(scrimlet.id()), + overrides.host_switch_zone_ports(scrimlet.id()), )?; } diff --git a/nexus/types/src/deployment/execution/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs index 7dc3ae0bf4d..bf46374d1dc 100644 --- a/nexus/types/src/deployment/execution/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -2,6 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use internal_dns_types::config::HostSwitchZonePorts; use omicron_common::address::DDMD_PORT; use omicron_common::address::DENDRITE_PORT; use omicron_common::address::Ipv6Subnet; @@ -80,6 +81,22 @@ impl Overridables { self.ddm_ports.get(&sled_id).copied().unwrap_or(DDMD_PORT) } + /// Returns the per-switch-zone service ports for this sled. + /// + /// Bundles the four switch-zone admin ports into a single + /// [`HostSwitchZonePorts`] so callers cannot swap fields by accident. + pub fn host_switch_zone_ports( + &self, + sled_id: SledUuid, + ) -> HostSwitchZonePorts { + HostSwitchZonePorts { + dendrite: self.dendrite_port(sled_id), + mgs: self.mgs_port(sled_id), + mgd: self.mgd_port(sled_id), + ddm: self.ddm_port(sled_id), + } + } + /// Specify the IP address of this switch zone pub fn override_switch_zone_ip( &mut self, diff --git a/package-manifest.toml b/package-manifest.toml index d742f5b5338..daf5ea0bfce 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -683,10 +683,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "7696ee48d5ee29a917dea459e281fe2e8ff20513" +source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "ce52b9094adf0ed567bd3ed1e3ac48ac1c983cc7859adacf4f392e415a1189ad" +source.sha256 = "c26d680b8d01210cee8f081fad24cb0b3aeb03be509b76ca04a99f38f3ea9121" output.type = "tarball" [package.mg-ddm] @@ -699,10 +699,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "7696ee48d5ee29a917dea459e281fe2e8ff20513" +source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "23950a4e73a07fa7f087ba3312e4bc5a8981fd9ebad54af2350baaa86ad6bbf3" +source.sha256 = "020a85457a5d868fb0078e9b29b7b7d126ca494408a87eaccbff5a2b4ef3c628" output.type = "zone" output.intermediate_only = true @@ -714,10 +714,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "7696ee48d5ee29a917dea459e281fe2e8ff20513" +source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "301d31ca481e4822f69484feacca31dd08a7c4aae87d96641d384bda3178d2f3" +source.sha256 = "69f6bc36806b799174897762f0b10885d600a747018d1f9dbf2caeae9c749841" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/rack-setup/src/plan/service.rs b/sled-agent/rack-setup/src/plan/service.rs index 59ffc7eb64e..63702c1090c 100644 --- a/sled-agent/rack-setup/src/plan/service.rs +++ b/sled-agent/rack-setup/src/plan/service.rs @@ -13,7 +13,7 @@ use iddqd::errors::DuplicateItem; use iddqd::id_upcast; use illumos_utils::zpool::ZpoolName; use internal_dns_types::config::{ - DnsConfigBuilder, DnsConfigParams, Host, Zone, + DnsConfigBuilder, DnsConfigParams, Host, HostSwitchZonePorts, Zone, }; use internal_dns_types::names::ServiceName; use nexus_types::deployment::LastAllocatedSubnetIpOffset; @@ -338,10 +338,12 @@ impl ServicePlan { .host_zone_switch( sled.sled_id, address, - DENDRITE_PORT, - MGS_PORT, - MGD_PORT, - DDMD_PORT, + HostSwitchZonePorts { + dendrite: DENDRITE_PORT, + mgs: MGS_PORT, + mgd: MGD_PORT, + ddm: DDMD_PORT, + }, ) .unwrap(); } diff --git a/test-utils/src/dev/maghemite.rs b/test-utils/src/dev/maghemite.rs index d3cf6524f1a..00225c8737a 100644 --- a/test-utils/src/dev/maghemite.rs +++ b/test-utils/src/dev/maghemite.rs @@ -4,13 +4,11 @@ //! Tools for managing Maghemite during development -use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6}; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::Duration; use anyhow::Context; -use slog::{Discard, Logger, o}; use tempfile::TempDir; use tokio::{ fs::File, @@ -135,7 +133,7 @@ async fn discover_port(logfile: String) -> Result { let timeout = Instant::now() + MGD_TIMEOUT; tokio::time::timeout_at(timeout, find_mgd_port_in_log(logfile)) .await - .context("time out while discovering mgd port number")? + .context("time out while discovering port number")? } async fn find_mgd_port_in_log(logfile: String) -> Result { @@ -165,91 +163,115 @@ async fn find_mgd_port_in_log(logfile: String) -> Result { } } -/// In-process stand-in for the `ddmd` (Delay Driven Multipath daemon) -/// admin API. +/// Test fixture that spawns and supervises a legit `ddmd` subprocess. /// -/// `ddmd` runs in sled global zones and switch zones in real deployments, -/// and depends on illumos networking facilities not available in a generic -/// dev test toolchain the way `mgd` is. This binds a dropshot server on an -/// auto-assigned port so the test suite has a real socket to publish in -/// internal DNS as `ServiceName::Ddm`. +/// Owns a `tokio::process::Child` and a tempdir; discovers the bound admin +/// port by scraping dropshot's startup `local_addr` records; kills the child +/// on `cleanup`/`Drop`. Mirrors `MgdInstance`. /// -/// This currently has no registered routes. Any integration needing -/// concrete endpoints (e.g., peer lists) must extend the `ApiDescription`. +/// `ddmd` runs in sled global zones and switch zones in production. Spawned +/// here with `--no-state-machine`, which serves only the admin API and skips +/// the discovery / exchange / routing daemons that need real network +/// interfaces and illumos-only kernel facilities. Only switch-zone instances +/// are registered in internal DNS as `ServiceName::Ddm`; sled-global-zone +/// instances are accessed locally by their own host (RSS, sled-agent's +/// prefix advertisement, etc.) and don't need DNS publication. pub struct DdmInstance { + /// Port number the ddmd instance is listening on. pub port: u16, - server: Option>, + /// Arguments provided to the `ddmd` cli command. + pub args: Vec, + /// Child process spawned by running `ddmd`. + pub child: Option, + /// Temporary directory where logging output and other files generated by + /// `ddmd` are stored. + pub data_dir: Option, } impl DdmInstance { - /// Start a DDM sim server bound to a random localhost port. + /// Start a `ddmd` instance with `--no-state-machine`, bound to an + /// auto-assigned admin port on localhost. pub async fn start() -> Result { - let dropshot_config = dropshot::ConfigDropshot { - bind_address: SocketAddr::V6(SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - )), - ..Default::default() - }; - - let api: dropshot::ApiDescription<()> = dropshot::ApiDescription::new(); - let log = Logger::root(Discard, o!()); - - let server = dropshot::ServerBuilder::new(api, (), log) - .config(dropshot_config) - .start() - .context("failed to start DDM sim server")?; - - let port = server.local_addr().port(); - Ok(Self { port, server: Some(server) }) + let temp_dir = TempDir::new()?; + + let args = vec![ + "--admin-addr".to_string(), + "::1".into(), + "--admin-port".into(), + "0".into(), + "--no-state-machine".into(), + "--data-dir".into(), + temp_dir.path().display().to_string(), + ]; + + let child = tokio::process::Command::new("ddmd") + .args(&args) + .stdin(Stdio::null()) + .stdout(Stdio::from(redirect_file(temp_dir.path(), "ddmd_stdout")?)) + .stderr(Stdio::from(redirect_file(temp_dir.path(), "ddmd_stderr")?)) + .spawn() + .with_context(|| { + format!("failed to spawn `ddmd` (with args: {:?})", &args) + })?; + + let child = Some(child); + + let temp_dir = temp_dir.keep(); + let port = + discover_port(temp_dir.join("ddmd_stdout").display().to_string()) + .await + .with_context(|| { + format!( + "failed to discover ddmd port from files in {}", + temp_dir.display() + ) + })?; + + Ok(Self { port, args, child, data_dir: Some(temp_dir) }) } - pub async fn cleanup(&mut self) { - if let Some(server) = self.server.take() { - server.close().await.expect("failed to close DDM sim server"); + pub async fn cleanup(&mut self) -> Result<(), anyhow::Error> { + if let Some(mut child) = self.child.take() { + child.start_kill().context("Sending SIGKILL to child")?; + child.wait().await.context("waiting for child")?; + } + if let Some(dir) = self.data_dir.take() { + std::fs::remove_dir_all(&dir).with_context(|| { + format!("cleaning up temporary directory {}", dir.display()) + })?; } + Ok(()) } } impl Drop for DdmInstance { fn drop(&mut self) { - if self.server.is_some() { + if self.child.is_some() || self.data_dir.is_some() { eprintln!( "WARN: dropped DdmInstance without cleaning it up first \ - (the dropshot server's tokio task may still be running)" + (there may still be a child process running and a \ + temporary directory leaked)" ); + if let Some(child) = self.child.as_mut() { + let _ = child.start_kill(); + } + if let Some(path) = self.data_dir.take() { + eprintln!( + "WARN: ddmd temporary directory leaked: {}", + path.display() + ); + } } } } #[cfg(test)] mod tests { - use super::DdmInstance; use super::find_mgd_port_in_log; use std::io::Write; use std::process::Stdio; use tempfile::NamedTempFile; - /// Smoke-test `DdmInstance`. We bind and serve a 404 for an unregistered - /// route, then shut down cleanly. - #[tokio::test] - async fn test_ddm_sim_binds_and_serves_404() { - let mut sim = DdmInstance::start().await.expect("DDM sim starts"); - assert!(sim.port > 0, "DDM sim should auto-assign a port"); - - let url = format!("http://[::1]:{}/peers", sim.port); - let resp = reqwest::get(&url).await.expect("server reachable"); - assert_eq!( - resp.status(), - reqwest::StatusCode::NOT_FOUND, - "no routes registered yet: expected 404" - ); - - sim.cleanup().await; - } - const EXPECTED_PORT: u16 = 4676; #[tokio::test] @@ -263,6 +285,16 @@ mod tests { .expect("Cannot find 'mgd' on PATH. Refer to README.md for installation instructions"); } + #[tokio::test] + async fn test_ddmd_in_path() { + tokio::process::Command::new("ddmd") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("Cannot find 'ddmd' on PATH. Refer to README.md for installation instructions"); + } + #[tokio::test] async fn test_discover_local_listening_port() { // Write some data to a fake log file diff --git a/tools/install_builder_prerequisites.sh b/tools/install_builder_prerequisites.sh index 0f1df7d2528..d79a923ca1f 100755 --- a/tools/install_builder_prerequisites.sh +++ b/tools/install_builder_prerequisites.sh @@ -230,6 +230,7 @@ retry xtask download \ console \ dendrite-stub \ maghemite-mgd \ + maghemite-ddmd \ transceiver-control # Validate the PATH: diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 060b3a13efb..b3bbfdd3016 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="7696ee48d5ee29a917dea459e281fe2e8ff20513" +COMMIT="3b54e1630e6f75cdc09a2580cec2438bfecade22" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 060b3a13efb..b3bbfdd3016 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="7696ee48d5ee29a917dea459e281fe2e8ff20513" +COMMIT="3b54e1630e6f75cdc09a2580cec2438bfecade22" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 470facaa671..de6542b8668 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,4 @@ -CIDL_SHA256="301d31ca481e4822f69484feacca31dd08a7c4aae87d96641d384bda3178d2f3" -MGD_LINUX_SHA256="95f9759a5fde2784d148c81df2218d29adde1d27fb72d5dbcf534de6450f0f7c" \ No newline at end of file +CIDL_SHA256="69f6bc36806b799174897762f0b10885d600a747018d1f9dbf2caeae9c749841" +MGD_LINUX_SHA256="aa2d1cda4a4d75f403856921abdd6cdffcf9c379f013b767be682b5ee1f32cea" +MG_DDM_SHA256="020a85457a5d868fb0078e9b29b7b7d126ca494408a87eaccbff5a2b4ef3c628" +DDMD_LINUX_SHA256="6bfdd936c4819bfb7b14db5968686051acb47d1aded0c819eca65fea2fefab75" \ No newline at end of file diff --git a/tools/update_maghemite.sh b/tools/update_maghemite.sh index 0051397b51d..0a482cb4440 100755 --- a/tools/update_maghemite.sh +++ b/tools/update_maghemite.sh @@ -54,6 +54,12 @@ function update_mgd { SHA_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "mgd" "linux") OUTPUT_LINUX=$(printf "MGD_LINUX_SHA256=\"%s\"\n" "$SHA_LINUX") + SHA_MG_DDM=$(get_sha "$REPO" "$TARGET_COMMIT" "mg-ddm" "image") + OUTPUT_MG_DDM=$(printf "MG_DDM_SHA256=\"%s\"\n" "$SHA_MG_DDM") + + SHA_DDMD_LINUX=$(get_sha "$REPO" "$TARGET_COMMIT" "ddmd" "linux") + OUTPUT_DDMD_LINUX=$(printf "DDMD_LINUX_SHA256=\"%s\"\n" "$SHA_DDMD_LINUX") + if [ -n "$DRY_RUN" ]; then MGD_PATH="/dev/null" else @@ -61,7 +67,7 @@ function update_mgd { fi echo "Updating Maghemite mgd from: $TARGET_COMMIT" set -x - printf "$OUTPUT\n$OUTPUT_LINUX" > $MGD_PATH + printf "$OUTPUT\n$OUTPUT_LINUX\n$OUTPUT_MG_DDM\n$OUTPUT_DDMD_LINUX" > $MGD_PATH set +x } From 982443676f40519d592fa3fe1a2a521d667eace1 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 8 May 2026 01:14:44 +0000 Subject: [PATCH 3/3] [deps] maghemite update --- Cargo.lock | 6 +++--- Cargo.toml | 6 +++--- package-manifest.toml | 10 +++++----- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bafba810445..9727708fc0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2525,7 +2525,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" +source = "git+https://github.com/oxidecomputer/maghemite?rev=974423895c17cc23711732f518e447b284425ccd#974423895c17cc23711732f518e447b284425ccd" dependencies = [ "oxnet", "progenitor 0.14.0", @@ -6509,7 +6509,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" +source = "git+https://github.com/oxidecomputer/maghemite?rev=974423895c17cc23711732f518e447b284425ccd#974423895c17cc23711732f518e447b284425ccd" dependencies = [ "chrono", "colored 3.1.1", @@ -11845,7 +11845,7 @@ dependencies = [ [[package]] name = "rdb-types" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=3b54e1630e6f75cdc09a2580cec2438bfecade22#3b54e1630e6f75cdc09a2580cec2438bfecade22" +source = "git+https://github.com/oxidecomputer/maghemite?rev=974423895c17cc23711732f518e447b284425ccd#974423895c17cc23711732f518e447b284425ccd" dependencies = [ "oxnet", "schemars 0.8.22", diff --git a/Cargo.toml b/Cargo.toml index 9d27de57639..3f2059ad6a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -604,8 +604,8 @@ ntp-admin-api = { path = "ntp-admin/api" } ntp-admin-client = { path = "clients/ntp-admin-client" } ntp-admin-types = { path = "ntp-admin/types" } ntp-admin-types-versions = { path = "ntp-admin/types/versions" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "974423895c17cc23711732f518e447b284425ccd" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "974423895c17cc23711732f518e447b284425ccd" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -742,7 +742,7 @@ rats-corim = { git = "https://github.com/oxidecomputer/rats-corim.git", rev = "f raw-cpuid = { git = "https://github.com/oxidecomputer/rust-cpuid.git", rev = "a4cf01df76f35430ff5d39dc2fe470bcb953503b" } rayon = "1.10" rcgen = "0.12.1" -rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "3b54e1630e6f75cdc09a2580cec2438bfecade22" } +rdb-types = { git = "https://github.com/oxidecomputer/maghemite", rev = "974423895c17cc23711732f518e447b284425ccd" } reconfigurator-cli = { path = "dev-tools/reconfigurator-cli" } reedline = "0.40.0" ref-cast = "1.0" diff --git a/package-manifest.toml b/package-manifest.toml index daf5ea0bfce..aa2a0c297c1 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -683,10 +683,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" +source.commit = "974423895c17cc23711732f518e447b284425ccd" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "c26d680b8d01210cee8f081fad24cb0b3aeb03be509b76ca04a99f38f3ea9121" +source.sha256 = "eed4c89343c29b42a74b16d74186c4e3c1a78701b0398ec2b81206122e4317d1" output.type = "tarball" [package.mg-ddm] @@ -699,10 +699,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" +source.commit = "974423895c17cc23711732f518e447b284425ccd" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "020a85457a5d868fb0078e9b29b7b7d126ca494408a87eaccbff5a2b4ef3c628" +source.sha256 = "ea97c636761cf7f622ddf0382ab365e68973604809eb6ebc93a0bbb94f758030" output.type = "zone" output.intermediate_only = true @@ -714,7 +714,7 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "3b54e1630e6f75cdc09a2580cec2438bfecade22" +source.commit = "974423895c17cc23711732f518e447b284425ccd" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt source.sha256 = "69f6bc36806b799174897762f0b10885d600a747018d1f9dbf2caeae9c749841" diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index b3bbfdd3016..cd2803a6591 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1 +1 @@ -COMMIT="3b54e1630e6f75cdc09a2580cec2438bfecade22" +COMMIT="974423895c17cc23711732f518e447b284425ccd" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index b3bbfdd3016..cd2803a6591 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1 +1 @@ -COMMIT="3b54e1630e6f75cdc09a2580cec2438bfecade22" +COMMIT="974423895c17cc23711732f518e447b284425ccd" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index de6542b8668..69aae8a4144 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,4 +1,4 @@ CIDL_SHA256="69f6bc36806b799174897762f0b10885d600a747018d1f9dbf2caeae9c749841" MGD_LINUX_SHA256="aa2d1cda4a4d75f403856921abdd6cdffcf9c379f013b767be682b5ee1f32cea" -MG_DDM_SHA256="020a85457a5d868fb0078e9b29b7b7d126ca494408a87eaccbff5a2b4ef3c628" -DDMD_LINUX_SHA256="6bfdd936c4819bfb7b14db5968686051acb47d1aded0c819eca65fea2fefab75" \ No newline at end of file +MG_DDM_SHA256="ea97c636761cf7f622ddf0382ab365e68973604809eb6ebc93a0bbb94f758030" +DDMD_LINUX_SHA256="25bee3739280df195949c36d402c43a3d76c5a47f92395009e2c2b0e6413d671" \ No newline at end of file