diff --git a/.github/buildomat/jobs/linux.sh b/.github/buildomat/jobs/linux.sh index 8fc9e7ee..5a21404a 100755 --- a/.github/buildomat/jobs/linux.sh +++ b/.github/buildomat/jobs/linux.sh @@ -28,6 +28,26 @@ #: series = "linux" #: name = "mgadm.sha256.txt" #: from_output = "/work/release/mgadm.sha256.txt" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmd" +#: from_output = "/work/release/ddmd" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmd.sha256.txt" +#: from_output = "/work/release/ddmd.sha256.txt" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmadm" +#: from_output = "/work/release/ddmadm" +#: +#: [[publish]] +#: series = "linux" +#: name = "ddmadm.sha256.txt" +#: from_output = "/work/release/ddmadm.sha256.txt" set -o errexit set -o pipefail @@ -64,3 +84,21 @@ popd cp target/debug/mgadm /work/debug cp target/release/mgadm /work/release digest /work/release/mgadm > /work/release/mgadm.sha256.txt + +banner "ddmd" +pushd ddmd +cargo build --bin ddmd --no-default-features +cargo build --bin ddmd --no-default-features --release +popd +cp target/debug/ddmd /work/debug +cp target/release/ddmd /work/release +digest /work/release/ddmd > /work/release/ddmd.sha256.txt + +banner "ddmadm" +pushd ddmadm +cargo build --bin ddmadm +cargo build --bin ddmadm --release +popd +cp target/debug/ddmadm /work/debug +cp target/release/ddmadm /work/release +digest /work/release/ddmadm > /work/release/ddmadm.sha256.txt diff --git a/ddm/Cargo.toml b/ddm/Cargo.toml index f505632d..06728e90 100644 --- a/ddm/Cargo.toml +++ b/ddm/Cargo.toml @@ -21,10 +21,6 @@ hyper.workspace = true hyper-util.workspace = true http-body-util.workspace = true serde_json.workspace = true -libnet.workspace = true -dpd-client.workspace = true -opte-ioctl.workspace = true -oxide-vpc.workspace = true sled.workspace = true mg-common.workspace = true chrono.workspace = true @@ -35,3 +31,15 @@ oxnet.workspace = true uuid.workspace = true ddm-api.workspace = true ddm-types.workspace = true + +# illumos-only deps used by the routing state machine and platform sys layer. +# Gated by the `state-machine` feature so stub builds (e.g. Linux test +# fixtures running ddmd with `--no-state-machine`) link cleanly. +libnet = { workspace = true, optional = true } +dpd-client = { workspace = true, optional = true } +opte-ioctl = { workspace = true, optional = true } +oxide-vpc = { workspace = true, optional = true } + +[features] +default = ["state-machine"] +state-machine = ["dep:libnet", "dep:dpd-client", "dep:opte-ioctl", "dep:oxide-vpc"] diff --git a/ddm/src/admin.rs b/ddm/src/admin.rs index 6d49a368..e2d9a2a5 100644 --- a/ddm/src/admin.rs +++ b/ddm/src/admin.rs @@ -12,8 +12,6 @@ use ddm_types::exchange::PathVector; use dropshot::ApiDescription; use dropshot::ApiDescriptionBuildErrors; use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; -use dropshot::ConfigLoggingLevel; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -23,7 +21,7 @@ use dropshot::TypedBody; use mg_common::lock; use mg_common::net::TunnelOrigin; use oxnet::Ipv6Net; -use slog::{Logger, error, info}; +use slog::{Logger, error, info, o}; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::sync::Arc; @@ -35,6 +33,8 @@ use tokio::task::JoinHandle; pub const DDM_STATS_PORT: u16 = 8001; +const UNIT_API_SERVER: &str = "api_server"; + #[derive(Default)] pub struct RouterStats { pub originated_underlay_prefixes: AtomicU64, @@ -68,11 +68,11 @@ pub fn handler( ..Default::default() }; - let ds_log = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Error, - } - .to_logger("admin") - .map_err(|e| e.to_string())?; + let ds_log = log.new(o!( + "component" => crate::COMPONENT_DDM, + "module" => crate::MOD_ADMIN, + "unit" => UNIT_API_SERVER, + )); let api = api_description().map_err(|e| e.to_string())?; diff --git a/ddm/src/discovery/mod.rs b/ddm/src/discovery/mod.rs new file mode 100644 index 00000000..df770569 --- /dev/null +++ b/ddm/src/discovery/mod.rs @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This module implements the ddm router discovery mechanisms. These +//! mechanisms are responsible for three primary things +//! +//! 1. Soliciting other routers through UDP/IPv6 link local multicast. +//! 2. Sending out router advertisements in response to solicitations. +//! 3. Continuously soliciting link-local at a configurable rate to keep +//! sessions alive and sending out notifications when peering arrangements +//! expire due to not getting a solicitation response within a configurable +//! time threshold. +//! +//! [`Version`] and [`DiscoveryError`] are platform-agnostic and stay in this +//! module so the state machine type definitions in [`crate::sm`] continue to +//! compile when the routing runtime is gated out (e.g. Linux test fixtures +//! running ddmd with `--no-state-machine`). The runtime helpers that drive +//! the protocol over UDPv6 sockets live in the [`runtime`] submodule and +//! are illumos-only. +//! +//! ## Protocol +//! +//! The general sequence of events is depicted in the following diagram. +//! +//! *==========* *==========* +//! | violin | | piano | +//! *==========* *==========* +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | advertise(fe80::47) | +//! |<--------------------------| +//! | | +//! | ... | +//! | | +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | advertise(fe80::47) | +//! |<--------------------------| +//! | | +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | solicit(ff02::dd) | +//! |-------------------------->| +//! | | +//! +----| | +//! expire | | | +//! piano | | | +//! +--->| | +//! +//! This shows violin sending a link-local multicast solicitation over the wire. +//! That solicitation is received by piano and piano respons with an +//! advertisement to violin's link-local unicast address. From this point +//! forward solicitations and responses continue. Each time violin gets a +//! response from piano, it updates the last seen timestamp for piano. If at +//! some point piano stops responding to solicitations and the last seen +//! timestamp is older than the expiration threshold, violin will expire the +//! session and send out a notification to the ddm state machine that started +//! it. Violin will continue to send out solicitations in case piano comes back. +//! +//! In the event that piano undergoes renumbering e.g. it's link-local unicast +//! address changes, this will be detected by violin and an advertisement update +//! will be sent to the ddm state machine through the notification channel +//! provided to the discovery subsystem. +//! +//! The DDM discovery multicast address is ff02::dd. Discovery packets are sent +//! over UDP using port number 0xddd. +//! +//! ## Packets +//! +//! Discovery packets follow a very simple format +//! +//! 1 2 3 +//! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! | version |S A r r r r r r| router kind | hostname len | +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! | hostname : +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! : .... : +//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +//! +//! The first byte indicates the version. The only valid version at present is +//! version 1. The second byte is a flags bitfield. The first position `S` +//! indicates a solicitation. The second position `A` indicates and +//! advertisement. All other positions are reserved for future use. The third +//! byte indicates the kind of router. Current values are 0 for a server router +//! and 1 for a transit routers. The fourth byte is a hostname length followed +//! directly by a hostname of up to 255 bytes in length. + +use thiserror::Error; + +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +mod runtime; + +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +pub(crate) use runtime::handler; + +#[derive(Debug, Copy, Clone)] +#[repr(u8)] +pub enum Version { + V2 = 2, + V3 = 3, +} + +#[derive(Error, Debug)] +pub enum DiscoveryError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("serialization error: {0}")] + Serialization(#[from] ispf::Error), +} diff --git a/ddm/src/discovery.rs b/ddm/src/discovery/runtime.rs similarity index 73% rename from ddm/src/discovery.rs rename to ddm/src/discovery/runtime.rs index fc4a84e8..7e326877 100644 --- a/ddm/src/discovery.rs +++ b/ddm/src/discovery/runtime.rs @@ -2,92 +2,14 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! This file implements the ddm router discovery mechanisms. These mechanisms -//! are responsible for three primary things -//! -//! 1. Soliciting other routers through UDP/IPv6 link local multicast. -//! 2. Sending out router advertisements in response to solicitations. -//! 3. Continuously soliciting link-local at a configurable rate to keep -//! sessions alive and sending out notifications when peering arrangements -//! expire due to not getting a solicitation response within a configurable -//! time threshold. -//! -//! ## Protocol -//! -//! The general sequence of events is depicted in the following diagram. -//! -//! *==========* *==========* -//! | violin | | piano | -//! *==========* *==========* -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | advertise(fe80::47) | -//! |<--------------------------| -//! | | -//! | ... | -//! | | -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | advertise(fe80::47) | -//! |<--------------------------| -//! | | -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | solicit(ff02::dd) | -//! |-------------------------->| -//! | | -//! +----| | -//! expire | | | -//! piano | | | -//! +--->| | -//! -//! This shows violin sending a link-local multicast solicitation over the wire. -//! That solicitation is received by piano and piano respons with an -//! advertisement to violin's link-local unicast address. From this point -//! forward solicitations and responses continue. Each time violin gets a -//! response from piano, it updates the last seen timestamp for piano. If at -//! some point piano stops responding to solicitations and the last seen -//! timestamp is older than the expiration threshold, violin will expire the -//! session and send out a notification to the ddm state machine that started -//! it. Violin will continue to send out solicitations in case piano comes back. -//! -//! In the event that piano undergoes renumbering e.g. it's link-local unicast -//! address changes, this will be detected by violin and an advertisement update -//! will be sent to the ddm state machine through the notification channel -//! provided to the discovery subsystem. -//! -//! The DDM discovery multicast address is ff02::dd. Discovery packets are sent -//! over UDP using port number 0xddd. -//! -//! ## Packets -//! -//! Discovery packets follow a very simple format -//! -//! 1 2 3 -//! 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | version |S A r r r r r r| router kind | hostname len | -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! | hostname : -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! : .... : -//! +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -//! -//! The first byte indicates the version. The only valid version at present is -//! version 1. The second byte is a flags bitfield. The first position `S` -//! indicates a solicitation. The second position `A` indicates and -//! advertisement. All other positions are reserved for future use. The third -//! byte indicates the kind of router. Current values are 0 for a server router -//! and 1 for a transit routers. The fourth byte is a hostname length followed -//! directly by a hostname of up to 255 bytes in length. +//! Runtime helpers for ddm router discovery: link-local UDPv6 sockets, +//! solicitation/advertisement loops, neighbor liveness, and the +//! [`handler`] entry point invoked by the routing state machine. +//! illumos-only. +use super::{DiscoveryError, Version}; use crate::db::Db; use crate::sm::{Config, Event, NeighborEvent, SessionStats}; -use crate::util::u8_slice_assume_init_ref; use crate::{dbg, err, inf, trc, wrn}; use ddm_types::db::{PeerInfo, PeerStatus, RouterKind}; use mg_common::lock; @@ -101,31 +23,22 @@ use std::sync::mpsc::Sender; use std::sync::{Arc, RwLock}; use std::thread::{sleep, spawn}; use std::time::{Duration, Instant}; -use thiserror::Error; const DDM_MADDR: Ipv6Addr = Ipv6Addr::new(0xff02, 0, 0, 0, 0, 0, 0, 0xdd); const DDM_PORT: u16 = 0xddd; const SOLICIT: u8 = 1; const ADVERTISE: u8 = 1 << 1; -#[derive(Debug, Copy, Clone)] -#[repr(u8)] -pub enum Version { - V2 = 2, - V3 = 3, -} - -#[derive(Error, Debug)] -pub enum DiscoveryError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("serialization error: {0}")] - Serialization(#[from] ispf::Error), +/// Reinterpret an initialized prefix of `[MaybeUninit]` as `[u8]`. +/// +/// TODO: trade for `MaybeUninit::slice_assume_init_ref` when it stabilizes. +#[inline(always)] +const unsafe fn u8_slice_assume_init_ref(slice: &[MaybeUninit]) -> &[u8] { + unsafe { &*(slice as *const [MaybeUninit] as *const [u8]) } } #[derive(Debug, Serialize, Deserialize)] -pub struct DiscoveryPacket { +struct DiscoveryPacket { version: u8, flags: u8, kind: RouterKind, @@ -134,7 +47,7 @@ pub struct DiscoveryPacket { } impl DiscoveryPacket { - pub fn new_solicitation(hostname: String, kind: RouterKind) -> Self { + fn new_solicitation(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, flags: SOLICIT, @@ -142,7 +55,7 @@ impl DiscoveryPacket { kind, } } - pub fn new_advertisement(hostname: String, kind: RouterKind) -> Self { + fn new_advertisement(hostname: String, kind: RouterKind) -> Self { Self { version: Version::V2 as u8, flags: ADVERTISE, @@ -150,18 +63,12 @@ impl DiscoveryPacket { kind, } } - pub fn is_solicitation(&self) -> bool { + fn is_solicitation(&self) -> bool { (self.flags & SOLICIT) != 0 } - pub fn is_advertisement(&self) -> bool { + fn is_advertisement(&self) -> bool { (self.flags & ADVERTISE) != 0 } - pub fn set_solicitation(&mut self) { - self.flags &= SOLICIT; - } - pub fn set_advertisement(&mut self) { - self.flags &= ADVERTISE; - } } #[derive(Clone)] diff --git a/ddm/src/exchange/mod.rs b/ddm/src/exchange/mod.rs new file mode 100644 index 00000000..02a5b264 --- /dev/null +++ b/ddm/src/exchange/mod.rs @@ -0,0 +1,325 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! This module implements the ddm router prefix exchange mechanisms. These +//! mechanisms are responsible for announcing and withdrawing prefix sets to +//! and from peers. +//! +//! The module has a set of request initiators and request handlers for +//! announcing, withdrawing, and synchronizing routes with a given peer. +//! Communication between peers is over HTTP(s) requests. +//! +//! This module only contains basic mechanisms for prefix information exchange +//! with peers. How those mechanisms are used in the overall state machine +//! model of a ddm router is defined in the state machine implementation in +//! [`crate::sm`]. +//! +//! The wire types ([`Update`], [`UnderlayUpdate`], [`TunnelUpdate`], and +//! their versioned counterparts) are platform-agnostic and stay in this +//! module. The runtime helpers that drive the HTTP exchange protocol and +//! program forwarding state live in the [`runtime`] submodule and are +//! illumos-only, since they call into [`crate::sys`] to install routes. + +use ddm_types::exchange::{PathVector, PathVectorV2}; +use mg_common::net::{TunnelOrigin, TunnelOriginV2}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use thiserror::Error; + +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +mod runtime; + +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +pub(crate) use runtime::{ + announce_tunnel, announce_underlay, do_pull, handler, pull, + withdraw_tunnel, withdraw_underlay, +}; + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 1. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV1 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV1 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UpdateV2 { + pub underlay: Option, + pub tunnel: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct Update { + pub underlay: Option, + pub tunnel: Option, +} + +impl From for Update { + fn from(value: UpdateV1) -> Self { + Update { + tunnel: None, + underlay: Some(UnderlayUpdate { + announce: value.announce, + withdraw: value.withdraw, + }), + } + } +} + +impl From for Update { + fn from(value: UpdateV2) -> Self { + Update { + tunnel: value.tunnel.map(TunnelUpdate::from), + underlay: value.underlay.map(UnderlayUpdate::from), + } + } +} + +impl From for UpdateV1 { + fn from(value: Update) -> Self { + let (announce, withdraw) = match value.underlay { + Some(underlay) => (underlay.announce, underlay.withdraw), + None => (HashSet::new(), HashSet::new()), + }; + UpdateV1 { announce, withdraw } + } +} + +impl From for UpdateV2 { + fn from(value: Update) -> Self { + UpdateV2 { + tunnel: value.tunnel.map(TunnelUpdateV2::from), + underlay: value.underlay.map(UnderlayUpdateV2::from), + } + } +} + +impl From for Update { + fn from(u: UnderlayUpdate) -> Self { + Update { + underlay: Some(u), + tunnel: None, + } + } +} + +impl From for Update { + fn from(t: TunnelUpdate) -> Self { + Update { + underlay: None, + tunnel: Some(t), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponse { + pub underlay: Option>, + pub tunnel: Option>, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct PullResponseV2 { + pub underlay: Option>, + pub tunnel: Option>, +} + +impl From for PullResponse { + fn from(value: PullResponseV2) -> Self { + PullResponse { + underlay: value + .underlay + .map(|x| x.into_iter().map(PathVector::from).collect()), + tunnel: value + .tunnel + .map(|x| x.into_iter().map(TunnelOrigin::from).collect()), + } + } +} + +impl From> for PullResponse { + fn from(value: HashSet) -> Self { + PullResponse { + underlay: Some(value), + tunnel: None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UnderlayUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct UnderlayUpdateV2 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl From for UnderlayUpdateV2 { + fn from(value: UnderlayUpdate) -> Self { + UnderlayUpdateV2 { + announce: value + .announce + .into_iter() + .map(PathVectorV2::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(PathVectorV2::from) + .collect(), + } + } +} + +impl From for UnderlayUpdate { + fn from(value: UnderlayUpdateV2) -> Self { + UnderlayUpdate { + announce: value + .announce + .into_iter() + .map(PathVector::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(PathVector::from) + .collect(), + } + } +} + +impl UnderlayUpdate { + pub fn announce(prefixes: HashSet) -> Self { + Self { + announce: prefixes, + ..Default::default() + } + } + pub fn withdraw(prefixes: HashSet) -> Self { + Self { + withdraw: prefixes, + ..Default::default() + } + } + pub fn with_path_element(&self, element: String) -> Self { + Self { + announce: self + .announce + .iter() + .map(|x| { + let mut pv = x.clone(); + pv.path.push(element.clone()); + pv + }) + .collect(), + withdraw: self + .withdraw + .iter() + .map(|x| { + let mut pv = x.clone(); + pv.path.push(element.clone()); + pv + }) + .collect(), + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct TunnelUpdate { + pub announce: HashSet, + pub withdraw: HashSet, +} + +/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE +/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS +/// DEFINITION SHALL NEVER CHANGE. +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub struct TunnelUpdateV2 { + pub announce: HashSet, + pub withdraw: HashSet, +} + +impl From for TunnelUpdate { + fn from(value: TunnelUpdateV2) -> Self { + TunnelUpdate { + announce: value + .announce + .into_iter() + .map(TunnelOrigin::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(TunnelOrigin::from) + .collect(), + } + } +} + +impl From for TunnelUpdateV2 { + fn from(value: TunnelUpdate) -> Self { + TunnelUpdateV2 { + announce: value + .announce + .into_iter() + .map(TunnelOriginV2::from) + .collect(), + withdraw: value + .withdraw + .into_iter() + .map(TunnelOriginV2::from) + .collect(), + } + } +} + +impl TunnelUpdate { + pub fn announce(prefixes: HashSet) -> Self { + Self { + announce: prefixes, + ..Default::default() + } + } + pub fn withdraw(prefixes: HashSet) -> Self { + Self { + withdraw: prefixes, + ..Default::default() + } + } +} + +#[derive(Error, Debug)] +pub enum ExchangeError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("hyper error: {0}")] + Hyper(#[from] hyper::Error), + + #[error("hyper client error: {0}")] + HyperClient(#[from] hyper_util::client::legacy::Error), + + #[error("timeout error: {0}")] + Timeout(#[from] tokio::time::error::Elapsed), + + #[error("json error: {0}")] + SerdeJson(#[from] serde_json::Error), +} diff --git a/ddm/src/exchange.rs b/ddm/src/exchange/runtime.rs similarity index 68% rename from ddm/src/exchange.rs rename to ddm/src/exchange/runtime.rs index 2c1cc876..a578dc9a 100644 --- a/ddm/src/exchange.rs +++ b/ddm/src/exchange/runtime.rs @@ -2,19 +2,15 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! This file implements the ddm router prefix exchange mechanisms. These -//! mechanisms are responsible for announcing and withdrawing prefix sets to and -//! from peers. -//! -//! This file has a set of request initiators and request handlers for -//! announcing, withdrawing and synchronizing routes with a a given peer. -//! Communication between peers is over HTTP(s) requests. -//! -//! This file only contains basic mechanisms for prefix information exchange -//! with peers. How those mechanisms are used in the overall state machine model -//! of a ddm router is defined in the state machine implementation in sm.rs. -//! - +//! Runtime helpers for the ddm prefix exchange protocol: HTTP push/pull +//! initiators and dropshot endpoint handlers, and the route programming +//! plumbing that drains received updates into the local DB and the +//! forwarding platform via [`crate::sys`]. illumos-only. + +use super::{ + ExchangeError, PullResponse, PullResponseV2, TunnelUpdate, UnderlayUpdate, + Update, UpdateV2, +}; use crate::db::{Route, effective_route_set}; use crate::discovery::Version; use crate::sm::{Config, Event, PeerEvent, SmContext}; @@ -23,8 +19,6 @@ use ddm_types::db::{RouterKind, TunnelRoute}; use ddm_types::exchange::{PathVector, PathVectorV2}; use dropshot::ApiDescription; use dropshot::ConfigDropshot; -use dropshot::ConfigLogging; -use dropshot::ConfigLoggingLevel; use dropshot::HttpError; use dropshot::HttpResponseOk; use dropshot::HttpResponseUpdatedNoContent; @@ -37,18 +31,17 @@ use hyper::body::Bytes; use hyper_util::client::legacy::Client; use hyper_util::rt::TokioExecutor; use mg_common::net::{TunnelOrigin, TunnelOriginV2}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use slog::Logger; +use slog::{Logger, o}; use std::collections::HashSet; use std::net::{Ipv6Addr, SocketAddrV6}; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::Duration; -use thiserror::Error; use tokio::sync::Mutex; use tokio::time::timeout; +const UNIT_EXCHANGE_SERVER: &str = "exchange_server"; + #[derive(Clone)] pub struct HandlerContext { ctx: SmContext, @@ -56,89 +49,10 @@ pub struct HandlerContext { log: Logger, } -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 1. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV1 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UpdateV1 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UpdateV2 { - pub underlay: Option, - pub tunnel: Option, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct Update { - pub underlay: Option, - pub tunnel: Option, -} - -impl From for Update { - fn from(value: UpdateV1) -> Self { - Update { - tunnel: None, - underlay: Some(UnderlayUpdate { - announce: value.announce, - withdraw: value.withdraw, - }), - } - } -} - -impl From for Update { - fn from(value: UpdateV2) -> Self { - Update { - tunnel: value.tunnel.map(TunnelUpdate::from), - underlay: value.underlay.map(UnderlayUpdate::from), - } - } -} - -impl From for UpdateV1 { - fn from(value: Update) -> Self { - let (announce, withdraw) = match value.underlay { - Some(underlay) => (underlay.announce, underlay.withdraw), - None => (HashSet::new(), HashSet::new()), - }; - UpdateV1 { announce, withdraw } - } -} - -impl From for UpdateV2 { - fn from(value: Update) -> Self { - UpdateV2 { - tunnel: value.tunnel.map(TunnelUpdateV2::from), - underlay: value.underlay.map(UnderlayUpdateV2::from), - } - } -} - -impl From for Update { - fn from(u: UnderlayUpdate) -> Self { - Update { - underlay: Some(u), - tunnel: None, - } - } -} - -impl From for Update { - fn from(t: TunnelUpdate) -> Self { - Update { - underlay: None, - tunnel: Some(t), - } - } -} - impl Update { + /// Build an `Update` whose underlay/tunnel halves carry the announcements + /// from `pr`. Used by [`pull`] to project a pull response back into the + /// update event stream. fn announce(pr: PullResponse) -> Self { Self { underlay: pr.underlay.map(UnderlayUpdate::announce), @@ -147,211 +61,6 @@ impl Update { } } -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct PullResponse { - pub underlay: Option>, - pub tunnel: Option>, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct PullResponseV2 { - pub underlay: Option>, - pub tunnel: Option>, -} - -impl From for PullResponse { - fn from(value: PullResponseV2) -> Self { - PullResponse { - underlay: value - .underlay - .map(|x| x.into_iter().map(PathVector::from).collect()), - tunnel: value - .tunnel - .map(|x| x.into_iter().map(TunnelOrigin::from).collect()), - } - } -} - -impl From> for PullResponse { - fn from(value: HashSet) -> Self { - PullResponse { - underlay: Some(value), - tunnel: None, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UnderlayUpdate { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct UnderlayUpdateV2 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -impl From for UnderlayUpdateV2 { - fn from(value: UnderlayUpdate) -> Self { - UnderlayUpdateV2 { - announce: value - .announce - .into_iter() - .map(PathVectorV2::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(PathVectorV2::from) - .collect(), - } - } -} - -impl From for UnderlayUpdate { - fn from(value: UnderlayUpdateV2) -> Self { - UnderlayUpdate { - announce: value - .announce - .into_iter() - .map(PathVector::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(PathVector::from) - .collect(), - } - } -} - -impl UnderlayUpdate { - pub fn announce(prefixes: HashSet) -> Self { - Self { - announce: prefixes, - ..Default::default() - } - } - pub fn withdraw(prefixes: HashSet) -> Self { - Self { - withdraw: prefixes, - ..Default::default() - } - } - pub fn with_path_element(&self, element: String) -> Self { - Self { - announce: self - .announce - .iter() - .map(|x| { - let mut pv = x.clone(); - pv.path.push(element.clone()); - pv - }) - .collect(), - withdraw: self - .withdraw - .iter() - .map(|x| { - let mut pv = x.clone(); - pv.path.push(element.clone()); - pv - }) - .collect(), - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct TunnelUpdate { - pub announce: HashSet, - pub withdraw: HashSet, -} - -/// THIS TYPE IS FOR DDM PROTOCOL VERSION 2. IT SHALL NEVER CHANGE. THIS TYPE -/// CAN BE REMOVED WHEN DDMV2 CLIENTS AND SERVERS NO LONGER EXIST BUT ITS -/// DEFINITION SHALL NEVER CHANGE. -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -pub struct TunnelUpdateV2 { - pub announce: HashSet, - pub withdraw: HashSet, -} - -impl From for TunnelUpdate { - fn from(value: TunnelUpdateV2) -> Self { - TunnelUpdate { - announce: value - .announce - .into_iter() - .map(TunnelOrigin::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(TunnelOrigin::from) - .collect(), - } - } -} - -impl From for TunnelUpdateV2 { - fn from(value: TunnelUpdate) -> Self { - TunnelUpdateV2 { - announce: value - .announce - .into_iter() - .map(TunnelOriginV2::from) - .collect(), - withdraw: value - .withdraw - .into_iter() - .map(TunnelOriginV2::from) - .collect(), - } - } -} - -impl TunnelUpdate { - pub fn announce(prefixes: HashSet) -> Self { - Self { - announce: prefixes, - ..Default::default() - } - } - pub fn withdraw(prefixes: HashSet) -> Self { - Self { - withdraw: prefixes, - ..Default::default() - } - } -} - -#[derive(Error, Debug)] -pub enum ExchangeError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("hyper error: {0}")] - Hyper(#[from] hyper::Error), - - #[error("hyper client error: {0}")] - HyperClient(#[from] hyper_util::client::legacy::Error), - - #[error("timeout error: {0}")] - Timeout(#[from] tokio::time::error::Elapsed), - - #[error("json error: {0}")] - SerdeJson(#[from] serde_json::Error), -} - pub(crate) fn announce_underlay( ctx: &SmContext, config: Config, @@ -587,11 +296,11 @@ pub fn handler( ..Default::default() }; - let ds_log = ConfigLogging::StderrTerminal { - level: ConfigLoggingLevel::Error, - } - .to_logger("exchange") - .map_err(|e| e.to_string())?; + let ds_log = log.new(o!( + "component" => crate::COMPONENT_DDM, + "module" => crate::MOD_EXCHANGE, + "unit" => UNIT_EXCHANGE_SERVER, + )); inf!(log, ctx.config.if_name, "exchange: listening on {}", sa); diff --git a/ddm/src/lib.rs b/ddm/src/lib.rs index 447109ba..ded32792 100644 --- a/ddm/src/lib.rs +++ b/ddm/src/lib.rs @@ -8,8 +8,12 @@ pub mod discovery; pub mod exchange; pub mod oxstats; pub mod sm; +#[cfg(all(feature = "state-machine", target_os = "illumos"))] pub mod sys; -mod util; + +pub const COMPONENT_DDM: &str = "ddm"; +pub const MOD_ADMIN: &str = "admin"; +pub const MOD_EXCHANGE: &str = "exchange"; #[macro_export] macro_rules! err { diff --git a/ddm/src/sm/mod.rs b/ddm/src/sm/mod.rs new file mode 100644 index 00000000..e6903a1f --- /dev/null +++ b/ddm/src/sm/mod.rs @@ -0,0 +1,194 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! State machine type definitions and the [`StateMachine`] handle. The +//! routing state machine implementation (discovery, solicit, exchange) lives +//! in the [`state`] submodule and is illumos-only, since it programs kernel +//! routes via [`crate::sys`] and reads interface addressing through `libnet`. + +use crate::db::Db; +use crate::discovery::{self, Version}; +use crate::exchange::Update; +use ddm_types::db::RouterKind; +use mg_common::net::TunnelOrigin; +use oxnet::Ipv6Net; +use slog::Logger; +use std::collections::HashSet; +use std::net::Ipv6Addr; +use std::sync::atomic::AtomicU64; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{Arc, Mutex}; +use thiserror::Error; + +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +mod state; + +#[derive(Debug)] +pub enum AdminEvent { + /// Announce a set of IPv6 prefixes + Announce(PrefixSet), + + /// Withdraw a set of IPv6 prefixes + Withdraw(PrefixSet), + + /// Expire the peer at the specified address + Expire(Ipv6Addr), + + /// Synchronize with active peers by pulling their prefixes. + Sync, +} + +#[derive(Debug)] +pub enum PrefixSet { + Underlay(HashSet), + Tunnel(HashSet), +} + +#[derive(Debug)] +pub enum PeerEvent { + Push(Update), +} + +#[derive(Debug)] +pub enum NeighborEvent { + Advertise((Ipv6Addr, Version)), + SolicitFail, + Expire, +} + +#[derive(Debug)] +pub enum Event { + Neighbor(NeighborEvent), + Peer(PeerEvent), + Admin(AdminEvent), +} + +impl From for Event { + fn from(e: NeighborEvent) -> Self { + Self::Neighbor(e) + } +} + +impl From for Event { + fn from(e: PeerEvent) -> Self { + Self::Peer(e) + } +} + +impl From for Event { + fn from(e: AdminEvent) -> Self { + Self::Admin(e) + } +} + +#[derive(Debug)] +pub enum StateType { + Solicit, + Exchange, +} + +#[derive(Debug)] +pub enum EventError { + InvalidEvent(StateType), +} + +#[derive(Debug)] +pub enum EventResponse { + Success, + Prefixes(Vec), +} + +#[derive(Error, Debug)] +pub enum SmError { + #[error("io error: {0}")] + Io(#[from] std::io::Error), + + #[error("discovery error: {0}")] + Discovery(#[from] discovery::DiscoveryError), +} + +#[derive(Clone)] +pub struct Config { + /// Interface this state machine is associated with. + pub if_index: u32, + + /// Interface name this state machine is associated with. + pub if_name: String, + + /// Address object name the state machine uses for peering. Must correspond + /// to IPv6 link local address. + pub aobj_name: String, + + /// Link local Ipv6 address this state machine is associated with + pub addr: Ipv6Addr, + + /// How long to wait between solicitations (milliseconds). + pub solicit_interval: u64, + + /// How often to check for link failure while waiting for discovery messges. + pub discovery_read_timeout: u64, + + /// How long to wait between attempts to get an IP address for a specified + /// address object. + pub ip_addr_wait: u64, + + /// How long to wait without a solicitation response before expiring a peer + /// (milliseconds). + pub expire_threshold: u64, + + /// How long to wait for a response to exchange messages. + pub exchange_timeout: u64, + + /// The kind of router this is, server or transit. + pub kind: RouterKind, + + /// TCP port to use for prefix exchange. + pub exchange_port: u16, + + /// Dendrite dpd config + pub dpd: Option, +} + +#[derive(Clone)] +pub struct DpdConfig { + pub host: String, + pub port: u16, +} + +#[derive(Default)] +pub struct SessionStats { + // Discovery + pub solicitations_sent: AtomicU64, + pub solicitations_received: AtomicU64, + pub advertisements_sent: AtomicU64, + pub advertisements_received: AtomicU64, + pub peer_expirations: AtomicU64, + pub peer_address_changes: AtomicU64, + pub peer_established: AtomicU64, + pub peer_address: Mutex>, + + // Exchange + pub updates_sent: AtomicU64, + pub updates_received: AtomicU64, + pub imported_underlay_prefixes: AtomicU64, + pub imported_tunnel_endpoints: AtomicU64, + pub update_send_fail: AtomicU64, +} + +#[derive(Clone)] +pub struct SmContext { + pub config: Config, + pub db: Db, + pub tx: Sender, + pub event_channels: Vec>, + pub rt: Arc, + pub hostname: String, + pub stats: Arc, + pub log: Logger, +} + +pub struct StateMachine { + pub ctx: SmContext, + pub rx: Option>, +} diff --git a/ddm/src/sm.rs b/ddm/src/sm/state.rs similarity index 85% rename from ddm/src/sm.rs rename to ddm/src/sm/state.rs index 24215795..49e6191a 100644 --- a/ddm/src/sm.rs +++ b/ddm/src/sm/state.rs @@ -2,194 +2,32 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::db::Db; -use crate::discovery::Version; +//! Routing state machine implementation. The `Init` -> `Solicit` -> +//! `Exchange` lifecycle drives kernel route programming via [`crate::sys`] +//! and reads interface addressing through `libnet`. This module is +//! illumos-only. + +use super::{ + AdminEvent, Event, NeighborEvent, PeerEvent, PrefixSet, SmContext, SmError, + StateMachine, +}; use crate::exchange::{TunnelUpdate, UnderlayUpdate, Update}; use crate::{dbg, discovery, err, exchange, inf, wrn}; use ddm_types::db::RouterKind; use ddm_types::exchange::PathVector; use libnet::get_ipaddr_info; -use mg_common::net::TunnelOrigin; -use oxnet::Ipv6Net; use slog::Logger; use std::collections::HashSet; -use std::net::{IpAddr, Ipv6Addr}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::mpsc::{Receiver, Sender}; -use std::sync::{Arc, Mutex}; -use std::thread::sleep; -use std::thread::spawn; +use std::net::IpAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::Receiver; +use std::thread::{sleep, spawn}; use std::time::Duration; -use thiserror::Error; - -#[derive(Debug)] -pub enum AdminEvent { - /// Announce a set of IPv6 prefixes - Announce(PrefixSet), - - /// Withdraw a set of IPv6 prefixes - Withdraw(PrefixSet), - - /// Expire the peer at the specified address - Expire(Ipv6Addr), - - /// Synchronize with active peers by pulling their prefixes. - Sync, -} - -#[derive(Debug)] -pub enum PrefixSet { - Underlay(HashSet), - Tunnel(HashSet), -} - -#[derive(Debug)] -pub enum PeerEvent { - Push(Update), -} - -#[derive(Debug)] -pub enum NeighborEvent { - Advertise((Ipv6Addr, Version)), - SolicitFail, - Expire, -} - -#[derive(Debug)] -pub enum Event { - Neighbor(NeighborEvent), - Peer(PeerEvent), - Admin(AdminEvent), -} - -impl From for Event { - fn from(e: NeighborEvent) -> Self { - Self::Neighbor(e) - } -} - -impl From for Event { - fn from(e: PeerEvent) -> Self { - Self::Peer(e) - } -} - -impl From for Event { - fn from(e: AdminEvent) -> Self { - Self::Admin(e) - } -} - -#[derive(Debug)] -pub enum StateType { - Solicit, - Exchange, -} - -#[derive(Debug)] -pub enum EventError { - InvalidEvent(StateType), -} -#[derive(Debug)] -pub enum EventResponse { - Success, - Prefixes(Vec), -} - -#[derive(Error, Debug)] -pub enum SmError { - #[error("io error: {0}")] - Io(#[from] std::io::Error), - - #[error("discovery error: {0}")] - Discovery(#[from] discovery::DiscoveryError), -} - -#[derive(Clone)] -pub struct Config { - /// Interface this state machine is associated with. - pub if_index: u32, - - /// Interface name this state machine is associated with. - pub if_name: String, - - /// Address object name the state machine uses for peering. Must correspond - /// to IPv6 link local address. - pub aobj_name: String, - - /// Link local Ipv6 address this state machine is associated with - pub addr: Ipv6Addr, - - /// How long to wait between solicitations (milliseconds). - pub solicit_interval: u64, - - /// How often to check for link failure while waiting for discovery messges. - pub discovery_read_timeout: u64, - - /// How long to wait between attempts to get an IP address for a specified - /// address object. - pub ip_addr_wait: u64, - - /// How long to wait without a solicitation response before expiring a peer - /// (milliseconds). - pub expire_threshold: u64, - - /// How long to wait for a response to exchange messages. - pub exchange_timeout: u64, - - /// The kind of router this is, server or transit. - pub kind: RouterKind, - - /// TCP port to use for prefix exchange. - pub exchange_port: u16, - - /// Dendrite dpd config - pub dpd: Option, -} - -#[derive(Clone)] -pub struct DpdConfig { - pub host: String, - pub port: u16, -} - -#[derive(Default)] -pub struct SessionStats { - // Discovery - pub solicitations_sent: AtomicU64, - pub solicitations_received: AtomicU64, - pub advertisements_sent: AtomicU64, - pub advertisements_received: AtomicU64, - pub peer_expirations: AtomicU64, - pub peer_address_changes: AtomicU64, - pub peer_established: AtomicU64, - pub peer_address: Mutex>, - - // Exchange - pub updates_sent: AtomicU64, - pub updates_received: AtomicU64, - pub imported_underlay_prefixes: AtomicU64, - pub imported_tunnel_endpoints: AtomicU64, - pub update_send_fail: AtomicU64, -} - -#[derive(Clone)] -pub struct SmContext { - pub config: Config, - pub db: Db, - pub tx: Sender, - pub event_channels: Vec>, - pub rt: Arc, - pub hostname: String, - pub stats: Arc, - pub log: Logger, -} - -pub struct StateMachine { - pub ctx: SmContext, - pub rx: Option>, -} +use crate::discovery::Version; +use mg_common::net::TunnelOrigin; +use std::net::Ipv6Addr; impl StateMachine { pub fn run(&mut self) -> Result<(), SmError> { @@ -369,8 +207,8 @@ impl State for Solicit { } } -pub struct Exchange { - pub peer: Ipv6Addr, +struct Exchange { + peer: Ipv6Addr, version: Version, ctx: SmContext, log: Logger, diff --git a/ddm/src/util.rs b/ddm/src/util.rs deleted file mode 100644 index f3a96c03..00000000 --- a/ddm/src/util.rs +++ /dev/null @@ -1,14 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at https://mozilla.org/MPL/2.0/. - -use std::mem::MaybeUninit; - -//TODO trade for `MaybeUninit::slice_assume_init_ref` when it becomes available -//in stable Rust. -#[inline(always)] -pub(crate) const unsafe fn u8_slice_assume_init_ref( - slice: &[MaybeUninit], -) -> &[u8] { - unsafe { &*(slice as *const [MaybeUninit] as *const [u8]) } -} diff --git a/ddmd/Cargo.toml b/ddmd/Cargo.toml index 5ec0804b..b92be304 100644 --- a/ddmd/Cargo.toml +++ b/ddmd/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -ddm = { path = "../ddm" } +ddm = { path = "../ddm", default-features = false } mg-common = { path = "../mg-common" } anyhow.workspace = true clap.workspace = true @@ -19,3 +19,7 @@ dpd-client.workspace = true anstyle.workspace = true uuid.workspace = true smf.workspace = true + +[features] +default = ["state-machine"] +state-machine = ["ddm/state-machine"] diff --git a/ddmd/src/main.rs b/ddmd/src/main.rs index d5db3f3c..2c52388e 100644 --- a/ddmd/src/main.rs +++ b/ddmd/src/main.rs @@ -6,11 +6,13 @@ use clap::Parser; use ddm::admin::{HandlerContext, RouterStats}; use ddm::db::Db; use ddm::sm::{DpdConfig, SmContext, StateMachine}; +#[cfg(all(feature = "state-machine", target_os = "illumos"))] use ddm::sys::Route; use ddm_types::db::RouterKind; use signal::handle_signals; use slog::{Drain, Logger, error}; use std::net::{IpAddr, Ipv6Addr}; +#[cfg(all(feature = "state-machine", target_os = "illumos"))] use std::sync::mpsc::channel; use std::sync::{Arc, Mutex}; use uuid::Uuid; @@ -100,6 +102,15 @@ struct Arg { /// Id of the sled this router is running on. #[arg(long)] sled_uuid: Option, + + /// Skip the routing state machine (discovery, exchange, route + /// synchronization). Only the admin API server runs, allowing test + /// fixtures to obtain a real ddmd admin endpoint without the kernel-level + /// networking the state machine requires. + /// + /// Analogous to `mgd --no-bgp-dispatcher`. + #[arg(long, default_value_t = false, conflicts_with = "addr")] + no_state_machine: bool, } #[derive(Debug, Parser, Clone)] @@ -121,11 +132,8 @@ async fn run() { .await .expect("set up refresh signal handler"); - let mut event_channels = Vec::new(); let db = Db::new(&format!("{}/ddmdb", arg.data_dir), log.clone()).unwrap(); - let mut sms = Vec::new(); - let dpd = match arg.dendrite { true => Some(DpdConfig { host: arg.dpd_host.clone(), @@ -140,54 +148,10 @@ async fn run() { .to_string_lossy() .to_string(); - for name in arg.addresses { - let (tx, rx) = channel(); - let config = ddm::sm::Config { - solicit_interval: arg.solicit_interval, - expire_threshold: arg.expire_threshold, - discovery_read_timeout: arg.discovery_read_timeout, - ip_addr_wait: arg.ip_addr_wait, - exchange_timeout: arg.exchange_timeout, - exchange_port: arg.exchange_port, - aobj_name: name.clone(), - if_name: String::new(), - if_index: 0, - kind: arg.kind, - dpd: dpd.clone(), - addr: Ipv6Addr::UNSPECIFIED, - }; - let ctx = SmContext { - config, - db: db.clone(), - event_channels: Vec::new(), - tx: tx.clone(), - log: log.clone(), - hostname: hostname.clone(), - rt: rt.clone(), - stats: Arc::new(ddm::sm::SessionStats::default()), - }; - let sm = StateMachine { ctx, rx: Some(rx) }; - sms.push(sm); - event_channels.push(tx); - } + let (sms, event_channels) = + start_state_machines(&arg, &db, &dpd, &hostname, &rt, &log); - // Add an event channel sender for each state machine to every other state - // machine. - for (i, sm) in sms.iter_mut().enumerate() { - for (j, e) in event_channels.iter().enumerate() { - // dont give a state machine an event sender to itself. - if i == j { - continue; - } - sm.ctx.event_channels.push(e.clone()); - } - } - - for sm in &mut sms { - sm.run().unwrap(); - } - - termination_handler(db.clone(), dpd, rt, log.clone()); + termination_handler(db.clone(), dpd.clone(), rt.clone(), log.clone()); let router_stats = Arc::new(RouterStats::default()); let peers: Vec = sms.iter().map(|x| x.ctx.clone()).collect(); @@ -237,6 +201,106 @@ async fn run() { std::thread::park(); } +/// Build, wire, and start the per-address routing state machines. +/// +/// Returns the running [`StateMachine`] handles plus the sender side of each +/// machine's event channel. When `--no-state-machine` is set the function +/// short-circuits to empty vectors, leaving the daemon to serve only its +/// admin API. The illumos and non-illumos variants share that early-exit +/// branch; only the actual machine setup is platform-specific. +#[cfg(all(feature = "state-machine", target_os = "illumos"))] +fn start_state_machines( + arg: &Arg, + db: &Db, + dpd: &Option, + hostname: &str, + rt: &Arc, + log: &Logger, +) -> ( + Vec, + Vec>, +) { + if arg.no_state_machine { + return (Vec::new(), Vec::new()); + } + + let mut sms = Vec::new(); + let mut event_channels = Vec::new(); + + for name in &arg.addresses { + let (tx, rx) = channel(); + + let config = ddm::sm::Config { + solicit_interval: arg.solicit_interval, + expire_threshold: arg.expire_threshold, + discovery_read_timeout: arg.discovery_read_timeout, + ip_addr_wait: arg.ip_addr_wait, + exchange_timeout: arg.exchange_timeout, + exchange_port: arg.exchange_port, + aobj_name: name.clone(), + if_name: String::new(), + if_index: 0, + kind: arg.kind, + dpd: dpd.clone(), + addr: Ipv6Addr::UNSPECIFIED, + }; + + let ctx = SmContext { + config, + db: db.clone(), + event_channels: Vec::new(), + tx: tx.clone(), + log: log.clone(), + hostname: hostname.to_string(), + rt: rt.clone(), + stats: Arc::new(ddm::sm::SessionStats::default()), + }; + + let sm = StateMachine { ctx, rx: Some(rx) }; + sms.push(sm); + event_channels.push(tx); + } + + // Add an event channel sender for each state machine to every other state + // machine. + for (i, sm) in sms.iter_mut().enumerate() { + for (j, e) in event_channels.iter().enumerate() { + // dont give a state machine an event sender to itself. + if i == j { + continue; + } + sm.ctx.event_channels.push(e.clone()); + } + } + + for sm in &mut sms { + sm.run().unwrap(); + } + + (sms, event_channels) +} + +/// Non-illumos variant: the routing state machine depends on illumos +/// kernel networking, so on every other platform the function returns +/// empty vectors and the daemon serves only its admin API. +#[cfg(not(all(feature = "state-machine", target_os = "illumos")))] +fn start_state_machines( + _arg: &Arg, + _db: &Db, + _dpd: &Option, + _hostname: &str, + _rt: &Arc, + _log: &Logger, +) -> ( + Vec, + Vec>, +) { + (Vec::new(), Vec::new()) +} + +/// Install a Ctrl-C handler that withdraws ddmd's imported routes from the +/// kernel before exiting. On non-illumos builds there are no kernel routes +/// to withdraw, so the handler just exits cleanly. fn termination_handler( db: Db, dendrite: Option, @@ -249,23 +313,30 @@ fn termination_handler( .expect("error setting termination handler"); const SIGTERM_EXIT: i32 = 130; - let imported = db.imported(); - let routes: Vec = - imported.iter().map(|x| (x.clone()).into()).collect(); - ddm::sys::remove_underlay_routes( - &log, - "shutdown-all", - &dendrite, - routes, - &rt, - ); - - let imported_tnl = db.imported_tunnel(); - if let Err(e) = - ddm::sys::remove_tunnel_routes(&log, "shutdown-all", &imported_tnl) + #[cfg(all(feature = "state-machine", target_os = "illumos"))] { - error!(log, "shutdown tunnel routes: {e}"); + let imported = db.imported(); + let routes: Vec = + imported.iter().map(|x| (x.clone()).into()).collect(); + ddm::sys::remove_underlay_routes( + &log, + "shutdown-all", + &dendrite, + routes, + &rt, + ); + + let imported_tnl = db.imported_tunnel(); + if let Err(e) = ddm::sys::remove_tunnel_routes( + &log, + "shutdown-all", + &imported_tnl, + ) { + error!(log, "shutdown tunnel routes: {e}"); + } } + #[cfg(not(all(feature = "state-machine", target_os = "illumos")))] + let _ = (db, dendrite, rt, log); std::process::exit(SIGTERM_EXIT); });