diff --git a/docs/proposals/network/README.md b/docs/proposals/network/README.md index 719ac686..71fa3d13 100644 --- a/docs/proposals/network/README.md +++ b/docs/proposals/network/README.md @@ -33,20 +33,32 @@ iptables rules for a new node. let node_id = "node"; let node_ip_addr = Ipv4Addr::from_str("10.0.0.1").unwrap(); let node_ip_cidr = Ipv4Inet::new(node_ip_addr, 24).unwrap(); +let nodes_ips = Vec::new(); -let request = SetupNodeRequest::new(node_id.to_string(), node_ip_cidr); +let request = SetupNodeRequest::new(node_id.to_string(), node_ip_cidr, nodes_ips); let response = setup_node(request).unwrap(); + println!("CNI name: {}", response.interface_name); ``` -After each node reboot, you need to reconfigure iptables running `setup_iptables` function from -`node` module. +After each node reboot, you need to reconfigure iptables running `setup_iptables` and +`add_other_nodes` functions from `node` module. ```rust let node_id = "node"; let request = SetupIptablesRequest::new(node_id.to_string()); +let other_node_id = "node2"; +let other_node_cluster_ip = Ipv4Inet::new(Ipv4Addr::from_str("10.0.1.1").unwrap(), 24).unwrap(); +let other_node_external_ip = Ipv4Addr::from_str("22.22.22.22").unwrap(); +let new_node_request = NodeRequest::new( + other_node_id.to_string(), + NodeRequest::new(other_node_cluster_ip, other_node_external_ip), +) +let nodes_ips = vec![new_node_request]; + setup_iptables(request).unwrap(); +add_other_nodes(nodes_ips).unwrap(); ``` ### Setup instance @@ -83,6 +95,38 @@ let namespace_name = get_namespace_name(instance_id.to_string()); println!("Namespace of {}: {}", instance_id, namespace_name); ``` +### Add other nodes + +When a new node join the cluster, you need to call `new_node_in_cluster` function from `node` +module. + +```rust +let node_id = "node"; +let node_cluster_ip = Ipv4Inet::new(Ipv4Addr::from_str("10.0.1.1").unwrap(), 24).unwrap(); +let node_external_ip = Ipv4Addr::from_str("22.22.22.22").unwrap(); +let new_node_request = NodeRequest::new( + node_id.to_string(), + NodeIp::new(node_cluster_ip, node_external_ip), +); + +new_node_in_cluster(new_node_request).unwrap(); +``` + +And you have to run `delete_node_in_cluster` function from `node` module when a node leave the +cluster. + +```rust +let node_id = "node"; +let node_cluster_ip = Ipv4Inet::new(Ipv4Addr::from_str("10.0.1.1").unwrap(), 24).unwrap(); +let node_external_ip = Ipv4Addr::from_str("22.22.22.22").unwrap(); +let delete_node_request = NodeRequest::new( + node_id.to_string(), + NodeIp::new(node_cluster_ip, node_external_ip), +); + +delete_node_in_cluster(delete_node_request).unwrap(); +``` + ### Clean up To delete CNI and iptables rules of a specific node, use `clean_node` function from `node` module. diff --git a/network/src/lib.rs b/network/src/lib.rs index 0ddac0dc..9c32bcc2 100644 --- a/network/src/lib.rs +++ b/network/src/lib.rs @@ -1,5 +1,6 @@ pub mod error; pub mod instance; pub mod node; +pub mod node_ip; pub mod port; pub mod utils; diff --git a/network/src/node/mod.rs b/network/src/node/mod.rs index 3f0163e4..60415d02 100644 --- a/network/src/node/mod.rs +++ b/network/src/node/mod.rs @@ -4,7 +4,7 @@ pub mod response; use crate::error::KudoNetworkError; use crate::utils::{bridge_name, run_command}; use default_net; -use request::{CleanNodeRequest, SetupIptablesRequest, SetupNodeRequest}; +use request::{CleanNodeRequest, NodeRequest, SetupIptablesRequest, SetupNodeRequest}; use response::SetupNodeResponse; /// Create a network interface and add iptables rules to make this device able to route instances @@ -31,9 +31,174 @@ pub fn setup_node(request: SetupNodeRequest) -> Result) -> Result<(), KudoNetworkError> { + for node in nodes { + new_node_in_cluster(node)?; + } + Ok(()) +} + +/// Add iptables rules when a node joins the cluster +pub fn new_node_in_cluster(request: NodeRequest) -> Result<(), KudoNetworkError> { + // Iptables rules to allow workloads to route to the node + run_command( + "iptables", + &[ + "-t", + "nat", + "-I", + "PREROUTING", + "-p", + "tcp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + run_command( + "iptables", + &[ + "-t", + "nat", + "-I", + "PREROUTING", + "-p", + "udp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + // Iptables rules to allow host machines to route to the node + run_command( + "iptables", + &[ + "-t", + "nat", + "-I", + "OUTPUT", + "-p", + "tcp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + run_command( + "iptables", + &[ + "-t", + "nat", + "-I", + "OUTPUT", + "-p", + "udp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + Ok(()) +} + +/// Remove iptables rules when a node leaves the cluster +pub fn delete_node_in_cluster(request: NodeRequest) -> Result<(), KudoNetworkError> { + run_command( + "iptables", + &[ + "-t", + "nat", + "-D", + "PREROUTING", + "-p", + "tcp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + run_command( + "iptables", + &[ + "-t", + "nat", + "-D", + "PREROUTING", + "-p", + "udp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + run_command( + "iptables", + &[ + "-t", + "nat", + "-D", + "OUTPUT", + "-p", + "tcp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + run_command( + "iptables", + &[ + "-t", + "nat", + "-D", + "OUTPUT", + "-p", + "udp", + "-d", + &request.nodes_ips.cluster_ip_cidr.to_string(), + "-j", + "DNAT", + "--to-destination", + &request.nodes_ips.external_ip_addr.to_string(), + ], + )?; + + Ok(()) +} + /// Add iptables rules to route instances traffic /// This function must be called after each reboot pub fn setup_iptables(request: SetupIptablesRequest) -> Result<(), KudoNetworkError> { diff --git a/network/src/node/request.rs b/network/src/node/request.rs index 53ad9398..415cc813 100644 --- a/network/src/node/request.rs +++ b/network/src/node/request.rs @@ -1,18 +1,23 @@ use cidr::Ipv4Inet; +use crate::node_ip::NodeIp; + // Setup pub struct SetupNodeRequest { /// Unique identifier of the node. This identifier is used to create a bridge interface pub node_id: String, /// Composed of the node ip and a mask pub node_ip_cidr: Ipv4Inet, + /// Nodes already in the cluster + pub nodes_ips: Vec, } impl SetupNodeRequest { - pub fn new(node_id: String, node_ip_cidr: Ipv4Inet) -> Self { + pub fn new(node_id: String, node_ip_cidr: Ipv4Inet, nodes_ips: Vec) -> Self { Self { node_id, node_ip_cidr, + nodes_ips, } } } @@ -27,6 +32,17 @@ impl SetupIptablesRequest { } } +pub struct NodeRequest { + pub node_id: String, + pub nodes_ips: NodeIp, +} + +impl NodeRequest { + pub fn new(node_id: String, nodes_ips: NodeIp) -> Self { + Self { node_id, nodes_ips } + } +} + // Clean up pub struct CleanNodeRequest { pub node_id: String, diff --git a/network/src/node_ip/mod.rs b/network/src/node_ip/mod.rs new file mode 100644 index 00000000..51af12fa --- /dev/null +++ b/network/src/node_ip/mod.rs @@ -0,0 +1,16 @@ +use cidr::Ipv4Inet; +use std::net::Ipv4Addr; + +pub struct NodeIp { + pub cluster_ip_cidr: Ipv4Inet, + pub external_ip_addr: Ipv4Addr, +} + +impl NodeIp { + pub fn new(cluster_ip_cidr: Ipv4Inet, external_ip_addr: Ipv4Addr) -> Self { + Self { + cluster_ip_cidr, + external_ip_addr, + } + } +} diff --git a/network/src/utils/mod.rs b/network/src/utils/mod.rs index d5d8f918..740a9620 100644 --- a/network/src/utils/mod.rs +++ b/network/src/utils/mod.rs @@ -8,7 +8,7 @@ use crate::error::KudoNetworkError; const IFACE_MAX_SIZE: usize = 12; -pub(crate) fn bridge_name(node_id: String) -> String { +pub fn bridge_name(node_id: String) -> String { format!("kbr{}", &node_id[..min(IFACE_MAX_SIZE, node_id.len())]) }