diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2a86c7537eb..0ab44abbacb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,7 @@ jobs: components: rustfmt - uses: Swatinem/rust-cache@v1 - run: sudo apt install -y libpcap-dev cmake + - run: cd mirrord-agent && cargo build - run: sudo PATH=/home/runner/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin:/usr/bin:/usr/sbin /home/runner/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/bin/cargo test -p mirrord-agent test_agent_image: @@ -102,4 +103,12 @@ jobs: - name: build mirrord run: cargo +nightly build --manifest-path=./Cargo.toml - name: run node tests - run: cargo test --manifest-path ./tests/Cargo.toml -- --test-threads 1 + run: cargo test --package tests --lib -- tests --nocapture --test-threads 1 + - name: switch minikube runtime + run: minikube delete && minikube start --container-runtime=docker && minikube image load test:latest + - name: setup nginx + run: kubectl apply -f tests/app.yaml + - name: wait for nginx pod + run: kubectl wait --for=condition=ready pod -l app=nginx --timeout=20s + - name: test with docker runtime + run: cargo test --package tests --lib -- tests --nocapture --ignored diff --git a/CHANGELOG.md b/CHANGELOG.md index bba35f3da8a..8e43fc4409a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] ### Added - Prompt user to update if their version is outdated in the VS Code extension or CLI. -- Set `MIRRORD_CHECK_VERSION` to false to make E2E tests not read update messages. +- Add support for docker runtime, closes [#95](https://github.com/metalbear-co/mirrord/issues/95). - Add a keep-alive to keep the agent-pod from exiting, closes [#63](https://github.com/metalbear-co/mirrord/issues/63) ## 2.0.0 diff --git a/Cargo.lock b/Cargo.lock index 9642ea9e3f5..cf796a3d164 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,6 +176,46 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d4b9e55620571c2200f4be87db2a9a69e2a107fc7d206a6accad58c3536cb" +dependencies = [ + "base64", + "bollard-stubs", + "bytes", + "chrono", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util 0.7.2", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295240332c78d04291f3ac857a281d5534a8e036f3dfcdaa294b22c0d424427" +dependencies = [ + "chrono", + "serde", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.9.1" @@ -245,6 +285,7 @@ dependencies = [ "num-integer", "num-traits", "serde", + "time", "winapi", ] @@ -793,6 +834,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.7" @@ -903,6 +950,19 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper", + "pin-project", + "tokio", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1265,6 +1325,7 @@ version = "2.0.4" dependencies = [ "actix-codec", "anyhow", + "bollard", "clap 3.1.18", "containerd-client", "futures", @@ -2046,6 +2107,12 @@ dependencies = [ "base64", ] +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.10" @@ -2206,6 +2273,29 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b827f2113224f3f19a665136f006709194bdfdcb1fdc1e4b2b5cbac8e0cced54" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_yaml" version = "0.8.24" @@ -2402,6 +2492,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "tinyvec" version = "1.6.0" diff --git a/mirrord-agent/Cargo.toml b/mirrord-agent/Cargo.toml index d600244974e..e5378a21676 100644 --- a/mirrord-agent/Cargo.toml +++ b/mirrord-agent/Cargo.toml @@ -33,6 +33,7 @@ tracing.workspace = true tracing-subscriber.workspace = true tokio-stream.workspace = true num-traits = "0.2.14" +bollard = "0.12.0" [dev-dependencies] test_bin = "0.4.0" \ No newline at end of file diff --git a/mirrord-agent/src/cli.rs b/mirrord-agent/src/cli.rs index db8a0dbb584..5005d6cd276 100644 --- a/mirrord-agent/src/cli.rs +++ b/mirrord-agent/src/cli.rs @@ -7,6 +7,10 @@ pub struct Args { #[clap(short, long)] pub container_id: Option, + /// Container runtime to use + #[clap(short = 'r', long)] + pub container_runtime: Option, + /// Port to use for communication #[clap(short = 'l', long, default_value_t = 61337)] pub communicate_port: u16, diff --git a/mirrord-agent/src/main.rs b/mirrord-agent/src/main.rs index f9a7fc5615d..b695a4f5761 100644 --- a/mirrord-agent/src/main.rs +++ b/mirrord-agent/src/main.rs @@ -158,6 +158,7 @@ async fn start() -> Result<()> { packet_command_rx, args.interface.clone(), args.container_id.clone(), + args.container_runtime.clone(), )); loop { select! { diff --git a/mirrord-agent/src/runtime.rs b/mirrord-agent/src/runtime.rs index 14bb49d1949..59a45c60c70 100644 --- a/mirrord-agent/src/runtime.rs +++ b/mirrord-agent/src/runtime.rs @@ -1,68 +1,65 @@ use std::{ fs::File, os::unix::io::{IntoRawFd, RawFd}, + path::PathBuf, }; use anyhow::{anyhow, Result}; +use bollard::{container::InspectContainerOptions, Docker}; use containerd_client::{ connect, - services::v1::{containers_client::ContainersClient, GetContainerRequest}, + services::v1::{tasks_client::TasksClient, GetRequest}, tonic::Request, with_namespace, }; use nix::sched::setns; -use serde::{Deserialize, Serialize}; const CONTAINERD_SOCK_PATH: &str = "/run/containerd/containerd.sock"; const DEFAULT_CONTAINERD_NAMESPACE: &str = "k8s.io"; -#[derive(Serialize, Deserialize, Debug)] -struct Namespace { - #[serde(rename = "type")] - ns_type: String, - path: Option, +pub async fn get_container_pid(container_id: &str, container_runtime: &str) -> Result { + match container_runtime { + "docker" => get_docker_container_pid(container_id.to_string()).await, + "containerd" => get_containerd_container_pid(container_id.to_string()).await, + _ => Err(anyhow!("Unsupported runtime: {}", container_runtime)), + } } -#[derive(Serialize, Deserialize, Debug)] -struct LinuxInfo { - namespaces: Vec, +async fn get_docker_container_pid(container_id: String) -> Result { + let client = Docker::connect_with_local_defaults()?; + let inspect_options = Some(InspectContainerOptions { size: false }); + let inspect_response = client + .inspect_container(&container_id, inspect_options) + .await?; + + let pid = inspect_response + .state + .and_then(|state| state.pid) + .and_then(|pid| if pid > 0 { Some(pid as u64) } else { None }) + .ok_or_else(|| anyhow!("No pid found"))?; + Ok(pid) } -#[derive(Serialize, Deserialize, Debug)] -struct Spec { - linux: LinuxInfo, +async fn get_containerd_container_pid(container_id: String) -> Result { + let channel = connect(CONTAINERD_SOCK_PATH).await?; + let mut client = TasksClient::new(channel); + let request = GetRequest { + container_id, + ..Default::default() + }; + let request = with_namespace!(request, DEFAULT_CONTAINERD_NAMESPACE); + let response = client.get(request).await?; + let pid = response + .into_inner() + .process + .ok_or_else(|| anyhow!("No pid found"))? + .pid; + + Ok(pid as u64) } -pub fn set_namespace(ns_path: &str) -> Result<()> { +pub fn set_namespace(ns_path: PathBuf) -> Result<()> { let fd: RawFd = File::open(ns_path)?.into_raw_fd(); setns(fd, nix::sched::CloneFlags::CLONE_NEWNET)?; Ok(()) } - -pub async fn get_container_namespace(container_id: String) -> Result { - let channel = connect(CONTAINERD_SOCK_PATH).await?; - let mut client = ContainersClient::new(channel); - let request = GetContainerRequest { id: container_id }; - let request = with_namespace!(request, DEFAULT_CONTAINERD_NAMESPACE); - let resp = client.get(request).await?; - let resp = resp.into_inner(); - let container = resp - .container - .ok_or_else(|| anyhow!("container not found"))?; - let spec: Spec = serde_json::from_slice( - &container - .spec - .ok_or_else(|| anyhow!("invalid data from containerd"))? - .value, - )?; - let ns_path = spec - .linux - .namespaces - .iter() - .find(|ns| ns.ns_type == "network") - .ok_or_else(|| anyhow!("network namespace not found"))? - .path - .as_ref() - .ok_or_else(|| anyhow!("no network namespace path"))?; - Ok(ns_path.to_owned()) -} diff --git a/mirrord-agent/src/sniffer.rs b/mirrord-agent/src/sniffer.rs index 447fb33b8c7..b7b0a49d00d 100644 --- a/mirrord-agent/src/sniffer.rs +++ b/mirrord-agent/src/sniffer.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, hash::{Hash, Hasher}, net::{IpAddr, Ipv4Addr}, + path::PathBuf, }; use anyhow::{anyhow, Result}; @@ -22,13 +23,15 @@ use tokio::{ use tracing::{debug, error}; use crate::{ - runtime::{get_container_namespace, set_namespace}, + runtime::{get_container_pid, set_namespace}, util::IndexAllocator, }; const DUMMY_BPF: &str = "tcp dst port 1 and tcp src port 1 and dst host 8.1.2.3 and src host 8.1.2.3"; +const DEFAULT_RUNTIME: &str = "containerd"; + type ConnectionID = u16; #[derive(Debug)] @@ -236,13 +239,28 @@ pub async fn packet_worker( mut rx: Receiver, interface: String, container_id: Option, + container_runtime: Option, ) -> Result<()> { debug!("setting namespace"); - if let Some(container_id) = container_id { - let namespace = get_container_namespace(container_id).await?; - debug!("Found namespace to attach to {:?}", &namespace); - set_namespace(&namespace)?; + + let pid = match (container_id, container_runtime) { + (Some(container_id), Some(container_runtime)) => { + get_container_pid(&container_id, &container_runtime) + .await + .ok() + } + (Some(container_id), None) => get_container_pid(&container_id, DEFAULT_RUNTIME).await.ok(), + (None, Some(_)) => return Err(anyhow!("Container ID not specified")), + _ => None, + }; + + if let Some(pid) = pid { + let namespace = PathBuf::from("/proc") + .join(PathBuf::from(pid.to_string())) + .join(PathBuf::from("ns/net")); + set_namespace(namespace).unwrap(); } + debug!("preparing sniffer"); let sniffer = prepare_sniffer(interface)?; debug!("done prepare sniffer"); diff --git a/mirrord-layer/src/pod_api.rs b/mirrord-layer/src/pod_api.rs index a642cd46ad3..a53c3c7d5e9 100644 --- a/mirrord-layer/src/pod_api.rs +++ b/mirrord-layer/src/pod_api.rs @@ -14,7 +14,9 @@ use tracing::{error, warn}; use crate::config; struct RuntimeData { container_id: String, + container_runtime: String, node_name: String, + socket_path: String, } impl RuntimeData { @@ -23,18 +25,28 @@ impl RuntimeData { let pod = pods_api.get(pod_name).await.unwrap(); let node_name = &pod.spec.unwrap().node_name; let container_statuses = pod.status.unwrap().container_statuses.unwrap(); - let container_id = container_statuses + let container_info = container_statuses .first() .unwrap() .container_id .as_ref() .unwrap() - .split("//") - .last() - .unwrap(); + .split("://") + .collect::>(); + + let (container_runtime, socket_path) = match container_info.first() { + Some(&"docker") => ("docker", "/var/run/docker.sock"), + Some(&"containerd") => ("containerd", "/run/containerd/containerd.sock"), + _ => panic!("unspported container runtime"), + }; + + let container_id = container_info.last().unwrap(); + RuntimeData { container_id: container_id.to_string(), + container_runtime: container_runtime.to_string(), node_name: node_name.as_ref().unwrap().to_string(), + socket_path: socket_path.to_string(), } } } @@ -78,9 +90,9 @@ pub async fn create_agent( "restartPolicy": "Never", "volumes": [ { - "name": "containerd", + "name": "sockpath", "hostPath": { - "path": "/run/containerd/containerd.sock" + "path": runtime_data.socket_path } } ], @@ -94,16 +106,18 @@ pub async fn create_agent( }, "volumeMounts": [ { - "mountPath": "/run/containerd/containerd.sock", - "name": "containerd" + "mountPath": runtime_data.socket_path, + "name": "sockpath" } ], "command": [ "./mirrord-agent", "--container-id", runtime_data.container_id, + "--container-runtime", + runtime_data.container_runtime, "-t", - "30" + "30", ], "env": [{"name": "RUST_LOG", "value": log_level}], } diff --git a/tests/src/sanity.rs b/tests/src/sanity.rs index 974ccbc096e..186d245a165 100644 --- a/tests/src/sanity.rs +++ b/tests/src/sanity.rs @@ -14,9 +14,14 @@ mod tests { use crate::utils::*; #[tokio::test] + async fn test_complete_node_api() { + _test_complete_node_api().await; + } + + // actual test function used with "containerd", "docker" runtimes // starts the node(express.js) api server, sends four different requests, validates data, // stops the server and validates if the agent job and pod are deleted - async fn test_complete_node_api() { + async fn _test_complete_node_api() { let client = setup_kube_client().await; let pod_namespace = "default"; @@ -288,4 +293,11 @@ mod tests { .unwrap() .unwrap(); } + + // docker runtime test + #[ignore] + #[tokio::test] + async fn test_docker_runtime() { + _test_complete_node_api().await; + } }