From 26ab808730df0ec8b29de3b67032b380a2e53211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:08:27 +0100 Subject: [PATCH 01/23] Fix port mirror rejection in intproxy (#2998) * Fixed handling rejection in intproxy * Changelog * fmt --- .../+mirrord-policy-rejection.fixed.md | 1 + .../src/proxies/incoming/subscriptions.rs | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 changelog.d/+mirrord-policy-rejection.fixed.md diff --git a/changelog.d/+mirrord-policy-rejection.fixed.md b/changelog.d/+mirrord-policy-rejection.fixed.md new file mode 100644 index 00000000000..ad7415c5411 --- /dev/null +++ b/changelog.d/+mirrord-policy-rejection.fixed.md @@ -0,0 +1 @@ +Fixed a bug where port mirroring block (due to active mirrord policies) would terminate the mirrord session. diff --git a/mirrord/intproxy/src/proxies/incoming/subscriptions.rs b/mirrord/intproxy/src/proxies/incoming/subscriptions.rs index 8439e79d8e1..7731707d8c8 100644 --- a/mirrord/intproxy/src/proxies/incoming/subscriptions.rs +++ b/mirrord/intproxy/src/proxies/incoming/subscriptions.rs @@ -242,6 +242,7 @@ impl SubscriptionsManager { Ok(subscription.confirm()) } + Err(ResponseError::PortAlreadyStolen(port)) => { let Some(subscription) = self.subscriptions.remove(&port) else { return Ok(vec![]); @@ -255,23 +256,30 @@ impl SubscriptionsManager { } } } + Err( - ref response_err @ ResponseError::Forbidden { - blocked_action: BlockedAction::Steal(ref steal_type), - .. + ref response_error @ ResponseError::Forbidden { + ref blocked_action, .. }, ) => { - tracing::warn!("Port subscribe blocked by policy: {response_err}"); - let Some(subscription) = self.subscriptions.remove(&steal_type.get_port()) else { + tracing::warn!(%response_error, "Port subscribe blocked by policy"); + + let port = match blocked_action { + BlockedAction::Steal(steal_type) => steal_type.get_port(), + BlockedAction::Mirror(port) => *port, + }; + let Some(subscription) = self.subscriptions.remove(&port) else { return Ok(vec![]); }; + subscription - .reject(response_err.clone()) - .map_err(|sub|{ - tracing::error!("Subscription {sub:?} was confirmed before, then requested again and blocked by a policy."); - IncomingProxyError::SubscriptionFailed(response_err.clone()) + .reject(response_error.clone()) + .map_err(|subscription|{ + tracing::error!(?subscription, "Subscription was confirmed before, then requested again and blocked by a policy."); + IncomingProxyError::SubscriptionFailed(response_error.clone()) }) } + Err(err) => Err(IncomingProxyError::SubscriptionFailed(err)), } } From 1a23d5604e1ce86354ee4bed096c375ac148607e Mon Sep 17 00:00:00 2001 From: Dmitry Dodzin Date: Thu, 2 Jan 2025 20:41:17 +0200 Subject: [PATCH 02/23] Update intproxy tracing subscriber print to console in container (#2881) * Copy * It works !!!!! * Tiny * Ops * Ops * Update message * Docs * use tokio adapter and wrapper instead of writing one --- changelog.d/2868.changed.md | 1 + mirrord/cli/Cargo.toml | 3 +- mirrord/cli/src/config.rs | 33 +- mirrord/cli/src/container.rs | 324 +++++-------------- mirrord/cli/src/container/command_builder.rs | 11 +- mirrord/cli/src/container/sidecar.rs | 126 ++++++++ mirrord/cli/src/external_proxy.rs | 25 +- mirrord/cli/src/internal_proxy.rs | 47 +-- mirrord/cli/src/logging.rs | 192 +++++++++++ mirrord/cli/src/main.rs | 31 +- 10 files changed, 448 insertions(+), 345 deletions(-) create mode 100644 changelog.d/2868.changed.md create mode 100644 mirrord/cli/src/container/sidecar.rs create mode 100644 mirrord/cli/src/logging.rs diff --git a/changelog.d/2868.changed.md b/changelog.d/2868.changed.md new file mode 100644 index 00000000000..f24bdc1461c --- /dev/null +++ b/changelog.d/2868.changed.md @@ -0,0 +1 @@ +Updated how intproxy is outputing logfile when using container mode, now logs will be written on host machine. diff --git a/mirrord/cli/Cargo.toml b/mirrord/cli/Cargo.toml index 420ec6e97f1..aea0b77d7cc 100644 --- a/mirrord/cli/Cargo.toml +++ b/mirrord/cli/Cargo.toml @@ -63,11 +63,12 @@ tempfile.workspace = true rcgen.workspace = true rustls-pemfile.workspace = true tokio-rustls.workspace = true -tokio-stream = { workspace = true, features = ["net"] } +tokio-stream = { workspace = true, features = ["io-util", "net"] } regex.workspace = true mid = "3.0.0" rand.workspace = true + [target.'cfg(target_os = "macos")'.dependencies] mirrord-sip = { path = "../sip" } diff --git a/mirrord/cli/src/config.rs b/mirrord/cli/src/config.rs index 1570cfbca35..367504ddbf4 100644 --- a/mirrord/cli/src/config.rs +++ b/mirrord/cli/src/config.rs @@ -834,6 +834,13 @@ pub struct RuntimeArgs { /// Supported command for using mirrord with container runtimes. #[derive(Subcommand, Debug, Clone)] pub(super) enum ContainerRuntimeCommand { + /// Execute a ` create` command with mirrord loaded. (not supported with ) + #[command(hide = true)] + Create { + /// Arguments that will be propogated to underlying ` create` command. + #[arg(allow_hyphen_values = true, trailing_var_arg = true)] + runtime_args: Vec, + }, /// Execute a ` run` command with mirrord loaded. Run { /// Arguments that will be propogated to underlying ` run` command. @@ -843,14 +850,17 @@ pub(super) enum ContainerRuntimeCommand { } impl ContainerRuntimeCommand { - pub fn run>(runtime_args: impl IntoIterator) -> Self { - ContainerRuntimeCommand::Run { + pub fn create>(runtime_args: impl IntoIterator) -> Self { + ContainerRuntimeCommand::Create { runtime_args: runtime_args.into_iter().map(T::into).collect(), } } pub fn has_publish(&self) -> bool { - let ContainerRuntimeCommand::Run { runtime_args } = self; + let runtime_args = match self { + ContainerRuntimeCommand::Run { runtime_args } => runtime_args, + _ => return false, + }; let mut hit_trailing_token = false; @@ -860,6 +870,15 @@ impl ContainerRuntimeCommand { !hit_trailing_token && matches!(runtime_arg.as_str(), "-p" | "--publish") }) } + + pub fn into_parts(self) -> (Vec, Vec) { + match self { + ContainerRuntimeCommand::Create { runtime_args } => { + (vec!["create".to_owned()], runtime_args) + } + ContainerRuntimeCommand::Run { runtime_args } => (vec!["run".to_owned()], runtime_args), + } + } } #[derive(Args, Debug)] @@ -947,7 +966,9 @@ mod tests { assert_eq!(runtime_args.runtime, ContainerRuntime::Podman); - let ContainerRuntimeCommand::Run { runtime_args } = runtime_args.command; + let ContainerRuntimeCommand::Run { runtime_args } = runtime_args.command else { + panic!("expected run command"); + }; assert_eq!(runtime_args, vec!["-it", "--rm", "debian"]); } @@ -965,7 +986,9 @@ mod tests { assert_eq!(runtime_args.runtime, ContainerRuntime::Podman); - let ContainerRuntimeCommand::Run { runtime_args } = runtime_args.command; + let ContainerRuntimeCommand::Run { runtime_args } = runtime_args.command else { + panic!("expected run command"); + }; assert_eq!(runtime_args, vec!["-it", "--rm", "debian"]); } diff --git a/mirrord/cli/src/container.rs b/mirrord/cli/src/container.rs index afffad9831d..aa396a0162f 100644 --- a/mirrord/cli/src/container.rs +++ b/mirrord/cli/src/container.rs @@ -2,7 +2,6 @@ use std::{ collections::HashMap, io::Write, net::SocketAddr, - ops::Not, path::{Path, PathBuf}, process::Stdio, time::Duration, @@ -15,7 +14,6 @@ use mirrord_config::{ external_proxy::{MIRRORD_EXTERNAL_TLS_CERTIFICATE_ENV, MIRRORD_EXTERNAL_TLS_KEY_ENV}, internal_proxy::{ MIRRORD_INTPROXY_CLIENT_TLS_CERTIFICATE_ENV, MIRRORD_INTPROXY_CLIENT_TLS_KEY_ENV, - MIRRORD_INTPROXY_CONTAINER_MODE_ENV, }, LayerConfig, MIRRORD_CONFIG_FILE_ENV, }; @@ -28,20 +26,22 @@ use tokio::{ use tracing::Level; use crate::{ - config::{ContainerRuntime, ContainerRuntimeCommand, ExecParams, RuntimeArgs}, + config::{ContainerRuntime, ExecParams, RuntimeArgs}, connection::AGENT_CONNECT_INFO_ENV_KEY, - container::command_builder::RuntimeCommandBuilder, + container::{command_builder::RuntimeCommandBuilder, sidecar::Sidecar}, error::{CliError, CliResult, ContainerError}, execution::{ MirrordExecution, LINUX_INJECTION_ENV_VAR, MIRRORD_CONNECT_TCP_ENV, MIRRORD_EXECUTION_KIND_ENV, }, + logging::pipe_intproxy_sidecar_logs, util::MIRRORD_CONSOLE_ADDR_ENV, }; static CONTAINER_EXECUTION_KIND: ExecutionKind = ExecutionKind::Container; mod command_builder; +mod sidecar; /// Format [`Command`] to look like the executated command (currently without env because we don't /// use it in these scenarios) @@ -65,10 +65,8 @@ async fn exec_and_get_first_line(command: &mut Command) -> Result .spawn() .map_err(ContainerError::UnableToExecuteCommand)?; - tracing::warn!(?child, "spawned watch for child"); - let stdout = child.stdout.take().expect("stdout should be piped"); - let stderr = child.stderr.take().expect("stdout should be piped"); + let stderr = child.stderr.take().expect("stderr should be piped"); let result = tokio::time::timeout(Duration::from_secs(30), async { BufReader::new(stdout) @@ -151,115 +149,6 @@ fn create_self_signed_certificate( Ok((certificate, private_key)) } -/// Create a "sidecar" container that is running `mirrord intproxy` that connects to `mirrord -/// extproxy` running on user machine to be used by execution container (via mounting on same -/// network) -#[tracing::instrument(level = Level::TRACE, ret)] -async fn create_sidecar_intproxy( - config: &LayerConfig, - base_command: &RuntimeCommandBuilder, - connection_info: Vec<(&str, &str)>, -) -> Result<(String, SocketAddr), ContainerError> { - let mut sidecar_command = base_command.clone(); - - sidecar_command.add_env(MIRRORD_INTPROXY_CONTAINER_MODE_ENV, "true"); - sidecar_command.add_envs(connection_info); - - let cleanup = config.container.cli_prevent_cleanup.not().then_some("--rm"); - - let sidecar_container_command = ContainerRuntimeCommand::run( - config - .container - .cli_extra_args - .iter() - .map(String::as_str) - .chain(cleanup) - .chain(["-d", &config.container.cli_image, "mirrord", "intproxy"]), - ); - - let (runtime_binary, sidecar_args) = sidecar_command - .with_command(sidecar_container_command) - .into_command_args(); - - let mut sidecar_container_spawn = Command::new(&runtime_binary); - - sidecar_container_spawn.args(sidecar_args); - - let sidecar_container_id = exec_and_get_first_line(&mut sidecar_container_spawn) - .await? - .ok_or_else(|| { - ContainerError::UnsuccesfulCommandOutput( - format_command(&sidecar_container_spawn), - "stdout and stderr were empty".to_owned(), - ) - })?; - - // For Docker runtime sometimes the sidecar doesn't start so we double check. - // See [#2927](https://github.com/metalbear-co/mirrord/issues/2927) - if matches!(base_command.runtime(), ContainerRuntime::Docker) { - let mut container_inspect_command = Command::new(&runtime_binary); - container_inspect_command - .args(["inspect", &sidecar_container_id]) - .stdout(Stdio::piped()); - - let container_inspect_output = container_inspect_command.output().await.map_err(|err| { - ContainerError::UnsuccesfulCommandOutput( - format_command(&container_inspect_command), - err.to_string(), - ) - })?; - - let (container_inspection,) = - serde_json::from_slice::<(serde_json::Value,)>(&container_inspect_output.stdout) - .unwrap_or_default(); - - let container_status = container_inspection - .get("State") - .and_then(|inspect| inspect.get("Status")); - - if container_status - .map(|status| status == "created") - .unwrap_or(false) - { - let mut container_start_command = Command::new(&runtime_binary); - - container_start_command - .args(["start", &sidecar_container_id]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - - let _ = container_start_command.status().await.map_err(|err| { - ContainerError::UnsuccesfulCommandOutput( - format_command(&container_start_command), - err.to_string(), - ) - })?; - } - } - - // After spawning sidecar with -d flag it prints container_id, now we need the address of - // intproxy running in sidecar to be used by mirrord-layer in execution container - let intproxy_address: SocketAddr = { - let mut attach_command = Command::new(&runtime_binary); - attach_command.args(["logs", "-f", &sidecar_container_id]); - - match exec_and_get_first_line(&mut attach_command).await? { - Some(line) => line - .parse() - .map_err(ContainerError::UnableParseProxySocketAddr)?, - None => { - return Err(ContainerError::UnsuccesfulCommandOutput( - format_command(&attach_command), - "stdout and stderr were empty".into(), - )) - } - } - }; - - Ok((sidecar_container_id, intproxy_address)) -} - type TlsGuard = (NamedTempFile, NamedTempFile); fn prepare_tls_certs_for_container( @@ -315,34 +204,15 @@ fn prepare_tls_certs_for_container( Ok((internal_proxy_tls_guards, external_proxy_tls_guards)) } -/// Main entry point for the `mirrord container` command. -/// This spawns: "agent" - "external proxy" - "intproxy sidecar" - "execution container" -pub(crate) async fn container_command( - runtime_args: RuntimeArgs, - exec_params: ExecParams, +/// Load [`LayerConfig`] from env and create [`AnalyticsReporter`] whilst reporting any warnings. +fn create_config_and_analytics( + progress: &mut P, watch: drain::Watch, -) -> CliResult { - let mut progress = ProgressTracker::from_env("mirrord container"); - - if runtime_args.command.has_publish() { - progress.warning("mirrord container may have problems with \"-p\" directly container in command, please add to \"contanier.cli_extra_args\" in config if you are planning to publish ports"); - } - - progress.warning("mirrord container is currently an unstable feature"); - - for (name, value) in exec_params.as_env_vars()? { - std::env::set_var(name, value); - } - - std::env::set_var( - MIRRORD_EXECUTION_KIND_ENV, - (CONTAINER_EXECUTION_KIND as u32).to_string(), - ); - - let (mut config, mut context) = LayerConfig::from_env_with_warnings()?; +) -> CliResult<(LayerConfig, AnalyticsReporter)> { + let (config, mut context) = LayerConfig::from_env_with_warnings()?; // Initialize only error analytics, extproxy will be the full AnalyticsReporter. - let mut analytics = + let analytics = AnalyticsReporter::only_error(config.telemetry, CONTAINER_EXECUTION_KIND, watch); config.verify(&mut context)?; @@ -350,16 +220,22 @@ pub(crate) async fn container_command( progress.warning(warning); } - let (_internal_proxy_tls_guards, _external_proxy_tls_guards) = - prepare_tls_certs_for_container(&mut config)?; - - let composed_config_file = create_composed_config(&config)?; - std::env::set_var(MIRRORD_CONFIG_FILE_ENV, composed_config_file.path()); + Ok((config, analytics)) +} +/// Create [`RuntimeCommandBuilder`] with the corresponding [`Sidecar`] connected to +/// [`MirrordExecution`] as extproxy. +async fn create_runtime_command_with_sidecar( + analytics: &mut AnalyticsReporter, + progress: &mut P, + config: &LayerConfig, + composed_config_path: &Path, + runtime: ContainerRuntime, +) -> CliResult<(RuntimeCommandBuilder, Sidecar, MirrordExecution)> { let mut sub_progress = progress.subtask("preparing to launch process"); let execution_info = - MirrordExecution::start_external(&config, &mut sub_progress, &mut analytics).await?; + MirrordExecution::start_external(config, &mut sub_progress, analytics).await?; let mut connection_info = Vec::new(); let mut execution_info_env_without_connection_info = Vec::new(); @@ -374,7 +250,7 @@ pub(crate) async fn container_command( sub_progress.success(None); - let mut runtime_command = RuntimeCommandBuilder::new(runtime_args.runtime); + let mut runtime_command = RuntimeCommandBuilder::new(runtime); if let Ok(console_addr) = std::env::var(MIRRORD_CONSOLE_ADDR_ENV) { if console_addr @@ -398,8 +274,7 @@ pub(crate) async fn container_command( ); runtime_command.add_env(MIRRORD_CONFIG_FILE_ENV, "/tmp/mirrord-config.json"); - runtime_command - .add_volume::(composed_config_file.path(), "/tmp/mirrord-config.json"); + runtime_command.add_volume::(composed_config_path, "/tmp/mirrord-config.json"); let mut load_env_and_mount_pem = |env: &str, path: &Path| { let container_path = format!("/tmp/{}.pem", env.to_lowercase()); @@ -426,11 +301,57 @@ pub(crate) async fn container_command( runtime_command.add_envs(execution_info_env_without_connection_info); - let (sidecar_container_id, sidecar_intproxy_address) = - create_sidecar_intproxy(&config, &runtime_command, connection_info).await?; + let sidecar = Sidecar::create_intproxy(config, &runtime_command, connection_info).await?; + + runtime_command.add_network(sidecar.as_network()); + runtime_command.add_volumes_from(&sidecar.container_id); + + Ok((runtime_command, sidecar, execution_info)) +} + +/// Main entry point for the `mirrord container` command. +/// This spawns: "agent" - "external proxy" - "intproxy sidecar" - "execution container" +pub(crate) async fn container_command( + runtime_args: RuntimeArgs, + exec_params: ExecParams, + watch: drain::Watch, +) -> CliResult { + let mut progress = ProgressTracker::from_env("mirrord container"); + + if runtime_args.command.has_publish() { + progress.warning("mirrord container may have problems with \"-p\" when used as part of container run command, please add the publish arguments to \"contanier.cli_extra_args\" in config if you are planning to publish ports"); + } + + progress.warning("mirrord container is currently an unstable feature"); + + for (name, value) in exec_params.as_env_vars()? { + std::env::set_var(name, value); + } + + std::env::set_var( + MIRRORD_EXECUTION_KIND_ENV, + (CONTAINER_EXECUTION_KIND as u32).to_string(), + ); - runtime_command.add_network(format!("container:{sidecar_container_id}")); - runtime_command.add_volumes_from(sidecar_container_id); + let (mut config, mut analytics) = create_config_and_analytics(&mut progress, watch)?; + + let (_internal_proxy_tls_guards, _external_proxy_tls_guards) = + prepare_tls_certs_for_container(&mut config)?; + + let composed_config_file = create_composed_config(&config)?; + std::env::set_var(MIRRORD_CONFIG_FILE_ENV, composed_config_file.path()); + + let (mut runtime_command, sidecar, _execution_info) = create_runtime_command_with_sidecar( + &mut analytics, + &mut progress, + &config, + composed_config_file.path(), + runtime_args.runtime, + ) + .await?; + + let (sidecar_intproxy_address, sidecar_intproxy_logs) = sidecar.start().await?; + tokio::spawn(pipe_intproxy_sidecar_logs(&config, sidecar_intproxy_logs).await?); runtime_command.add_env(LINUX_INJECTION_ENV_VAR, config.container.cli_image_lib_path); runtime_command.add_env( @@ -505,15 +426,8 @@ pub(crate) async fn container_ext_command( std::env::set_var("MIRRORD_IMPERSONATED_TARGET", target.clone()); env.insert("MIRRORD_IMPERSONATED_TARGET".into(), target.to_string()); } - let (mut config, mut context) = LayerConfig::from_env_with_warnings()?; - - // Initialize only error analytics, extproxy will be the full AnalyticsReporter. - let mut analytics = AnalyticsReporter::only_error(config.telemetry, Default::default(), watch); - config.verify(&mut context)?; - for warning in context.get_warnings() { - progress.warning(warning); - } + let (mut config, mut analytics) = create_config_and_analytics(&mut progress, watch)?; let (_internal_proxy_tls_guards, _external_proxy_tls_guards) = prepare_tls_certs_for_container(&mut config)?; @@ -521,86 +435,22 @@ pub(crate) async fn container_ext_command( let composed_config_file = create_composed_config(&config)?; std::env::set_var(MIRRORD_CONFIG_FILE_ENV, composed_config_file.path()); - let mut sub_progress = progress.subtask("preparing to launch process"); - - let execution_info = - MirrordExecution::start_external(&config, &mut sub_progress, &mut analytics).await?; - - let mut connection_info = Vec::new(); - let mut execution_info_env_without_connection_info = Vec::new(); - - for (key, value) in &execution_info.environment { - if key == MIRRORD_CONNECT_TCP_ENV || key == AGENT_CONNECT_INFO_ENV_KEY { - connection_info.push((key.as_str(), value.as_str())); - } else { - execution_info_env_without_connection_info.push((key.as_str(), value.as_str())) - } - } - - sub_progress.success(None); - let container_runtime = std::env::var("MIRRORD_CONTAINER_USE_RUNTIME") .ok() .and_then(|value| ContainerRuntime::from_str(&value, true).ok()) .unwrap_or(ContainerRuntime::Docker); - let mut runtime_command = RuntimeCommandBuilder::new(container_runtime); - - if let Ok(console_addr) = std::env::var(MIRRORD_CONSOLE_ADDR_ENV) { - if console_addr - .parse() - .map(|addr: SocketAddr| !addr.ip().is_loopback()) - .unwrap_or_default() - { - runtime_command.add_env(MIRRORD_CONSOLE_ADDR_ENV, console_addr); - } else { - tracing::warn!( - ?console_addr, - "{MIRRORD_CONSOLE_ADDR_ENV} needs to be a non loopback address when used with containers" - ); - } - } - - runtime_command.add_env(MIRRORD_PROGRESS_ENV, "off"); - runtime_command.add_env( - MIRRORD_EXECUTION_KIND_ENV, - (CONTAINER_EXECUTION_KIND as u32).to_string(), - ); - - runtime_command.add_env(MIRRORD_CONFIG_FILE_ENV, "/tmp/mirrord-config.json"); - runtime_command - .add_volume::(composed_config_file.path(), "/tmp/mirrord-config.json"); - - let mut load_env_and_mount_pem = |env: &str, path: &Path| { - let container_path = format!("/tmp/{}.pem", env.to_lowercase()); - - runtime_command.add_env(env, &container_path); - runtime_command.add_volume::(path, container_path); - }; - - if let Some(path) = config.internal_proxy.client_tls_certificate.as_ref() { - load_env_and_mount_pem(MIRRORD_INTPROXY_CLIENT_TLS_CERTIFICATE_ENV, path) - } - - if let Some(path) = config.internal_proxy.client_tls_key.as_ref() { - load_env_and_mount_pem(MIRRORD_INTPROXY_CLIENT_TLS_KEY_ENV, path) - } - - if let Some(path) = config.external_proxy.tls_certificate.as_ref() { - load_env_and_mount_pem(MIRRORD_EXTERNAL_TLS_CERTIFICATE_ENV, path) - } - - if let Some(path) = config.external_proxy.tls_key.as_ref() { - load_env_and_mount_pem(MIRRORD_EXTERNAL_TLS_KEY_ENV, path) - } - - runtime_command.add_envs(execution_info_env_without_connection_info); - - let (sidecar_container_id, sidecar_intproxy_address) = - create_sidecar_intproxy(&config, &runtime_command, connection_info).await?; - - runtime_command.add_network(format!("container:{sidecar_container_id}")); - runtime_command.add_volumes_from(sidecar_container_id); + let (mut runtime_command, sidecar, execution_info) = create_runtime_command_with_sidecar( + &mut analytics, + &mut progress, + &config, + composed_config_file.path(), + container_runtime, + ) + .await?; + + let (sidecar_intproxy_address, sidecar_intproxy_logs) = sidecar.start().await?; + tokio::spawn(pipe_intproxy_sidecar_logs(&config, sidecar_intproxy_logs).await?); runtime_command.add_env(LINUX_INJECTION_ENV_VAR, config.container.cli_image_lib_path); runtime_command.add_env( diff --git a/mirrord/cli/src/container/command_builder.rs b/mirrord/cli/src/container/command_builder.rs index 20e0fac883b..c5bf3a6ae39 100644 --- a/mirrord/cli/src/container/command_builder.rs +++ b/mirrord/cli/src/container/command_builder.rs @@ -17,10 +17,6 @@ pub struct RuntimeCommandBuilder { } impl RuntimeCommandBuilder { - pub(super) fn runtime(&self) -> &ContainerRuntime { - &self.runtime - } - fn push_arg(&mut self, value: V) where V: Into, @@ -152,13 +148,12 @@ impl RuntimeCommandBuilder { step, } = self; - let (runtime_command, runtime_args) = match step.command { - ContainerRuntimeCommand::Run { runtime_args } => ("run".to_owned(), runtime_args), - }; + let (runtime_command, runtime_args) = step.command.into_parts(); ( runtime.to_string(), - std::iter::once(runtime_command) + runtime_command + .into_iter() .chain(extra_args) .chain(runtime_args), ) diff --git a/mirrord/cli/src/container/sidecar.rs b/mirrord/cli/src/container/sidecar.rs new file mode 100644 index 00000000000..28641070058 --- /dev/null +++ b/mirrord/cli/src/container/sidecar.rs @@ -0,0 +1,126 @@ +use std::{net::SocketAddr, ops::Not, process::Stdio, time::Duration}; + +use mirrord_config::{internal_proxy::MIRRORD_INTPROXY_CONTAINER_MODE_ENV, LayerConfig}; +use tokio::{ + io::{AsyncBufReadExt, BufReader}, + process::{ChildStderr, ChildStdout, Command}, +}; +use tokio_stream::{wrappers::LinesStream, StreamExt}; +use tracing::Level; + +use crate::{ + config::ContainerRuntimeCommand, + container::{command_builder::RuntimeCommandBuilder, exec_and_get_first_line, format_command}, + error::ContainerError, +}; + +#[derive(Debug)] +pub(crate) struct Sidecar { + pub container_id: String, + pub runtime_binary: String, +} + +impl Sidecar { + /// Create a "sidecar" container that is running `mirrord intproxy` that connects to `mirrord + /// extproxy` running on user machine to be used by execution container (via mounting on same + /// network) + #[tracing::instrument(level = Level::TRACE)] + pub async fn create_intproxy( + config: &LayerConfig, + base_command: &RuntimeCommandBuilder, + connection_info: Vec<(&str, &str)>, + ) -> Result { + let mut sidecar_command = base_command.clone(); + + sidecar_command.add_env(MIRRORD_INTPROXY_CONTAINER_MODE_ENV, "true"); + sidecar_command.add_envs(connection_info); + + let cleanup = config.container.cli_prevent_cleanup.not().then_some("--rm"); + + let sidecar_container_command = ContainerRuntimeCommand::create( + config + .container + .cli_extra_args + .iter() + .map(String::as_str) + .chain(cleanup) + .chain([&config.container.cli_image, "mirrord", "intproxy"]), + ); + + let (runtime_binary, sidecar_args) = sidecar_command + .with_command(sidecar_container_command) + .into_command_args(); + + let mut sidecar_container_spawn = Command::new(&runtime_binary); + sidecar_container_spawn.args(sidecar_args); + + let container_id = exec_and_get_first_line(&mut sidecar_container_spawn) + .await? + .ok_or_else(|| { + ContainerError::UnsuccesfulCommandOutput( + format_command(&sidecar_container_spawn), + "stdout and stderr were empty".to_owned(), + ) + })?; + + Ok(Sidecar { + container_id, + runtime_binary, + }) + } + + pub fn as_network(&self) -> String { + let Sidecar { container_id, .. } = self; + format!("container:{container_id}") + } + + #[tracing::instrument(level = Level::TRACE)] + pub async fn start(&self) -> Result<(SocketAddr, SidecarLogs), ContainerError> { + let mut command = Command::new(&self.runtime_binary); + command.args(["start", "--attach", &self.container_id]); + + let mut child = command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(ContainerError::UnableToExecuteCommand)?; + + let mut stdout = + BufReader::new(child.stdout.take().expect("stdout should be piped")).lines(); + let stderr = BufReader::new(child.stderr.take().expect("stderr should be piped")).lines(); + + let first_line = tokio::time::timeout(Duration::from_secs(30), async { + stdout.next_line().await.map_err(|error| { + ContainerError::UnableReadCommandStdout(format_command(&command), error) + }) + }) + .await + .map_err(|_| { + ContainerError::UnsuccesfulCommandOutput( + format_command(&command), + "timeout reached for reading first line".into(), + ) + })?? + .ok_or_else(|| { + ContainerError::UnsuccesfulCommandOutput( + format_command(&command), + "unexpected EOF".into(), + ) + })?; + + let internal_proxy_addr: SocketAddr = first_line + .parse() + .map_err(ContainerError::UnableParseProxySocketAddr)?; + + Ok(( + internal_proxy_addr, + LinesStream::new(stdout).merge(LinesStream::new(stderr)), + )) + } +} + +type SidecarLogs = tokio_stream::adapters::Merge< + LinesStream>, + LinesStream>, +>; diff --git a/mirrord/cli/src/external_proxy.rs b/mirrord/cli/src/external_proxy.rs index b728416b83a..a6ab4bb8674 100644 --- a/mirrord/cli/src/external_proxy.rs +++ b/mirrord/cli/src/external_proxy.rs @@ -20,7 +20,7 @@ //! ``` use std::{ - fs::{File, OpenOptions}, + fs::File, io, io::BufReader, net::{Ipv4Addr, SocketAddr}, @@ -41,13 +41,13 @@ use tokio::net::{TcpListener, TcpStream}; use tokio_rustls::server::TlsStream; use tokio_util::{either::Either, sync::CancellationToken}; use tracing::Level; -use tracing_subscriber::EnvFilter; use crate::{ connection::AGENT_CONNECT_INFO_ENV_KEY, error::{CliResult, ExternalProxyError}, execution::MIRRORD_EXECUTION_KIND_ENV, internal_proxy::connect_and_ping, + logging::init_extproxy_tracing_registry, util::{create_listen_socket, detach_io}, }; @@ -62,28 +62,9 @@ fn print_addr(listener: &TcpListener) -> io::Result<()> { pub async fn proxy(listen_port: u16, watch: drain::Watch) -> CliResult<()> { let config = LayerConfig::from_env()?; + init_extproxy_tracing_registry(&config)?; tracing::info!(?config, "external_proxy starting"); - if let Some(log_destination) = config.external_proxy.log_destination.as_ref() { - let output_file = OpenOptions::new() - .create(true) - .append(true) - .open(log_destination) - .map_err(|e| ExternalProxyError::OpenLogFile(log_destination.clone(), e))?; - - let tracing_registry = tracing_subscriber::fmt() - .with_writer(output_file) - .with_ansi(false); - - if let Some(log_level) = config.external_proxy.log_level.as_ref() { - tracing_registry - .with_env_filter(EnvFilter::builder().parse_lossy(log_level)) - .init(); - } else { - tracing_registry.init(); - } - } - let agent_connect_info = std::env::var(AGENT_CONNECT_INFO_ENV_KEY) .ok() .map(|var| { diff --git a/mirrord/cli/src/internal_proxy.rs b/mirrord/cli/src/internal_proxy.rs index 1c8763a0d92..b1b9a20b72a 100644 --- a/mirrord/cli/src/internal_proxy.rs +++ b/mirrord/cli/src/internal_proxy.rs @@ -11,12 +11,9 @@ //! or let the [`OperatorApi`](mirrord_operator::client::OperatorApi) handle the connection. use std::{ - env, - fs::OpenOptions, - io, + env, io, net::{Ipv4Addr, SocketAddr}, - path::PathBuf, - time::{Duration, SystemTime}, + time::Duration, }; use mirrord_analytics::{AnalyticsReporter, CollectAnalytics, Reporter}; @@ -28,15 +25,14 @@ use mirrord_intproxy::{ }; use mirrord_protocol::{ClientMessage, DaemonMessage, LogLevel, LogMessage}; use nix::sys::resource::{setrlimit, Resource}; -use rand::{distributions::Alphanumeric, Rng}; use tokio::net::TcpListener; use tracing::{warn, Level}; -use tracing_subscriber::EnvFilter; use crate::{ connection::AGENT_CONNECT_INFO_ENV_KEY, error::{CliResult, InternalProxyError}, execution::MIRRORD_EXECUTION_KIND_ENV, + logging::init_intproxy_tracing_registry, util::{create_listen_socket, detach_io}, }; @@ -56,44 +52,9 @@ pub(crate) async fn proxy( ) -> CliResult<(), InternalProxyError> { let config = LayerConfig::from_env()?; + init_intproxy_tracing_registry(&config)?; tracing::info!(?config, "internal_proxy starting"); - // Setting up default logging for intproxy. - let log_destination = config - .internal_proxy - .log_destination - .as_ref() - .map(PathBuf::from) - .unwrap_or_else(|| { - let random_name: String = rand::thread_rng() - .sample_iter(&Alphanumeric) - .take(7) - .map(char::from) - .collect(); - let timestamp = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs(); - - PathBuf::from(format!( - "/tmp/mirrord-intproxy-{timestamp}-{random_name}.log" - )) - }); - - let output_file = OpenOptions::new() - .create(true) - .append(true) - .open(&log_destination) - .map_err(|fail| { - InternalProxyError::OpenLogFile(log_destination.to_string_lossy().to_string(), fail) - })?; - - let log_level = config.internal_proxy.log_level.as_deref().unwrap_or("info"); - - tracing_subscriber::fmt() - .with_writer(output_file) - .with_ansi(false) - .with_env_filter(EnvFilter::builder().parse_lossy(log_level)) - .pretty() - .init(); - // According to https://wilsonmar.github.io/maximum-limits/ this is the limit on macOS // so we assume Linux can be higher and set to that. if let Err(error) = setrlimit(Resource::RLIMIT_NOFILE, 12288, 12288) { diff --git a/mirrord/cli/src/logging.rs b/mirrord/cli/src/logging.rs new file mode 100644 index 00000000000..8b9fba787ba --- /dev/null +++ b/mirrord/cli/src/logging.rs @@ -0,0 +1,192 @@ +use std::{ + fs::OpenOptions, + future::Future, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use futures::StreamExt; +use mirrord_config::LayerConfig; +use rand::{distributions::Alphanumeric, Rng}; +use tokio::io::AsyncWriteExt; +use tokio_stream::Stream; +use tracing_subscriber::{prelude::*, EnvFilter}; + +use crate::{ + config::Commands, + error::{CliError, ExternalProxyError, InternalProxyError}, +}; + +// only ls and ext commands need the errors in json format +// error logs are disabled for extensions +fn init_ext_error_handler(commands: &Commands) -> bool { + match commands { + Commands::ListTargets(_) | Commands::ExtensionExec(_) => { + let _ = miette::set_hook(Box::new(|_| Box::new(miette::JSONReportHandler::new()))); + + true + } + _ => false, + } +} + +pub async fn init_tracing_registry( + command: &Commands, + watch: drain::Watch, +) -> Result<(), CliError> { + if let Ok(console_addr) = std::env::var("MIRRORD_CONSOLE_ADDR") { + mirrord_console::init_async_logger(&console_addr, watch.clone(), 124).await?; + + return Ok(()); + } + + if matches!( + command, + Commands::InternalProxy { .. } | Commands::ExternalProxy { .. } + ) { + return Ok(()); + } + + // There are situations where even if running "ext" commands that shouldn't log, we want those + // to log to be able to debug issues. + let force_log = std::env::var("MIRRORD_FORCE_LOG") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(false); + + if force_log || init_ext_error_handler(command) { + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) + .with(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + } + + Ok(()) +} + +fn default_logfile_path(prefix: &str) -> PathBuf { + let random_name: String = rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(7) + .map(char::from) + .collect(); + let timestamp = SystemTime::UNIX_EPOCH.elapsed().unwrap().as_secs(); + + PathBuf::from(format!("/tmp/{prefix}-{timestamp}-{random_name}.log")) +} + +fn init_proxy_tracing_registry( + log_destination: &Path, + log_level: Option<&str>, +) -> std::io::Result<()> { + if std::env::var("MIRRORD_CONSOLE_ADDR").is_ok() { + return Ok(()); + } + + let output_file = OpenOptions::new() + .create(true) + .append(true) + .open(log_destination)?; + + let env_filter = log_level + .map(|log_level| EnvFilter::builder().parse_lossy(log_level)) + .unwrap_or_else(EnvFilter::from_default_env); + + tracing_subscriber::fmt() + .with_writer(output_file) + .with_ansi(false) + .with_env_filter(env_filter) + .pretty() + .init(); + + Ok(()) +} + +pub fn init_intproxy_tracing_registry(config: &LayerConfig) -> Result<(), InternalProxyError> { + if !config.internal_proxy.container_mode { + // Setting up default logging for intproxy. + let log_destination = config + .internal_proxy + .log_destination + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| default_logfile_path("mirrord-intproxy")); + + init_proxy_tracing_registry(&log_destination, config.internal_proxy.log_level.as_deref()) + .map_err(|fail| { + InternalProxyError::OpenLogFile(log_destination.to_string_lossy().to_string(), fail) + }) + } else { + let env_filter = config + .internal_proxy + .log_level + .as_ref() + .map(|log_level| EnvFilter::builder().parse_lossy(log_level)) + .unwrap_or_else(EnvFilter::from_default_env); + + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_ansi(false) + .with_env_filter(env_filter) + .pretty() + .init(); + + Ok(()) + } +} + +pub fn init_extproxy_tracing_registry(config: &LayerConfig) -> Result<(), ExternalProxyError> { + // Setting up default logging for extproxy. + let log_destination = config + .external_proxy + .log_destination + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| default_logfile_path("mirrord-extproxy")); + + init_proxy_tracing_registry(&log_destination, config.external_proxy.log_level.as_deref()) + .map_err(|fail| { + ExternalProxyError::OpenLogFile(log_destination.to_string_lossy().to_string(), fail) + }) +} + +pub async fn pipe_intproxy_sidecar_logs<'s, S>( + config: &LayerConfig, + stream: S, +) -> Result + 's, InternalProxyError> +where + S: Stream> + 's, +{ + let log_destination = config + .internal_proxy + .log_destination + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| default_logfile_path("mirrord-intproxy")); + + let mut output_file = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&log_destination) + .await + .map_err(|fail| { + InternalProxyError::OpenLogFile(log_destination.to_string_lossy().to_string(), fail) + })?; + + Ok(async move { + let mut stream = std::pin::pin!(stream); + + while let Some(line) = stream.next().await { + let result: std::io::Result<_> = try { + output_file.write_all(line?.as_bytes()).await?; + output_file.write_u8(b'\n').await?; + + output_file.flush().await?; + }; + + if let Err(error) = result { + tracing::error!(?error, "unable to pipe logs from intproxy"); + } + } + }) +} diff --git a/mirrord/cli/src/main.rs b/mirrord/cli/src/main.rs index 27923cab4fd..dde03d6a73e 100644 --- a/mirrord/cli/src/main.rs +++ b/mirrord/cli/src/main.rs @@ -18,7 +18,6 @@ use execution::MirrordExecution; use extension::extension_exec; use extract::extract_library; use kube::Client; -use miette::JSONReportHandler; use mirrord_analytics::{ AnalyticsError, AnalyticsReporter, CollectAnalytics, ExecutionKind, NullReporter, Reporter, }; @@ -45,7 +44,6 @@ use regex::Regex; use semver::{Version, VersionReq}; use serde_json::json; use tracing::{error, info, warn}; -use tracing_subscriber::{fmt, prelude::*, registry, EnvFilter}; use which::which; mod config; @@ -58,6 +56,7 @@ mod extension; mod external_proxy; mod extract; mod internal_proxy; +mod logging; mod operator; pub mod port_forward; mod teams; @@ -650,21 +649,8 @@ fn main() -> miette::Result<()> { let (signal, watch) = drain::channel(); - // There are situations where even if running "ext" commands that shouldn't log, we want those - // to log to be able to debug issues. - let force_log = std::env::var("MIRRORD_FORCE_LOG") - .map(|s| s.parse().unwrap_or(false)) - .unwrap_or(false); - let res: CliResult<(), CliError> = rt.block_on(async move { - if let Ok(console_addr) = std::env::var("MIRRORD_CONSOLE_ADDR") { - mirrord_console::init_async_logger(&console_addr, watch.clone(), 124).await?; - } else if force_log || !init_ext_error_handler(&cli.commands) { - registry() - .with(fmt::layer().with_writer(std::io::stderr)) - .with(EnvFilter::from_default_env()) - .init(); - } + logging::init_tracing_registry(&cli.commands, watch.clone()).await?; match cli.commands { Commands::Exec(args) => exec(&args, watch).await?, @@ -720,19 +706,6 @@ fn main() -> miette::Result<()> { res.map_err(Into::into) } -// only ls and ext commands need the errors in json format -// error logs are disabled for extensions -fn init_ext_error_handler(commands: &Commands) -> bool { - match commands { - Commands::ListTargets(_) | Commands::ExtensionExec(_) => { - let _ = miette::set_hook(Box::new(|_| Box::new(JSONReportHandler::new()))); - true - } - Commands::InternalProxy { .. } | Commands::ExternalProxy { .. } => true, - _ => false, - } -} - async fn prompt_outdated_version(progress: &ProgressTracker) { let mut progress = progress.subtask("version check"); let check_version: bool = std::env::var("MIRRORD_CHECK_VERSION") From eac208c141008c036c7fd5b1061a9bed1abfafc1 Mon Sep 17 00:00:00 2001 From: Aviram Hassan Date: Mon, 6 Jan 2025 20:41:19 +0200 Subject: [PATCH 03/23] use file buffering by default to improve performance (#3005) * use file buffering by default to improve performance * le change --- changelog.d/3004.changed.md | 1 + mirrord/config/src/experimental.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/3004.changed.md diff --git a/changelog.d/3004.changed.md b/changelog.d/3004.changed.md new file mode 100644 index 00000000000..4a6367bf967 --- /dev/null +++ b/changelog.d/3004.changed.md @@ -0,0 +1 @@ +use file buffering by default to improve performance diff --git a/mirrord/config/src/experimental.rs b/mirrord/config/src/experimental.rs index 1fd9213a762..302cb14fc27 100644 --- a/mirrord/config/src/experimental.rs +++ b/mirrord/config/src/experimental.rs @@ -68,7 +68,7 @@ pub struct ExperimentalConfig { /// Setting to 0 disables file buffering. /// /// - #[config(default = 0)] + #[config(default = 128000)] pub readonly_file_buffer: u64, } From 4f5cff8d6417487d8ed114758eff0a9d86a00496 Mon Sep 17 00:00:00 2001 From: meowjesty <43983236+meowjesty@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:47:36 -0300 Subject: [PATCH 04/23] Change misleading error when agent fails IO operation. (#3001) * Change misleading error when agent fails IO operation. * trace and debug * changelog * protocol * improve error with display impl --- Cargo.lock | 2 +- changelog.d/2992.fixed.md | 1 + mirrord/agent/src/entrypoint.rs | 4 ++-- mirrord/agent/src/file.rs | 8 ++++---- mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/error.rs | 17 +++++++++++++++-- 6 files changed, 24 insertions(+), 10 deletions(-) create mode 100644 changelog.d/2992.fixed.md diff --git a/Cargo.lock b/Cargo.lock index 2cfeea353a5..bcd9d13b8a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4453,7 +4453,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.13.1" +version = "1.13.2" dependencies = [ "actix-codec", "bincode", diff --git a/changelog.d/2992.fixed.md b/changelog.d/2992.fixed.md new file mode 100644 index 00000000000..0e907384f90 --- /dev/null +++ b/changelog.d/2992.fixed.md @@ -0,0 +1 @@ +Fix misleading agent IO operation error that always mentioned getaddrinfo. diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index 28eb5f26634..f345fa0d5c9 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -24,7 +24,7 @@ use tokio::{ time::{timeout, Duration}, }; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn, Level}; use tracing_subscriber::{fmt::format::FmtSpan, prelude::*}; use crate::{ @@ -396,7 +396,7 @@ impl ClientConnectionHandler { /// Handles incoming messages from the connected client (`mirrord-layer`). /// /// Returns `false` if the client disconnected. - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] async fn handle_client_message(&mut self, message: ClientMessage) -> Result { match message { ClientMessage::FileRequest(req) => { diff --git a/mirrord/agent/src/file.rs b/mirrord/agent/src/file.rs index 9261b0bcb69..54432a9779b 100644 --- a/mirrord/agent/src/file.rs +++ b/mirrord/agent/src/file.rs @@ -147,7 +147,7 @@ pub fn resolve_path + std::fmt::Debug, R: AsRef + std::fmt: impl FileManager { /// Executes the request and returns the response. - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] pub fn handle_message(&mut self, request: FileRequest) -> Result> { Ok(match request { FileRequest::Open(OpenFileRequest { path, open_options }) => { @@ -261,7 +261,7 @@ impl FileManager { }) } - #[tracing::instrument(level = "trace")] + #[tracing::instrument(level = Level::TRACE, ret)] pub fn new(pid: Option) -> Self { let root_path = get_root_path_from_optional_pid(pid); trace!("Agent root path >> {root_path:?}"); @@ -272,7 +272,7 @@ impl FileManager { } } - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] fn open( &mut self, path: PathBuf, @@ -299,7 +299,7 @@ impl FileManager { Ok(OpenFileResponse { fd }) } - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] fn open_relative( &mut self, relative_fd: u64, diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 70f33186ba1..34832bbe47f 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.13.1" +version = "1.13.2" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/error.rs b/mirrord/protocol/src/error.rs index 20ac38a149d..9a8e451658c 100644 --- a/mirrord/protocol/src/error.rs +++ b/mirrord/protocol/src/error.rs @@ -153,13 +153,26 @@ impl From for RemoteError { /// Our internal version of Rust's `std::io::Error` that can be passed between mirrord-layer and /// mirrord-agent. -#[derive(Encode, Decode, Debug, PartialEq, Clone, Eq, Error)] -#[error("Failed performing `getaddrinfo` with {raw_os_error:?} and kind {kind:?}!")] +/// +/// ### `Display` +/// +/// We manually implement `Display` as this error is mostly seen from a [`ResponseError`] context. +#[derive(Encode, Decode, Debug, PartialEq, Clone, Eq)] pub struct RemoteIOError { pub raw_os_error: Option, pub kind: ErrorKindInternal, } +impl core::fmt::Display for RemoteIOError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self.kind)?; + if let Some(code) = self.raw_os_error { + write!(f, " (error code {code})")?; + } + Ok(()) + } +} + /// Our internal version of Rust's `std::io::Error` that can be passed between mirrord-layer and /// mirrord-agent. /// From 2ec5ab35e73b5cc1ea82fa51b289bb687e0680d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:05:28 +0100 Subject: [PATCH 05/23] Add namespaces to `mirrord ls` output (#3003) * Additional command flag, rich output skeleton * listing functions extracted * Added namespaces to the output * mirrord_cli::list docs * mirrord_kube::api::kubernetes::list docs * Removed dead code * flag doc * Changelog * Moved output control to env var * Clippy * Reduce print_targets visibility, change display wrapper name * FoundTargetsList doc * Expect message fixed * print_targets doc * Move listing utils to KubeResourceSeeker --- changelog.d/2999.added.md | 1 + mirrord/cli/src/config.rs | 9 +- mirrord/cli/src/list.rs | 215 ++++++++++++++++++++++ mirrord/cli/src/main.rs | 123 ++----------- mirrord/cli/src/port_forward.rs | 22 +-- mirrord/kube/src/api/kubernetes/seeker.rs | 140 +++++++++----- 6 files changed, 335 insertions(+), 175 deletions(-) create mode 100644 changelog.d/2999.added.md create mode 100644 mirrord/cli/src/list.rs diff --git a/changelog.d/2999.added.md b/changelog.d/2999.added.md new file mode 100644 index 00000000000..ac9b5676f4d --- /dev/null +++ b/changelog.d/2999.added.md @@ -0,0 +1 @@ +Added available namespaces to `mirrord ls` output. New output format is enabled with `--rich` flag. diff --git a/mirrord/cli/src/config.rs b/mirrord/cli/src/config.rs index 367504ddbf4..96548235ad4 100644 --- a/mirrord/cli/src/config.rs +++ b/mirrord/cli/src/config.rs @@ -712,11 +712,18 @@ pub(super) struct ListTargetArgs { #[arg(short = 'n', long = "namespace")] pub namespace: Option, - /// Specify config file to use + /// Specify config file to use. #[arg(short = 'f', long, value_hint = ValueHint::FilePath)] pub config_file: Option, } +impl ListTargetArgs { + /// Controls the output of `mirrord ls`. + /// If set to `true`, the command outputs a JSON object that contains more data. + /// Otherwise, it outputs a plain array of target paths. + pub(super) const RICH_OUTPUT_ENV: &str = "MIRRORD_LS_RICH_OUTPUT"; +} + #[derive(Args, Debug)] pub(super) struct ExtensionExecArgs { /// Specify config file to use diff --git a/mirrord/cli/src/list.rs b/mirrord/cli/src/list.rs new file mode 100644 index 00000000000..2b8059bad62 --- /dev/null +++ b/mirrord/cli/src/list.rs @@ -0,0 +1,215 @@ +use std::sync::LazyLock; + +use futures::TryStreamExt; +use k8s_openapi::api::core::v1::Namespace; +use kube::Client; +use mirrord_analytics::NullReporter; +use mirrord_config::{ + config::{ConfigContext, MirrordConfig}, + LayerConfig, LayerFileConfig, +}; +use mirrord_kube::{ + api::kubernetes::{create_kube_config, seeker::KubeResourceSeeker}, + error::KubeApiError, +}; +use mirrord_operator::client::OperatorApi; +use semver::VersionReq; +use serde::{ser::SerializeSeq, Serialize, Serializer}; + +use crate::{util, CliError, CliResult, Format, ListTargetArgs}; + +/// A mirrord target found in the cluster. +#[derive(Serialize)] +struct FoundTarget { + /// E.g `pod/my-pod-1234/container/my-container`. + path: String, + + /// Whether this target is currently available. + /// + /// # Note + /// + /// Right now this is always true. Some preliminary checks are done in the + /// [`KubeResourceSeeker`] and results come filtered. + /// + /// This field is here for forward compatibility, because in the future we might want to return + /// unavailable targets as well (along with some validation error message) to improve UX. + available: bool, +} + +/// Result of mirrord targets lookup in the cluster. +#[derive(Serialize)] +struct FoundTargets { + /// In order: + /// 1. deployments + /// 2. rollouts + /// 3. statefulsets + /// 4. cronjobs + /// 5. jobs + /// 6. pods + targets: Vec, + + /// Current lookup namespace. + /// + /// Taken from [`LayerConfig::target`], defaults to [`Client`]'s default namespace. + current_namespace: String, + + /// Available lookup namespaces. + namespaces: Vec, +} + +impl FoundTargets { + /// Performs a lookup of mirrord targets in the cluster. + /// + /// Unless the operator is explicitly disabled, attempts to connect with it. + /// Operator lookup affects returned results (e.g some targets are only available via the + /// operator). + /// + /// If `fetch_namespaces` is set, returned [`FoundTargets`] will contain info about namespaces + /// available in the cluster. + async fn resolve(config: LayerConfig, fetch_namespaces: bool) -> CliResult { + let client = create_kube_config( + config.accept_invalid_certificates, + config.kubeconfig.clone(), + config.kube_context.clone(), + ) + .await + .and_then(|config| Client::try_from(config).map_err(From::from)) + .map_err(|error| { + CliError::friendlier_error_or_else(error, CliError::CreateKubeApiFailed) + })?; + + let mut reporter = NullReporter::default(); + let operator_api = if config.operator != Some(false) + && let Some(api) = OperatorApi::try_new(&config, &mut reporter).await? + { + let api = api.prepare_client_cert(&mut reporter).await; + + api.inspect_cert_error( + |error| tracing::error!(%error, "failed to prepare client certificate"), + ); + + Some(api) + } else { + None + }; + + let seeker = KubeResourceSeeker { + client: &client, + namespace: config.target.namespace.as_deref(), + }; + let paths = match operator_api { + None if config.operator == Some(true) => Err(CliError::OperatorNotInstalled), + + Some(api) + if ALL_TARGETS_SUPPORTED_OPERATOR_VERSION + .matches(&api.operator().spec.operator_version) => + { + seeker.all().await.map_err(|error| { + CliError::friendlier_error_or_else(error, CliError::ListTargetsFailed) + }) + } + + _ => seeker.all_open_source().await.map_err(|error| { + CliError::friendlier_error_or_else(error, CliError::ListTargetsFailed) + }), + }?; + + let targets = paths + .into_iter() + .map(|path| FoundTarget { + path, + available: true, + }) + .collect(); + let current_namespace = config + .target + .namespace + .as_deref() + .unwrap_or(client.default_namespace()) + .to_owned(); + + let namespaces = if fetch_namespaces { + seeker + .list_all_clusterwide::(None) + .try_filter_map(|namespace| std::future::ready(Ok(namespace.metadata.name))) + .try_collect::>() + .await + .map_err(KubeApiError::KubeError) + .map_err(|error| { + CliError::friendlier_error_or_else(error, CliError::ListTargetsFailed) + })? + } else { + Default::default() + }; + + Ok(Self { + targets, + current_namespace, + namespaces, + }) + } +} + +/// Thin wrapper over [`FoundTargets`] that implements [`Serialize`]. +/// Its serialized format is a sequence of available target paths. +/// +/// Used to print available targets when the plugin/extension does not support the full format +/// (backward compatibility). +struct FoundTargetsList<'a>(&'a FoundTargets); + +impl Serialize for FoundTargetsList<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let count = self.0.targets.iter().filter(|t| t.available).count(); + let mut list = serializer.serialize_seq(Some(count))?; + + for target in self.0.targets.iter().filter(|t| t.available) { + list.serialize_element(&target.path)?; + } + + list.end() + } +} + +/// Controls whether we support listing all targets or just the open source ones. +static ALL_TARGETS_SUPPORTED_OPERATOR_VERSION: LazyLock = + LazyLock::new(|| ">=3.84.0".parse().expect("version should be valid")); + +/// Fetches mirrord targets from the cluster and prints output to stdout. +/// +/// When `rich_output` is set, targets info is printed as a JSON object containing extra data. +/// Otherwise, targets are printed as a plain JSON array of strings (backward compatibility). +pub(super) async fn print_targets(args: ListTargetArgs, rich_output: bool) -> CliResult<()> { + let mut layer_config = if let Some(config) = &args.config_file { + let mut cfg_context = ConfigContext::default(); + LayerFileConfig::from_path(config)?.generate_config(&mut cfg_context)? + } else { + LayerConfig::from_env()? + }; + + if let Some(namespace) = args.namespace { + layer_config.target.namespace.replace(namespace); + }; + + if !layer_config.use_proxy { + util::remove_proxy_env(); + } + + let targets = FoundTargets::resolve(layer_config, rich_output).await?; + + match args.output { + Format::Json => { + let serialized = if rich_output { + serde_json::to_string(&targets).unwrap() + } else { + serde_json::to_string(&FoundTargetsList(&targets)).unwrap() + }; + + println!("{serialized}"); + } + } + + Ok(()) +} diff --git a/mirrord/cli/src/main.rs b/mirrord/cli/src/main.rs index dde03d6a73e..4631af499dc 100644 --- a/mirrord/cli/src/main.rs +++ b/mirrord/cli/src/main.rs @@ -5,7 +5,7 @@ use std::{ collections::HashMap, env::vars, ffi::CString, net::SocketAddr, os::unix::ffi::OsStrExt, - sync::LazyLock, time::Duration, + time::Duration, }; use clap::{CommandFactory, Parser}; @@ -17,12 +17,10 @@ use diagnose::diagnose_command; use execution::MirrordExecution; use extension::extension_exec; use extract::extract_library; -use kube::Client; use mirrord_analytics::{ - AnalyticsError, AnalyticsReporter, CollectAnalytics, ExecutionKind, NullReporter, Reporter, + AnalyticsError, AnalyticsReporter, CollectAnalytics, ExecutionKind, Reporter, }; use mirrord_config::{ - config::{ConfigContext, MirrordConfig}, feature::{ fs::FsModeConfig, network::{ @@ -33,16 +31,13 @@ use mirrord_config::{ LayerConfig, LayerFileConfig, MIRRORD_CONFIG_FILE_ENV, }; use mirrord_intproxy::agent_conn::{AgentConnection, AgentConnectionError}; -use mirrord_kube::api::kubernetes::{create_kube_config, seeker::KubeResourceSeeker}; -use mirrord_operator::client::OperatorApi; use mirrord_progress::{messages::EXEC_CONTAINER_BINARY, Progress, ProgressTracker}; #[cfg(all(target_os = "macos", target_arch = "aarch64"))] use nix::errno::Errno; use operator::operator_command; use port_forward::{PortForwardError, PortForwarder, ReversePortForwarder}; use regex::Regex; -use semver::{Version, VersionReq}; -use serde_json::json; +use semver::Version; use tracing::{error, info, warn}; use which::which; @@ -56,9 +51,10 @@ mod extension; mod external_proxy; mod extract; mod internal_proxy; +mod list; mod logging; mod operator; -pub mod port_forward; +mod port_forward; mod teams; mod util; mod verify_config; @@ -67,12 +63,6 @@ mod vpn; pub(crate) use error::{CliError, CliResult}; use verify_config::verify_config; -use crate::util::remove_proxy_env; - -/// Controls whether we support listing all targets or just the open source ones. -static ALL_TARGETS_SUPPORTED_OPERATOR_VERSION: LazyLock = - LazyLock::new(|| ">=3.84.0".parse().expect("verion should be valid")); - async fn exec_process

( config: LayerConfig, args: &ExecArgs, @@ -381,100 +371,6 @@ async fn exec(args: &ExecArgs, watch: drain::Watch) -> CliResult<()> { execution_result } -/// Lists targets based on whether or not the operator has been enabled in `layer_config`. -/// If the operator is enabled (and we can reach it), then we list [`KubeResourceSeeker::all`] -/// targets, otherwise we list [`KubeResourceSeeker::all_open_source`] only. -async fn list_targets(layer_config: &LayerConfig, args: &ListTargetArgs) -> CliResult> { - let client = create_kube_config( - layer_config.accept_invalid_certificates, - layer_config.kubeconfig.clone(), - layer_config.kube_context.clone(), - ) - .await - .and_then(|config| Client::try_from(config).map_err(From::from)) - .map_err(|error| CliError::friendlier_error_or_else(error, CliError::CreateKubeApiFailed))?; - - let namespace = args - .namespace - .as_deref() - .or(layer_config.target.namespace.as_deref()); - - let seeker = KubeResourceSeeker { - client: &client, - namespace, - }; - - let mut reporter = NullReporter::default(); - - let operator_api = if layer_config.operator != Some(false) - && let Some(api) = OperatorApi::try_new(layer_config, &mut reporter).await? - { - let api = api.prepare_client_cert(&mut reporter).await; - - api.inspect_cert_error( - |error| tracing::error!(%error, "failed to prepare client certificate"), - ); - - Some(api) - } else { - None - }; - - match operator_api { - None if layer_config.operator == Some(true) => Err(CliError::OperatorNotInstalled), - Some(api) - if ALL_TARGETS_SUPPORTED_OPERATOR_VERSION - .matches(&api.operator().spec.operator_version) => - { - seeker.all().await.map_err(|error| { - CliError::friendlier_error_or_else(error, CliError::ListTargetsFailed) - }) - } - _ => seeker.all_open_source().await.map_err(|error| { - CliError::friendlier_error_or_else(error, CliError::ListTargetsFailed) - }), - } -} - -/// Lists all possible target paths. -/// Tries to use operator if available, otherwise falls back to k8s API (if operator isn't -/// explicitly true). Example: -/// ``` -/// [ -/// "pod/metalbear-deployment-85c754c75f-982p5", -/// "pod/nginx-deployment-66b6c48dd5-dc9wk", -/// "pod/py-serv-deployment-5c57fbdc98-pdbn4/container/py-serv", -/// "deployment/nginx-deployment" -/// "deployment/nginx-deployment/container/nginx" -/// "rollout/nginx-rollout" -/// "statefulset/nginx-statefulset" -/// "statefulset/nginx-statefulset/container/nginx" -/// ] -/// ``` -async fn print_targets(args: &ListTargetArgs) -> CliResult<()> { - let mut layer_config = if let Some(config) = &args.config_file { - let mut cfg_context = ConfigContext::default(); - LayerFileConfig::from_path(config)?.generate_config(&mut cfg_context)? - } else { - LayerConfig::from_env()? - }; - - if let Some(namespace) = &args.namespace { - layer_config.target.namespace = Some(namespace.clone()); - }; - - if !layer_config.use_proxy { - remove_proxy_env(); - } - - // The targets come sorted in the following order: - // `deployments - rollouts - statefulsets - cronjobs - jobs - pods` - let targets = list_targets(&layer_config, args).await?; - let json_obj = json!(targets); - println!("{json_obj}"); - Ok(()) -} - async fn port_forward(args: &PortForwardArgs, watch: drain::Watch) -> CliResult<()> { fn hash_port_mappings( args: &PortForwardArgs, @@ -661,7 +557,14 @@ fn main() -> miette::Result<()> { false, )?; } - Commands::ListTargets(args) => print_targets(&args).await?, + Commands::ListTargets(args) => { + let rich_output = std::env::var(ListTargetArgs::RICH_OUTPUT_ENV) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or_default(); + + list::print_targets(*args, rich_output).await? + } Commands::Operator(args) => operator_command(*args).await?, Commands::ExtensionExec(args) => { extension_exec(*args, watch).await?; diff --git a/mirrord/cli/src/port_forward.rs b/mirrord/cli/src/port_forward.rs index ba59679d945..002a904ae1b 100644 --- a/mirrord/cli/src/port_forward.rs +++ b/mirrord/cli/src/port_forward.rs @@ -29,8 +29,7 @@ use mirrord_protocol::{ LayerClose, LayerConnect, LayerWrite, SocketAddress, }, tcp::{Filter, HttpFilter, LayerTcp, LayerTcpSteal, StealType}, - ClientMessage, ConnectionId, DaemonMessage, LogLevel, Port, ResponseError, - CLIENT_READY_FOR_LOGS, + ClientMessage, ConnectionId, DaemonMessage, LogLevel, Port, CLIENT_READY_FOR_LOGS, }; use thiserror::Error; use tokio::{ @@ -969,19 +968,12 @@ impl IncomingMode { #[derive(Debug, Error)] pub enum PortForwardError { - // setup errors - #[error("wrong combination of arguments used: {0}")] - ArgsError(String), - #[error("multiple port forwarding mappings found for local address `{0}`")] PortMapSetupError(SocketAddr), #[error("multiple port forwarding mappings found for desination port `{0:?}`")] ReversePortMapSetupError(RemotePort), - #[error("no port forwarding mappings were provided")] - NoMappingsError(), - // running errors #[error("agent closed connection with error: `{0}`")] AgentError(String), @@ -992,18 +984,9 @@ pub enum PortForwardError { #[error("error from Incoming Proxy task")] IncomingProxyError(IntProxyError), - #[error("failed to send Ping to agent: `{0}`")] - PingError(String), - #[error("TcpListener operation failed with error: `{0}`")] TcpListenerError(std::io::Error), - #[error("TcpStream operation failed with error: `{0}`")] - TcpStreamError(std::io::Error), - - #[error("no destination address found for local address `{0}`")] - SocketMappingNotFound(SocketAddr), - #[error("no task for socket {0} ready to receive connection ID: `{1}`")] ReadyTaskNotFound(SocketAddr, ConnectionId), @@ -1012,9 +995,6 @@ pub enum PortForwardError { #[error("failed to establish connection with remote process: `{0}`")] ConnectionError(String), - - #[error("failed to subscribe to remote port: `{0}`")] - SubscriptionError(ResponseError), } impl From> for PortForwardError { diff --git a/mirrord/kube/src/api/kubernetes/seeker.rs b/mirrord/kube/src/api/kubernetes/seeker.rs index b9429610f8e..e5208fcd8b5 100644 --- a/mirrord/kube/src/api/kubernetes/seeker.rs +++ b/mirrord/kube/src/api/kubernetes/seeker.rs @@ -1,6 +1,5 @@ use std::fmt; -use async_stream::stream; use futures::{stream, Stream, StreamExt, TryStreamExt}; use k8s_openapi::{ api::{ @@ -8,17 +7,17 @@ use k8s_openapi::{ batch::v1::{CronJob, Job}, core::v1::Pod, }, - Metadata, NamespaceResourceScope, + ClusterResourceScope, Metadata, NamespaceResourceScope, }; -use kube::{api::ListParams, Resource}; -use serde::de; +use kube::{api::ListParams, Api, Resource}; +use serde::de::{self, DeserializeOwned}; use crate::{ api::{ container::SKIP_NAMES, kubernetes::{get_k8s_resource_api, rollout::Rollout}, }, - error::Result, + error::{KubeApiError, Result}, }; pub struct KubeResourceSeeker<'a> { @@ -95,7 +94,7 @@ impl KubeResourceSeeker<'_> { Some((name, containers)) } - self.list_resource::(Some("status.phase=Running")) + self.list_all_namespaced(Some("status.phase=Running")) .try_filter(|pod| std::future::ready(check_pod_status(pod))) .try_filter_map(|pod| std::future::ready(Ok(create_pod_container_map(pod)))) .map_ok(|(pod, containers)| { @@ -111,6 +110,7 @@ impl KubeResourceSeeker<'_> { .try_flatten() .try_collect() .await + .map_err(KubeApiError::KubeError) } /// The list of deployments that have at least 1 `Replicas` and a deployment name. @@ -123,7 +123,7 @@ impl KubeResourceSeeker<'_> { .unwrap_or(false) } - self.list_resource::(None) + self.list_all_namespaced::(None) .filter(|response| std::future::ready(response.is_ok())) .try_filter(|deployment| std::future::ready(check_deployment_replicas(deployment))) .try_filter_map(|deployment| { @@ -134,60 +134,114 @@ impl KubeResourceSeeker<'_> { }) .try_collect() .await + .map_err(From::from) } - /// Helper to get the list of a resource type ([`Pod`], [`Deployment`], [`Rollout`], [`Job`], - /// [`CronJob`], [`StatefulSet`], or whatever satisfies `R`) through the kube api. - fn list_resource<'s, R>( - &self, - field_selector: Option<&'s str>, - ) -> impl Stream> + 's + async fn simple_list_resource<'s, R>(&self, prefix: &'s str) -> Result> where - R: Clone + fmt::Debug + for<'de> de::Deserialize<'de> + 's, - R: Resource, + R: 'static + + Clone + + fmt::Debug + + for<'de> de::Deserialize<'de> + + Resource + + Metadata + + Send, { - let Self { client, namespace } = self; - let resource_api = get_k8s_resource_api::(client, *namespace); + self.list_all_namespaced::(None) + .filter(|response| std::future::ready(response.is_ok())) + .try_filter_map(|rollout| { + std::future::ready(Ok(rollout + .meta() + .name + .as_ref() + .map(|name| format!("{prefix}/{name}")))) + }) + .try_collect() + .await + .map_err(From::from) + } + + /// Prepares [`ListParams`] that: + /// 1. Excludes our own resources + /// 2. Adds a limit for item count in a response + fn make_list_params(field_selector: Option<&str>) -> ListParams { + ListParams { + label_selector: Some("app!=mirrord,!operator.metalbear.co/owner".to_string()), + field_selector: field_selector.map(ToString::to_string), + limit: Some(500), + ..Default::default() + } + } - stream! { - let mut params = ListParams { - label_selector: Some("app!=mirrord,!operator.metalbear.co/owner".to_string()), - field_selector: field_selector.map(ToString::to_string), - limit: Some(500), - ..Default::default() - }; + /// Returns a [`Stream`] of all objects in this [`KubeResourceSeeker`]'s namespace. + /// + /// 1. `field_selector` can be used for filtering. + /// 2. Our own resources are excluded. + pub fn list_all_namespaced( + &self, + field_selector: Option<&str>, + ) -> impl 'static + Stream> + Send + where + R: 'static + + Resource + + fmt::Debug + + Clone + + DeserializeOwned + + Send, + { + let api = get_k8s_resource_api(self.client, self.namespace); + let mut params = Self::make_list_params(field_selector); + async_stream::stream! { loop { - let resource = resource_api.list(¶ms).await?; + let response = api.list(¶ms).await?; - for resource in resource.items { + for resource in response.items { yield Ok(resource); } - if let Some(continue_token) = resource.metadata.continue_ && !continue_token.is_empty() { - params = params.continue_token(&continue_token); - } else { + let continue_token = response.metadata.continue_.unwrap_or_default(); + if continue_token.is_empty() { break; } + params.continue_token.replace(continue_token); } } } - async fn simple_list_resource<'s, R>(&self, prefix: &'s str) -> Result> + /// Returns a [`Stream`] of all objects in the cluster. + /// + /// 1. `field_selector` can be used for filtering. + /// 2. Our own resources are excluded. + pub fn list_all_clusterwide( + &self, + field_selector: Option<&str>, + ) -> impl 'static + Stream> + Send where - R: Clone + fmt::Debug + for<'de> de::Deserialize<'de>, - R: Resource + Metadata, + R: 'static + + Resource + + fmt::Debug + + Clone + + DeserializeOwned + + Send, { - self.list_resource::(None) - .filter(|response| std::future::ready(response.is_ok())) - .try_filter_map(|rollout| { - std::future::ready(Ok(rollout - .meta() - .name - .as_ref() - .map(|name| format!("{prefix}/{name}")))) - }) - .try_collect() - .await + let api = Api::all(self.client.clone()); + let mut params = Self::make_list_params(field_selector); + + async_stream::stream! { + loop { + let response = api.list(¶ms).await?; + + for resource in response.items { + yield Ok(resource); + } + + let continue_token = response.metadata.continue_.unwrap_or_default(); + if continue_token.is_empty() { + break; + } + params.continue_token.replace(continue_token); + } + } } } From a51e981ac6f54fa04c978e326acb3a570d53319e Mon Sep 17 00:00:00 2001 From: meowjesty <43983236+meowjesty@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:18:11 -0300 Subject: [PATCH 06/23] Add policy for file ops. (#2997) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add policy to exclude env vars. * docs * changelog * fix tests * Make policy optional for backwards compat. * typo * other typo * default serde and more explicit docs * fix mismatching policy * make it look more like config * make it pub * fix test * better docs Co-authored-by: MichaÅ‚ Smolarek <34063647+Razz4780@users.noreply.github.com> * better docs due Co-authored-by: MichaÅ‚ Smolarek <34063647+Razz4780@users.noreply.github.com> * rustfmt * Add policy for file ops. * no exclude * update protocol with open_local_version * bump protocol * lint test * docs * fix test * change min protocol version * e2e test for fspolicy * Ignore fs policy test/ * namespaced test * hopefully fixed policy test * the children get to live * fix test policy name Co-authored-by: t4lz * fix go fs test * remove read_write * add newline to python test * changelog --------- Co-authored-by: MichaÅ‚ Smolarek <34063647+Razz4780@users.noreply.github.com> Co-authored-by: t4lz --- Cargo.lock | 377 ++++++++++-------- changelog.d/+104-policy-fs.added.md | 1 + mirrord/layer/src/detour.rs | 4 + mirrord/layer/src/error.rs | 1 + mirrord/layer/src/file/ops.rs | 7 +- mirrord/operator/src/crd/policy.rs | 34 ++ mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/error.rs | 3 + mirrord/protocol/src/file.rs | 3 + tests/go-e2e-dir/main.go | 15 +- .../fspolicy/test_operator_fs_policy.mjs | 54 +++ tests/python-e2e/files_ro.py | 4 +- tests/python-e2e/ops.py | 2 +- tests/src/operator/policies.rs | 9 + tests/src/operator/policies/fs.rs | 87 ++++ tests/src/utils.rs | 8 + 16 files changed, 437 insertions(+), 174 deletions(-) create mode 100644 changelog.d/+104-policy-fs.added.md create mode 100644 tests/node-e2e/fspolicy/test_operator_fs_policy.mjs create mode 100644 tests/src/operator/policies/fs.rs diff --git a/Cargo.lock b/Cargo.lock index bcd9d13b8a7..89528a03ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -182,7 +182,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -440,7 +440,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "synstructure 0.13.1", ] @@ -463,7 +463,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -508,7 +508,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -540,7 +540,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -551,13 +551,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -583,9 +583,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.12" +version = "1.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "649316840239f4e58df0b7f620c428f5fababbbca2d504488c641534050bd141" +checksum = "c03a50b30228d3af8865ce83376b4e99e1ffa34728220fe2860e4df0bb5278d6" dependencies = [ "aws-credential-types", "aws-runtime", @@ -650,9 +650,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f6f1124d6e19ab6daf7f2e615644305dc6cb2d706892a8a8c0b98db35de020" +checksum = "b16d1aa50accc11a4b4d5c50f7fb81cc0cf60328259c587d0e6b0f11385bde46" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -675,9 +675,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.52.0" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9466fd797274f6c55454ea5aac51fc55a9bac6ca2116ed32cfb3134bb3fbcf0" +checksum = "6493ce2b27a2687b0d8a2453bf6ad2499012e9720c3367cb1206496ede475443" dependencies = [ "aws-credential-types", "aws-runtime", @@ -697,9 +697,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.52.0" +version = "1.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb25f7129c74d36afe33405af4517524df8f74b635af8c2c8e91c1552b8397b2" +checksum = "1605dc0bf9f0a4b05b451441a17fcb0bda229db384f23bf5cead3adbab0664ac" dependencies = [ "aws-credential-types", "aws-runtime", @@ -719,9 +719,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.53.0" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03a3d5ef14851625eafd89660a751776f938bf32f309308b20dcca41c44b568" +checksum = "59f3f73466ff24f6ad109095e0f3f2c830bfb4cd6c8b12f744c8e61ebf4d3ba1" dependencies = [ "aws-credential-types", "aws-runtime", @@ -741,9 +741,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.53.0" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3a9f073ae3a53b54421503063dfb87ff1ea83b876f567d92e8b8d9942ba91b" +checksum = "249b2acaa8e02fd4718705a9494e3eb633637139aa4bb09d70965b0448e865db" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1109,7 +1109,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.93", + "syn 2.0.95", "which 4.4.2", ] @@ -1269,9 +1269,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "serde", @@ -1380,9 +1380,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d6dbb628b8f8555f86d0323c2eb39e3ec81901f4b83e091db8a6a76d316a333" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "jobserver", "libc", @@ -1470,9 +1470,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "9560b07a799281c7e0958b9296854d6fafd4c5f31444a7e5bb1ad6dde5ccf1bd" dependencies = [ "clap_builder", "clap_derive", @@ -1480,9 +1480,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "874e0dd3eb68bf99058751ac9712f622e61e6f393a94f7128fa26e3f02f5c7cd" dependencies = [ "anstream", "anstyle", @@ -1492,23 +1492,23 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2e663e3e3bed2d32d065a8404024dad306e699a04263ec59919529f803aee9" +checksum = "942dc5991a34d8cf58937ec33201856feba9cbceeeab5adf04116ec7c763bff1" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1789,7 +1789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1821,7 +1821,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1845,7 +1845,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1856,7 +1856,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1932,7 +1932,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1942,7 +1942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -1955,7 +1955,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2054,7 +2054,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2116,7 +2116,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2168,7 +2168,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2188,7 +2188,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2200,7 +2200,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2466,7 +2466,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcca6a476beb610ebeab9c0bc60b3535aa103b52a2c265dc9d3d26209bea666c" dependencies = [ - "reqwest 0.12.11", + "reqwest 0.12.12", "tar", "xz", ] @@ -2585,7 +2585,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -2624,6 +2624,19 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -2657,7 +2670,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -3204,7 +3217,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -3331,7 +3344,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -3771,7 +3784,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -3913,6 +3926,19 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -3962,7 +3988,7 @@ dependencies = [ "clap", "glob", "rand", - "syn 2.0.93", + "syn 2.0.95", "thiserror 2.0.9", "tracing", "tracing-subscriber", @@ -4030,7 +4056,7 @@ checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -4123,7 +4149,7 @@ dependencies = [ "rand", "rcgen", "regex", - "reqwest 0.12.11", + "reqwest 0.12.12", "rstest", "rustls 0.23.20", "rustls-pemfile 2.2.0", @@ -4204,7 +4230,7 @@ dependencies = [ "assert-json-diff", "base64 0.22.1", "drain", - "reqwest 0.12.11", + "reqwest 0.12.12", "serde", "serde_json", "tokio", @@ -4222,7 +4248,7 @@ dependencies = [ "k8s-openapi", "kube", "pem", - "reqwest 0.12.11", + "reqwest 0.12.12", "serde", "serde_yaml", "thiserror 2.0.9", @@ -4262,7 +4288,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -4299,7 +4325,7 @@ dependencies = [ "mirrord-operator", "mirrord-protocol", "rand", - "reqwest 0.12.11", + "reqwest 0.12.12", "rstest", "rustls 0.23.20", "rustls-pemfile 2.2.0", @@ -4395,7 +4421,7 @@ version = "3.128.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -4405,7 +4431,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "semver 1.0.24", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -4453,7 +4479,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.13.2" +version = "1.13.3" dependencies = [ "actix-codec", "bincode", @@ -4530,26 +4556,25 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] name = "moka" -version = "0.12.8" +version = "0.12.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" dependencies = [ "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", - "once_cell", + "loom", "parking_lot", - "quanta", + "portable-atomic", "rustc_version", "smallvec", "tagptr", "thiserror 1.0.69", - "triomphe", "uuid", ] @@ -4704,7 +4729,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -4952,7 +4977,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5011,7 +5036,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5037,18 +5062,18 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -5056,9 +5081,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", "rand", @@ -5066,38 +5091,38 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -5207,7 +5232,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5302,12 +5327,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" dependencies = [ "proc-macro2", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5361,7 +5386,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5381,7 +5406,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "version_check", "yansi", ] @@ -5437,7 +5462,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.93", + "syn 2.0.95", "tempfile", ] @@ -5451,7 +5476,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -5463,21 +5488,6 @@ dependencies = [ "prost", ] -[[package]] -name = "quanta" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773ce68d0bb9bc7ef20be3536ffe94e223e1f365bd374108b2659fac0c65cfe6" -dependencies = [ - "crossbeam-utils", - "libc", - "once_cell", - "raw-cpuid", - "wasi", - "web-sys", - "winapi", -] - [[package]] name = "quick-error" version = "1.2.3" @@ -5628,15 +5638,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "raw-cpuid" -version = "11.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0" -dependencies = [ - "bitflags 2.6.0", -] - [[package]] name = "rawsocket" version = "0.1.0" @@ -5823,9 +5824,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.11" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe060fe50f524be480214aba758c71f99f90ee8c83c5a36b5e9e1d568eb4eb3" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64 0.22.1", "bytes", @@ -5919,7 +5920,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.93", + "syn 2.0.95", "unicode-ident", ] @@ -6086,7 +6087,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.1.0", + "security-framework 3.2.0", ] [[package]] @@ -6217,9 +6218,15 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.93", + "syn 2.0.95", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -6243,7 +6250,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -6294,9 +6301,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -6307,9 +6314,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -6367,7 +6374,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -6378,14 +6385,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.134" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -6411,7 +6418,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -6544,9 +6551,9 @@ checksum = "5dd19be0257552dd56d1bb6946f89f193c6e5b9f13cc9327c4bc84a357507c74" [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -6686,7 +6693,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -6729,9 +6736,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.93" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c786062daee0d6db1132800e623df74274a0a87322d8e183338e01b3d98d058" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -6773,7 +6780,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -6832,12 +6839,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -6943,7 +6951,7 @@ dependencies = [ "mirrord-operator", "rand", "regex", - "reqwest 0.12.11", + "reqwest 0.12.12", "rstest", "rustls 0.23.20", "serde", @@ -6990,7 +6998,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -7001,7 +7009,7 @@ checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -7105,7 +7113,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -7273,7 +7281,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -7363,7 +7371,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -7418,12 +7426,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "triomphe" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" - [[package]] name = "try-lock" version = "0.2.5" @@ -7783,7 +7785,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "wasm-bindgen-shared", ] @@ -7818,7 +7820,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7942,6 +7944,16 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -7951,6 +7963,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.95", +] + [[package]] name = "windows-registry" version = "0.2.0" @@ -8131,9 +8178,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" dependencies = [ "memchr", ] @@ -8263,9 +8310,9 @@ dependencies = [ [[package]] name = "xattr" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" dependencies = [ "libc", "linux-raw-sys", @@ -8337,7 +8384,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "synstructure 0.13.1", ] @@ -8359,7 +8406,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -8379,7 +8426,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", "synstructure 0.13.1", ] @@ -8400,7 +8447,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] @@ -8422,7 +8469,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.93", + "syn 2.0.95", ] [[package]] diff --git a/changelog.d/+104-policy-fs.added.md b/changelog.d/+104-policy-fs.added.md new file mode 100644 index 00000000000..1ad43a13736 --- /dev/null +++ b/changelog.d/+104-policy-fs.added.md @@ -0,0 +1 @@ +Add policy to control file ops. diff --git a/mirrord/layer/src/detour.rs b/mirrord/layer/src/detour.rs index a89e79ce0f6..92a014d20dd 100644 --- a/mirrord/layer/src/detour.rs +++ b/mirrord/layer/src/detour.rs @@ -215,6 +215,10 @@ pub(crate) enum Bypass { /// Useful for operations that are version gated, and we want to bypass when the protocol /// doesn't support them. NotImplemented, + + /// File `open` (any `open`-ish operation) was forced to be local, instead of remote, most + /// likely due to an operator fs policy. + OpenLocal, } impl Bypass { diff --git a/mirrord/layer/src/error.rs b/mirrord/layer/src/error.rs index ea0cbabe3c8..da4797a1916 100644 --- a/mirrord/layer/src/error.rs +++ b/mirrord/layer/src/error.rs @@ -248,6 +248,7 @@ impl From for i64 { HookError::BincodeEncode(_) => libc::EINVAL, HookError::ResponseError(response_fail) => match response_fail { ResponseError::IdsExhausted(_) => libc::ENOMEM, + ResponseError::OpenLocal => libc::ENOENT, ResponseError::NotFound(_) => libc::ENOENT, ResponseError::NotDirectory(_) => libc::ENOTDIR, ResponseError::NotFile(_) => libc::EISDIR, diff --git a/mirrord/layer/src/file/ops.rs b/mirrord/layer/src/file/ops.rs index 580f8eacc8a..ca9dd10f951 100644 --- a/mirrord/layer/src/file/ops.rs +++ b/mirrord/layer/src/file/ops.rs @@ -206,7 +206,12 @@ pub(crate) fn open(path: Detour, open_options: OpenOptionsInternal) -> ensure_not_ignored!(path, open_options.is_write()); - let OpenFileResponse { fd: remote_fd } = RemoteFile::remote_open(path.clone(), open_options)?; + let OpenFileResponse { fd: remote_fd } = RemoteFile::remote_open(path.clone(), open_options) + .or_else(|fail| match fail { + // The operator has a policy that matches this `path` as local-only. + HookError::ResponseError(ResponseError::OpenLocal) => Detour::Bypass(Bypass::OpenLocal), + other => Detour::Error(other), + })?; // TODO: Need a way to say "open a directory", right now `is_dir` always returns false. // This requires having a fake directory name (`/fake`, for example), instead of just converting diff --git a/mirrord/operator/src/crd/policy.rs b/mirrord/operator/src/crd/policy.rs index 1ad9447d1e8..e236164da98 100644 --- a/mirrord/operator/src/crd/policy.rs +++ b/mirrord/operator/src/crd/policy.rs @@ -58,6 +58,11 @@ pub struct MirrordPolicySpec { /// target. #[serde(default)] pub env: EnvPolicy, + + /// Overrides fs ops behaviour, granting control over them to the operator policy, instead of + /// the user config. + #[serde(default)] + pub fs: FsPolicy, } /// Custom cluster-wide resource for policies that limit what mirrord features users can use. @@ -90,6 +95,11 @@ pub struct MirrordClusterPolicySpec { /// target. #[serde(default)] pub env: EnvPolicy, + + /// Overrides fs ops behaviour, granting control over them to the operator policy, instead of + /// the user config. + #[serde(default)] + pub fs: FsPolicy, } /// Policy for controlling environment variables access from mirrord instances. @@ -104,9 +114,33 @@ pub struct EnvPolicy { /// Variable names can be matched using `*` and `?` where `?` matches exactly one occurrence of /// any character and `*` matches arbitrary many (including zero) occurrences of any character, /// e.g. `DATABASE_*` will match `DATABASE_URL` and `DATABASE_PORT`. + #[serde(default)] pub exclude: HashSet, } +/// File operations policy that mimics the mirrord fs config. +/// +/// Allows the operator control over remote file ops behaviour, overriding what the user has set in +/// their mirrord config file, if it matches something in one of the lists (regex sets) of this +/// struct. +#[derive(Clone, Default, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "kebab-case")] +pub struct FsPolicy { + /// The file can only be opened in read-only mode, otherwise the operator returns an IO error. + #[serde(default)] + pub read_only: HashSet, + + /// The file cannot be opened in the remote target. + /// + /// `open` calls that match this are forced to be opened in the local user's machine. + #[serde(default)] + pub local: HashSet, + + /// Any file that matches this returns a file not found error from the operator. + #[serde(default)] + pub not_found: HashSet, +} + #[test] fn check_one_api_group() { use kube::Resource; diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 34832bbe47f..1904cc97f1c 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.13.2" +version = "1.13.3" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/error.rs b/mirrord/protocol/src/error.rs index 9a8e451658c..efb7ff08198 100644 --- a/mirrord/protocol/src/error.rs +++ b/mirrord/protocol/src/error.rs @@ -67,6 +67,9 @@ pub enum ResponseError { #[error("Failed stripping path with `{0}`!")] StripPrefix(String), + + #[error("File has to be opened locally!")] + OpenLocal, } impl From for ResponseError { diff --git a/mirrord/protocol/src/file.rs b/mirrord/protocol/src/file.rs index b2e8e3773f8..c0b4cfe2f18 100644 --- a/mirrord/protocol/src/file.rs +++ b/mirrord/protocol/src/file.rs @@ -25,6 +25,9 @@ pub static READDIR_BATCH_VERSION: LazyLock = pub static MKDIR_VERSION: LazyLock = LazyLock::new(|| ">=1.13.0".parse().expect("Bad Identifier")); +pub static OPEN_LOCAL_VERSION: LazyLock = + LazyLock::new(|| ">=1.13.3".parse().expect("Bad Identifier")); + /// Internal version of Metadata across operating system (macOS, Linux) /// Only mutual attributes #[derive(Encode, Decode, Debug, PartialEq, Clone, Copy, Eq, Default)] diff --git a/tests/go-e2e-dir/main.go b/tests/go-e2e-dir/main.go index b608f01b53b..80006ce58c5 100644 --- a/tests/go-e2e-dir/main.go +++ b/tests/go-e2e-dir/main.go @@ -15,13 +15,20 @@ func main() { os.Exit(-1) } fmt.Printf("DirEntries: %s\n", dir) + // `os.ReadDir` does not include `.` and `..`. - if len(dir) != 2 { + if len(dir) < 2 { os.Exit(-1) } - // `os.ReadDir` sorts the result by file name. - if dir[0].Name() != "app.py" || dir[1].Name() != "test.txt" { - os.Exit(-1) + + // Iterate over the files in this dir, exiting if it's not an expected file name. + for i := 0; i < len(dir); i++ { + dirName := dir[i].Name() + + if dirName != "app.py" && dirName != "test.txt" && dirName != "file.local" && dirName != "file.not-found" && dirName != "file.read-only" && dirName != "file.read-write" { + os.Exit(-1) + } + } err = os.Mkdir("/app/test_mkdir", 0755) diff --git a/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs b/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs new file mode 100644 index 00000000000..8e58bf52cec --- /dev/null +++ b/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs @@ -0,0 +1,54 @@ +import fs from 'fs'; + +fs.open("/app/file.local", (fail, fd) => { + console.log(`open file.local ${fd}`); + if (fd) { + console.log(`SUCCESS /app/file.local ${fd}`); + } + + if (fail) { + console.error(`FAIL /app/file.local ${fail}`); + } +}); + +fs.open("/app/file.not-found", (fail, fd) => { + console.log(`open file.not-found ${fd}`); + if (fd) { + console.log(`SUCCESS /app/file.not-found ${fd}`); + } + + if (fail) { + console.error(`FAIL /app/file.not-found ${fail}`); + } +}); + +fs.open("/app/file.read-only", (fail, fd) => { + if (fd) { + console.log(`SUCCESS /app/file.read-only ${fd}`); + } + + if (fail) { + console.error(`FAIL /app/file.read-only ${fail}`); + } +}); + +fs.open("/app/file.read-only", "r+", (fail, fd) => { + if (fd) { + console.log(`SUCCESS r+ /app/file.read-only ${fd}`); + } + + if (fail) { + console.error(`FAIL r+ /app/file.read-only ${fail}`); + } +}); + +fs.open("/app/file.read-write", "r+", (fail, fd) => { + if (fd) { + console.log(`SUCCESS /app/file.read-write ${fd}`); + } + + if (fail) { + console.error(`FAIL /app/file.read-write ${fail}`); + } +}); + diff --git a/tests/python-e2e/files_ro.py b/tests/python-e2e/files_ro.py index ed99eab5d7f..c6e4e8d5631 100644 --- a/tests/python-e2e/files_ro.py +++ b/tests/python-e2e/files_ro.py @@ -3,7 +3,7 @@ import uuid import unittest -TEXT = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." +TEXT = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n" class FileOpsTest(unittest.TestCase): @@ -22,4 +22,4 @@ def test_read_only(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/python-e2e/ops.py b/tests/python-e2e/ops.py index 8e83271628f..9f94425921e 100644 --- a/tests/python-e2e/ops.py +++ b/tests/python-e2e/ops.py @@ -2,7 +2,7 @@ import uuid import unittest -TEXT = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." +TEXT = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.\n" class FileOpsTest(unittest.TestCase): diff --git a/tests/src/operator/policies.rs b/tests/src/operator/policies.rs index 61a17d90af5..114d42968e6 100644 --- a/tests/src/operator/policies.rs +++ b/tests/src/operator/policies.rs @@ -18,6 +18,8 @@ use crate::utils::{ config_dir, kube_client, service, Application, KubeService, ResourceGuard, TestProcess, }; +mod fs; + /// Guard that deletes a mirrord policy when dropped. struct PolicyGuard { _inner: ResourceGuard, @@ -128,6 +130,7 @@ fn block_steal_without_qualifiers() -> PolicyTestCase { selector: None, block: vec![BlockedFeature::Steal], env: Default::default(), + fs: Default::default(), }, ), service_b_can_steal: No, @@ -147,6 +150,7 @@ fn block_steal_with_path_pattern() -> PolicyTestCase { selector: None, block: vec![BlockedFeature::Steal], env: Default::default(), + fs: Default::default(), }, ), service_b_can_steal: EvenWithoutFilter, @@ -166,6 +170,7 @@ fn block_unfiltered_steal_with_path_pattern() -> PolicyTestCase { selector: None, block: vec![BlockedFeature::StealWithoutFilter], env: Default::default(), + fs: Default::default(), }, ), service_b_can_steal: EvenWithoutFilter, @@ -185,6 +190,7 @@ fn block_unfiltered_steal_with_deployment_path_pattern() -> PolicyTestCase { selector: None, block: vec![BlockedFeature::StealWithoutFilter], env: Default::default(), + fs: Default::default(), }, ), service_a_can_steal: OnlyWithFilter, @@ -210,6 +216,7 @@ fn block_steal_with_label_selector() -> PolicyTestCase { }), block: vec![BlockedFeature::Steal], env: Default::default(), + fs: Default::default(), }, ), service_b_can_steal: EvenWithoutFilter, @@ -236,6 +243,7 @@ fn block_steal_with_unmatching_policy() -> PolicyTestCase { }), block: vec![BlockedFeature::Steal], env: Default::default(), + fs: Default::default(), }, ), service_b_can_steal: EvenWithoutFilter, @@ -377,6 +385,7 @@ pub async fn create_cluster_policy_and_try_to_mirror( selector: None, block: vec![BlockedFeature::Mirror], env: Default::default(), + fs: Default::default(), }, ), ) diff --git a/tests/src/operator/policies/fs.rs b/tests/src/operator/policies/fs.rs new file mode 100644 index 00000000000..d54ed7bb98d --- /dev/null +++ b/tests/src/operator/policies/fs.rs @@ -0,0 +1,87 @@ +use std::{collections::HashSet, time::Duration}; + +use mirrord_operator::crd::policy::{FsPolicy, MirrordPolicy, MirrordPolicySpec}; +use rstest::{fixture, rstest}; + +use crate::{ + operator::policies::PolicyGuard, + utils::{kube_client, service, Application, KubeService}, +}; + +#[fixture] +async fn fs_service(#[future] kube_client: kube::Client) -> KubeService { + let namespace = format!("e2e-tests-fs-policies-{}", crate::utils::random_string()); + + service( + &namespace, + "NodePort", + "ghcr.io/metalbear-co/mirrord-pytest:latest", + "fs-policy-e2e-test-service", + false, + kube_client, + ) + .await +} + +#[rstest] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[timeout(Duration::from_secs(60))] +pub async fn create_cluster_fs_policy_and_try_file_operations( + #[future] service: KubeService, + #[future] kube_client: kube::Client, +) { + let kube_client = kube_client.await; + let service = service.await; + + // Create policy, delete it when test exits. + let _policy_guard = PolicyGuard::namespaced( + kube_client, + &MirrordPolicy::new( + "e2e-test-fs-policy-with-path-pattern", + MirrordPolicySpec { + target_path: Some("fs_policy_e2e-test-*".into()), + selector: None, + block: Default::default(), + env: Default::default(), + fs: FsPolicy { + read_only: HashSet::from_iter(vec!["file.read-only".to_string()]), + local: HashSet::from_iter(vec!["file.local".to_string()]), + not_found: HashSet::from_iter(vec!["file.not-found".to_string()]), + }, + }, + ), + &service.namespace, + ) + .await; + + let application = Application::NodeFsPolicy; + println!("Running mirrord {application:?} against {}", &service.name); + + let mut test_process = application + .run( + &service.target, + Some(&service.namespace), + Some(vec!["--fs-mode=write"]), + None, + ) + .await; + + test_process.wait_assert_success().await; + + test_process + .assert_stderr_contains("FAIL /app/file.local") + .await; + test_process + .assert_stderr_contains("FAIL /app/file.not-found") + .await; + test_process + .assert_stderr_contains("FAIL r+ /app/file.read-only") + .await; + + test_process + .assert_stdout_contains("SUCCESS /app/file.read-only") + .await; + test_process + .assert_stdout_contains("SUCCESS /app/file.read-write") + .await; +} diff --git a/tests/src/utils.rs b/tests/src/utils.rs index bf807337a09..538a4d753e6 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -100,6 +100,11 @@ pub enum Application { PythonCloseSocketKeepConnection, RustWebsockets, RustSqs, + /// Tries to open files in the remote target, but these operations should succeed or fail based + /// on mirrord `FsPolicy`. + /// + /// - `node-e2e/fspolicy/test_operator_fs_policy.mjs` + NodeFsPolicy, } #[derive(Debug)] @@ -408,6 +413,9 @@ impl Application { Application::NodeHTTP2 => { vec!["node", "node-e2e/http2/test_http2_traffic_steal.mjs"] } + Application::NodeFsPolicy => { + vec!["node", "node-e2e/fspolicy/test_operator_fs_policy.mjs"] + } Application::Go21HTTP => vec!["go-e2e/21.go_test_app"], Application::Go22HTTP => vec!["go-e2e/22.go_test_app"], Application::Go23HTTP => vec!["go-e2e/23.go_test_app"], From 81da4e4eb36889f04892419517c8a4b71c1856df Mon Sep 17 00:00:00 2001 From: t4lz Date: Thu, 9 Jan 2025 22:34:17 +0100 Subject: [PATCH 07/23] IPv6 support for traffic stealing (#2976) * E2E IPv6 steal test * e2e ipv6 service * Local E2E IPv6 testing * No ephemeral, need to delete or uncomment later * For local testing. DROP * add ipv6 flag * allow IPv6 in socket if enabled in config * enable ipv6 in test config * don't change CONTRIBUTING.md formatting * Use IpAddr instad of Ipv4Addr for pod IPs * E2E test with portforwarding * fix tests import * move ipv6 config up to network * Propagate ipv6 setting to an agent arg * fallback agent listener * stealer, iptables - start * add ipv6 listener and iptables, still need to adapt more places * iptable listeners * use filter table for ipv6 * oh no * Revert "oh no" This reverts commit 8fa0954f95c434497b2839db4483269fc2db5132. * try with flush connections * use input chain for IPv6 * fix dumb bug (ip6tables command switch) * add debug logs * add debug logs * revert some stuff * use nat table in ip6tables * ipv6 manual test app * fix test request * fix doc? * thanks clippy * ignore ipv6 test * fix config test * cfg test for ipv6 utils * easy way out * fix tests utils * ipv6 support default to false * fix iptables tests * remove unused methods * fix policies test * update schema * run medschool * fix kube UT * use test image agent * changelog * use published test image again * TODOs * add ipv6 test to CI * add kind cluster config for IPv6 * fix cluster config * CI IPv6 job name * patch kind config to fix fail * use kind bash script * fix cargo test command * agent logs? * maybe with a longer TTL I'll get some logs? * print intproxy logs on failure * show nodes on failure * modprobe? * exec modprobe as command * which modprobe * docker file install kmod * modprobe ip6_tables * load 3 modules * unused vars * undo modprobes * protocol cargo * don't test ipv6 on CI * delete kind cluster creation script, since not testing in CI * CR * apply change to new policy test --- CONTRIBUTING.md | 19 ++ changelog.d/2956.added.md | 1 + mirrord-schema.json | 10 +- mirrord/agent/src/cli.rs | 11 +- mirrord/agent/src/entrypoint.rs | 44 ++- mirrord/agent/src/error.rs | 4 + mirrord/agent/src/steal/connection.rs | 26 +- mirrord/agent/src/steal/ip_tables.rs | 22 +- mirrord/agent/src/steal/ip_tables/output.rs | 5 +- mirrord/agent/src/steal/subscriptions.rs | 253 +++++++++++++++--- mirrord/config/configuration.md | 4 + mirrord/config/src/config/from_env.rs | 4 + mirrord/config/src/feature/network.rs | 17 +- .../config/src/feature/network/incoming.rs | 12 +- mirrord/config/src/lib.rs | 1 + mirrord/kube/src/api/container.rs | 9 +- mirrord/kube/src/api/container/job.rs | 10 +- mirrord/kube/src/api/container/util.rs | 3 +- mirrord/kube/src/api/kubernetes.rs | 10 +- mirrord/kube/src/api/runtime.rs | 8 +- mirrord/layer/src/socket/ops.rs | 3 +- mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/lib.rs | 2 + tests/ipv6-app.yaml | 49 ++++ tests/kind-cluster-ipv6-config.yaml | 9 + tests/src/env.rs | 2 +- tests/src/file_ops.rs | 38 ++- tests/src/http.rs | 14 +- tests/src/issue1317.rs | 9 +- tests/src/lib.rs | 1 + tests/src/operator/concurrent_steal.rs | 15 +- tests/src/operator/policies.rs | 6 +- tests/src/operator/policies/fs.rs | 2 +- tests/src/traffic.rs | 138 +++++++--- tests/src/traffic/steal.rs | 88 ++++-- tests/src/utils.rs | 74 +++-- tests/src/utils/ipv6.rs | 100 +++++++ 37 files changed, 838 insertions(+), 187 deletions(-) create mode 100644 changelog.d/2956.added.md create mode 100644 tests/ipv6-app.yaml create mode 100644 tests/kind-cluster-ipv6-config.yaml create mode 100644 tests/src/utils/ipv6.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 818a39e3a89..1b99ddc4c65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,6 +113,25 @@ For example, a test which only tests sanity of the ephemeral container feature s On Linux, running tests may exhaust a large amount of RAM and crash the machine. To prevent this, limit the number of concurrent jobs by running the command with e.g. `-j 4` +### IPv6 + +Some tests create a single-stack IPv6 service. They can only be run on clusters with IPv6 enabled. +In order to test IPv6 on a local cluster on macOS, you can use Kind: + +1. `brew install kind` +2. ```shell + cat >kind-config.yaml < Result<()> { trace!("start_agent -> Starting agent with args: {args:?}"); - let listener = TcpListener::bind(SocketAddrV4::new( + // listen for client connections + let ipv4_listener_result = TcpListener::bind(SocketAddrV4::new( Ipv4Addr::UNSPECIFIED, args.communicate_port, )) - .await?; + .await; + + let listener = if args.ipv6 && ipv4_listener_result.is_err() { + debug!("IPv6 Support enabled, and IPv4 bind failed, binding IPv6 listener"); + TcpListener::bind(SocketAddrV6::new( + Ipv6Addr::UNSPECIFIED, + args.communicate_port, + 0, + 0, + )) + .await + } else { + ipv4_listener_result + }?; + + match listener.local_addr() { + Ok(addr) => debug!( + client_listener_address = addr.to_string(), + "Created listener." + ), + Err(err) => error!(%err, "listener local address error"), + } let state = State::new(&args).await?; @@ -566,13 +588,15 @@ async fn start_agent(args: Args) -> Result<()> { let cancellation_token = cancellation_token.clone(); let watched_task = WatchedTask::new( TcpConnectionStealer::TASK_NAME, - TcpConnectionStealer::new(stealer_command_rx).and_then(|stealer| async move { - let res = stealer.start(cancellation_token).await; - if let Err(err) = res.as_ref() { - error!("Stealer failed: {err}"); - } - res - }), + TcpConnectionStealer::new(stealer_command_rx, args.ipv6).and_then( + |stealer| async move { + let res = stealer.start(cancellation_token).await; + if let Err(err) = res.as_ref() { + error!("Stealer failed: {err}"); + } + res + }, + ), ); let status = watched_task.status(); let task = run_thread_in_namespace( diff --git a/mirrord/agent/src/error.rs b/mirrord/agent/src/error.rs index ad04e49c8c5..d9ae7cb8b9d 100644 --- a/mirrord/agent/src/error.rs +++ b/mirrord/agent/src/error.rs @@ -84,6 +84,10 @@ pub(crate) enum AgentError { /// Temporary error for vpn feature #[error("Generic error in vpn: {0}")] VpnError(String), + + /// When we neither create a redirector for IPv4, nor for IPv6 + #[error("Could not create a listener for stolen connections")] + CannotListenForStolenConnections, } impl From> for AgentError { diff --git a/mirrord/agent/src/steal/connection.rs b/mirrord/agent/src/steal/connection.rs index 463c61f88d0..37435176b8b 100644 --- a/mirrord/agent/src/steal/connection.rs +++ b/mirrord/agent/src/steal/connection.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - net::{IpAddr, Ipv4Addr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, }; use fancy_regex::Regex; @@ -289,6 +289,9 @@ pub(crate) struct TcpConnectionStealer { /// Set of active connections stolen by [`Self::port_subscriptions`]. connections: StolenConnections, + + /// Shen set, the stealer will use IPv6 if needed. + support_ipv6: bool, } impl TcpConnectionStealer { @@ -297,14 +300,21 @@ impl TcpConnectionStealer { /// Initializes a new [`TcpConnectionStealer`], but doesn't start the actual work. /// You need to call [`TcpConnectionStealer::start`] to do so. #[tracing::instrument(level = "trace")] - pub(crate) async fn new(command_rx: Receiver) -> Result { + pub(crate) async fn new( + command_rx: Receiver, + support_ipv6: bool, + ) -> Result { let config = envy::prefixed("MIRRORD_AGENT_") .from_env::() .unwrap_or_default(); let port_subscriptions = { - let redirector = - IpTablesRedirector::new(config.stealer_flush_connections, config.pod_ips).await?; + let redirector = IpTablesRedirector::new( + config.stealer_flush_connections, + config.pod_ips, + support_ipv6, + ) + .await?; PortSubscriptions::new(redirector, 4) }; @@ -315,6 +325,7 @@ impl TcpConnectionStealer { clients: HashMap::with_capacity(8), clients_closed: Default::default(), connections: StolenConnections::with_capacity(8), + support_ipv6, }) } @@ -371,9 +382,14 @@ impl TcpConnectionStealer { #[tracing::instrument(level = "trace", skip(self))] async fn incoming_connection(&mut self, stream: TcpStream, peer: SocketAddr) -> Result<()> { let mut real_address = orig_dst::orig_dst_addr(&stream)?; + let localhost = if self.support_ipv6 && real_address.is_ipv6() { + IpAddr::V6(Ipv6Addr::LOCALHOST) + } else { + IpAddr::V4(Ipv4Addr::LOCALHOST) + }; // If we use the original IP we would go through prerouting and hit a loop. // localhost should always work. - real_address.set_ip(IpAddr::V4(Ipv4Addr::LOCALHOST)); + real_address.set_ip(localhost); let Some(port_subscription) = self.port_subscriptions.get(real_address.port()).cloned() else { diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 5583b485000..68bddb6a406 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -111,6 +111,18 @@ pub fn new_iptables() -> iptables::IPTables { .expect("IPTables initialization may not fail!") } +/// wrapper around iptables::new that uses nft or legacy based on env +pub fn new_ip6tables() -> iptables::IPTables { + if let Ok(val) = std::env::var("MIRRORD_AGENT_NFTABLES") + && val.to_lowercase() == "true" + { + iptables::new_with_cmd("/usr/sbin/ip6tables-nft") + } else { + iptables::new_with_cmd("/usr/sbin/ip6tables-legacy") + } + .expect("IPTables initialization may not fail!") +} + impl Debug for IPTablesWrapper { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("IPTablesWrapper") @@ -140,7 +152,7 @@ impl IPTables for IPTablesWrapper { } } - #[tracing::instrument(level = "trace")] + #[tracing::instrument(level = tracing::Level::TRACE, skip(self), ret, fields(table_name=%self.table_name))] fn create_chain(&self, name: &str) -> Result<()> { self.tables .new_chain(self.table_name, name) @@ -220,6 +232,7 @@ where ipt: IPT, flush_connections: bool, pod_ips: Option<&str>, + ipv6: bool, ) -> Result { let ipt = Arc::new(ipt); @@ -231,6 +244,7 @@ where _ => Redirects::Mesh(MeshRedirect::create(ipt.clone(), vendor, pod_ips)?), } } else { + tracing::trace!(ipv6 = ipv6, "creating standard redirect"); match StandardRedirect::create(ipt.clone(), pod_ips) { Err(err) => { warn!("Unable to create StandardRedirect chain: {err}"); @@ -280,7 +294,7 @@ where /// Adds the redirect rule to iptables. /// /// Used to redirect packets when mirrord incoming feature is set to `steal`. - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = tracing::Level::DEBUG, skip(self))] pub(super) async fn add_redirect( &self, redirected_port: Port, @@ -408,7 +422,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let ipt = SafeIpTables::create(mock, false, None) + let ipt = SafeIpTables::create(mock, false, None, false) .await .expect("Create Failed"); @@ -541,7 +555,7 @@ mod tests { .times(1) .returning(|_| Ok(())); - let ipt = SafeIpTables::create(mock, false, None) + let ipt = SafeIpTables::create(mock, false, None, false) .await .expect("Create Failed"); diff --git a/mirrord/agent/src/steal/ip_tables/output.rs b/mirrord/agent/src/steal/ip_tables/output.rs index 944bc26f95b..2286469c00c 100644 --- a/mirrord/agent/src/steal/ip_tables/output.rs +++ b/mirrord/agent/src/steal/ip_tables/output.rs @@ -20,8 +20,11 @@ where { const ENTRYPOINT: &'static str = "OUTPUT"; + #[tracing::instrument(skip(ipt), level = tracing::Level::TRACE)] pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> Result { - let managed = IPTableChain::create(ipt, chain_name)?; + let managed = IPTableChain::create(ipt, chain_name.clone()).inspect_err( + |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), + )?; let exclude_source_ips = pod_ips .map(|pod_ips| format!("! -s {pod_ips}")) diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 0ff0e1fa8ea..0468719bc9c 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -1,16 +1,20 @@ use std::{ collections::{hash_map::Entry, HashMap}, - net::{Ipv4Addr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, + ops::Not, sync::Arc, }; use dashmap::{mapref::entry::Entry as DashMapEntry, DashMap}; use mirrord_protocol::{Port, RemoteResult, ResponseError}; -use tokio::net::{TcpListener, TcpStream}; +use tokio::{ + net::{TcpListener, TcpStream}, + select, +}; use super::{ http::HttpFilter, - ip_tables::{new_iptables, IPTablesWrapper, SafeIpTables}, + ip_tables::{new_ip6tables, new_iptables, IPTablesWrapper, SafeIpTables}, }; use crate::{error::AgentError, util::ClientId}; @@ -47,19 +51,82 @@ pub trait PortRedirector { async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error>; } -/// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by -/// redirecting TCP packets to inner [`TcpListener`]. -pub(crate) struct IpTablesRedirector { +/// A TCP listener, together with an iptables wrapper to set rules that send traffic to the +/// listener. +pub(crate) struct IptablesListener { /// For altering iptables rules. iptables: Option>, - /// Whether exisiting connections should be flushed when adding new redirects. - flush_connections: bool, - /// Port of [`IpTablesRedirector::listener`]. + /// Port of [`listener`](Self::listener). redirect_to: Port, /// Listener to which redirect all connections. listener: TcpListener, - + /// Optional comma-seperated list of IPs of the pod, originating in the pod's `Status.PodIps` pod_ips: Option, + /// Whether existing connections should be flushed when adding new redirects. + flush_connections: bool, + /// Is this for connections incoming over IPv6 + ipv6: bool, +} + +#[async_trait::async_trait] +impl PortRedirector for IptablesListener { + type Error = AgentError; + + #[tracing::instrument(skip(self), err, ret, level=tracing::Level::DEBUG, fields(self.ipv6 = %self.ipv6))] + async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { + let iptables = if let Some(iptables) = self.iptables.as_ref() { + iptables + } else { + let safe = crate::steal::ip_tables::SafeIpTables::create( + if self.ipv6 { + new_ip6tables() + } else { + new_iptables() + } + .into(), + self.flush_connections, + self.pod_ips.as_deref(), + self.ipv6, + ) + .await?; + self.iptables.insert(safe) + }; + iptables.add_redirect(from, self.redirect_to).await + } + + async fn remove_redirection(&mut self, from: Port) -> Result<(), Self::Error> { + if let Some(iptables) = self.iptables.as_ref() { + iptables.remove_redirect(from, self.redirect_to).await?; + } + + Ok(()) + } + + async fn cleanup(&mut self) -> Result<(), Self::Error> { + if let Some(iptables) = self.iptables.take() { + iptables.cleanup().await?; + } + + Ok(()) + } + + async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error> { + self.listener.accept().await.map_err(Into::into) + } +} + +/// Implementation of [`PortRedirector`] that manipulates iptables to steal connections by +/// redirecting TCP packets to inner [`TcpListener`]. +/// +/// Holds TCP listeners + iptables, for redirecting IPv4 and/or IPv6 connections. +pub(crate) enum IpTablesRedirector { + Ipv4Only(IptablesListener), + /// Could be used if IPv6 support is enabled, and we cannot bind an IPv4 address. + Ipv6Only(IptablesListener), + Dual { + ipv4_listener: IptablesListener, + ipv6_listener: IptablesListener, + }, } impl IpTablesRedirector { @@ -67,28 +134,116 @@ impl IpTablesRedirector { /// [`Ipv4Addr::UNSPECIFIED`] address and a random port. This listener will be used to accept /// redirected connections. /// + /// If `support_ipv6` is set, will also listen on IPv6, and a fail to listen over IPv4 will be + /// accepted. + /// /// # Note /// /// Does not yet alter iptables. /// /// # Params /// - /// * `flush_connections` - whether exisitng connections should be flushed when adding new + /// * `flush_connections` - whether existing connections should be flushed when adding new /// redirects pub(crate) async fn new( flush_connections: bool, pod_ips: Option, + support_ipv6: bool, ) -> Result { - let listener = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await?; - let redirect_to = listener.local_addr()?.port(); - - Ok(Self { - iptables: None, - flush_connections, - redirect_to, - listener, - pod_ips, - }) + let (pod_ips4, pod_ips6) = pod_ips.map_or_else( + || (None, None), + |ips| { + // TODO: probably nicer to split at the client and avoid the conversion to and back + // from a string. + let (ip4s, ip6s): (Vec<_>, Vec<_>) = ips.split(',').partition(|ip_str| { + ip_str + .parse::() + .inspect_err(|e| tracing::warn!(%e, "failed to parse pod IP {ip_str}")) + .as_ref() + .map(IpAddr::is_ipv4) + .unwrap_or_default() + }); + // Convert to options, `None` if vector is empty. + ( + ip4s.is_empty().not().then(|| ip4s.join(",")), + ip6s.is_empty().not().then(|| ip6s.join(",")), + ) + }, + ); + tracing::debug!("pod IPv4 addresses: {pod_ips4:?}, pod IPv6 addresses: {pod_ips6:?}"); + + tracing::debug!("Creating IPv4 iptables redirection listener"); + let listener4 = TcpListener::bind((Ipv4Addr::UNSPECIFIED, 0)).await + .inspect_err( + |err| tracing::debug!(%err, "Could not bind IPv4, continuing with IPv6 only."), + ) + .ok() + .and_then(|listener| { + let redirect_to = listener + .local_addr() + .inspect_err( + |err| tracing::debug!(%err, "Get IPv4 listener address, continuing with IPv6 only."), + ) + .ok()? + .port(); + Some(IptablesListener { + iptables: None, + redirect_to, + listener, + pod_ips: pod_ips4, + flush_connections, + ipv6: false, + }) + }); + tracing::debug!("Creating IPv6 iptables redirection listener"); + let listener6 = if support_ipv6 { + TcpListener::bind((Ipv6Addr::UNSPECIFIED, 0)).await + .inspect_err( + |err| tracing::debug!(%err, "Could not bind IPv6, continuing with IPv4 only."), + ) + .ok() + .and_then(|listener| { + let redirect_to = listener + .local_addr() + .inspect_err( + |err| tracing::debug!(%err, "Get IPv6 listener address, continuing with IPv4 only."), + ) + .ok()? + .port(); + Some(IptablesListener { + iptables: None, + redirect_to, + listener, + pod_ips: pod_ips6, + flush_connections, + ipv6: true, + }) + }) + } else { + None + }; + match (listener4, listener6) { + (None, None) => Err(AgentError::CannotListenForStolenConnections), + (Some(ipv4_listener), None) => Ok(Self::Ipv4Only(ipv4_listener)), + (None, Some(ipv6_listener)) => Ok(Self::Ipv6Only(ipv6_listener)), + (Some(ipv4_listener), Some(ipv6_listener)) => Ok(Self::Dual { + ipv4_listener, + ipv6_listener, + }), + } + } + + pub(crate) fn get_listeners_mut( + &mut self, + ) -> (Option<&mut IptablesListener>, Option<&mut IptablesListener>) { + match self { + IpTablesRedirector::Ipv4Only(ipv4_listener) => (Some(ipv4_listener), None), + IpTablesRedirector::Ipv6Only(ipv6_listener) => (None, Some(ipv6_listener)), + IpTablesRedirector::Dual { + ipv4_listener, + ipv6_listener, + } => (Some(ipv4_listener), Some(ipv6_listener)), + } } } @@ -97,41 +252,53 @@ impl PortRedirector for IpTablesRedirector { type Error = AgentError; async fn add_redirection(&mut self, from: Port) -> Result<(), Self::Error> { - let iptables = match self.iptables.as_ref() { - Some(iptables) => iptables, - None => { - let iptables = new_iptables(); - let safe = SafeIpTables::create( - iptables.into(), - self.flush_connections, - self.pod_ips.as_deref(), - ) - .await?; - self.iptables.insert(safe) - } - }; - - iptables.add_redirect(from, self.redirect_to).await + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + tracing::debug!("Adding IPv4 redirection from port {from}"); + ip4_listener.add_redirection(from).await?; + } + if let Some(ip6_listener) = ipv6_listener { + tracing::debug!("Adding IPv6 redirection from port {from}"); + ip6_listener.add_redirection(from).await?; + } + Ok(()) } async fn remove_redirection(&mut self, from: Port) -> Result<(), Self::Error> { - if let Some(iptables) = self.iptables.as_ref() { - iptables.remove_redirect(from, self.redirect_to).await?; + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + ip4_listener.remove_redirection(from).await?; + } + if let Some(ip6_listener) = ipv6_listener { + ip6_listener.remove_redirection(from).await?; } - Ok(()) } async fn cleanup(&mut self) -> Result<(), Self::Error> { - if let Some(iptables) = self.iptables.take() { - iptables.cleanup().await?; + let (ipv4_listener, ipv6_listener) = self.get_listeners_mut(); + if let Some(ip4_listener) = ipv4_listener { + ip4_listener.cleanup().await?; + } + if let Some(ip6_listener) = ipv6_listener { + ip6_listener.cleanup().await?; } - Ok(()) } async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error> { - self.listener.accept().await.map_err(Into::into) + match self { + Self::Dual { + ipv4_listener, + ipv6_listener, + } => { + select! { + con = ipv4_listener.next_connection() => con, + con = ipv6_listener.next_connection() => con, + } + } + Self::Ipv4Only(listener) | Self::Ipv6Only(listener) => listener.next_connection().await, + } } } diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md index 93dab79c3c0..3a3f8b4fa57 100644 --- a/mirrord/config/configuration.md +++ b/mirrord/config/configuration.md @@ -1266,6 +1266,10 @@ List of ports to mirror/steal traffic from. Other ports will remain local. Mutually exclusive with [`feature.network.incoming.ignore_ports`](#feature-network-ignore_ports). +### feature.network.ipv6 {#feature-network-dns} + +Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. + ### feature.network.outgoing {#feature-network-outgoing} Tunnel outgoing network operations through mirrord. diff --git a/mirrord/config/src/config/from_env.rs b/mirrord/config/src/config/from_env.rs index 9770456721a..0f87ef59034 100644 --- a/mirrord/config/src/config/from_env.rs +++ b/mirrord/config/src/config/from_env.rs @@ -20,6 +20,10 @@ where { type Value = T; + /// Returns: + /// - `None` if there is no env var with that name. + /// - `Some(Err(ConfigError::InvalidValue{...}))` if the value of the env var cannot be parsed. + /// - `Some(Ok(...))` if the env var exists and was parsed successfully. fn source_value(self, _context: &mut ConfigContext) -> Option> { std::env::var(self.0).ok().map(|var| { var.parse::() diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index 2f2b7901aee..227bd82a915 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -6,10 +6,12 @@ use serde::Serialize; use self::{incoming::*, outgoing::*}; use crate::{ - config::{ConfigContext, ConfigError}, + config::{from_env::FromEnv, source::MirrordConfigSource, ConfigContext, ConfigError}, util::MirrordToggleableConfig, }; +const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; + pub mod dns; pub mod filter; pub mod incoming; @@ -67,14 +69,26 @@ pub struct NetworkConfig { /// ### feature.network.dns {#feature-network-dns} #[config(toggleable, nested)] pub dns: DnsConfig, + + /// ### feature.network.ipv6 {#feature-network-dns} + /// + /// Enable ipv6 support. Turn on if your application listens to incoming traffic over IPv6. + #[config(env = IPV6_ENV_VAR, default = false)] + pub ipv6: bool, } impl MirrordToggleableConfig for NetworkFileConfig { fn disabled_config(context: &mut ConfigContext) -> Result { + let ipv6 = FromEnv::new(IPV6_ENV_VAR) + .source_value(context) + .transpose()? + .unwrap_or_default(); + Ok(NetworkConfig { incoming: IncomingFileConfig::disabled_config(context)?, dns: DnsFileConfig::disabled_config(context)?, outgoing: OutgoingFileConfig::disabled_config(context)?, + ipv6, }) } } @@ -84,6 +98,7 @@ impl CollectAnalytics for &NetworkConfig { analytics.add("incoming", &self.incoming); analytics.add("outgoing", &self.outgoing); analytics.add("dns", &self.dns); + analytics.add("ipv6", self.ipv6); } } diff --git a/mirrord/config/src/feature/network/incoming.rs b/mirrord/config/src/feature/network/incoming.rs index d56199e003e..fddd46260d5 100644 --- a/mirrord/config/src/feature/network/incoming.rs +++ b/mirrord/config/src/feature/network/incoming.rs @@ -58,7 +58,7 @@ use http_filter::*; /// }, /// "port_mapping": [[ 7777, 8888 ]], /// "ignore_localhost": false, -/// "ignore_ports": [9999, 10000] +/// "ignore_ports": [9999, 10000], /// "listen_ports": [[80, 8111]] /// } /// } @@ -96,9 +96,7 @@ impl MirrordConfig for IncomingFileConfig { .unwrap_or_default(), http_filter: HttpFilterFileConfig::default().generate_config(context)?, on_concurrent_steal: FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") - .layer(|layer| { - Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer) - }) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(), @@ -129,9 +127,7 @@ impl MirrordConfig for IncomingFileConfig { .unwrap_or_default(), on_concurrent_steal: FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") .or(advanced.on_concurrent_steal) - .layer(|layer| { - Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer) - }) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(), @@ -149,7 +145,7 @@ impl MirrordToggleableConfig for IncomingFileConfig { .unwrap_or_else(|| Ok(IncomingMode::Off))?; let on_concurrent_steal = FromEnv::new("MIRRORD_OPERATOR_ON_CONCURRENT_STEAL") - .layer(|layer| Unstable::new("IncomingFileConfig", "on_concurrent_steal", layer)) + .layer(|layer| Unstable::new("incoming", "on_concurrent_steal", layer)) .source_value(context) .transpose()? .unwrap_or_default(); diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index 8ffd1d48c29..055adac98d2 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -878,6 +878,7 @@ mod tests { udp: Some(false), ..Default::default() })), + ipv6: None, })), copy_target: None, hostname: None, diff --git a/mirrord/kube/src/api/container.rs b/mirrord/kube/src/api/container.rs index b87a088a412..a651dc13458 100644 --- a/mirrord/kube/src/api/container.rs +++ b/mirrord/kube/src/api/container.rs @@ -44,10 +44,16 @@ pub struct ContainerParams { /// the agent container. pub tls_cert: Option, pub pod_ips: Option, + /// Support IPv6-only clusters + pub support_ipv6: bool, } impl ContainerParams { - pub fn new(tls_cert: Option, pod_ips: Option) -> ContainerParams { + pub fn new( + tls_cert: Option, + pod_ips: Option, + support_ipv6: bool, + ) -> ContainerParams { let port: u16 = rand::thread_rng().gen_range(30000..=65535); let gid: u16 = rand::thread_rng().gen_range(3000..u16::MAX); @@ -64,6 +70,7 @@ impl ContainerParams { port, tls_cert, pod_ips, + support_ipv6, } } } diff --git a/mirrord/kube/src/api/container/job.rs b/mirrord/kube/src/api/container/job.rs index 907aefffeeb..d9958e6620b 100644 --- a/mirrord/kube/src/api/container/job.rs +++ b/mirrord/kube/src/api/container/job.rs @@ -241,12 +241,14 @@ mod test { fn targetless() -> Result<(), Box> { let mut config_context = ConfigContext::default(); let agent = AgentFileConfig::default().generate_config(&mut config_context)?; + let support_ipv6 = false; let params = ContainerParams { name: "foobar".to_string(), port: 3000, gid: 13, tls_cert: None, pod_ips: None, + support_ipv6, }; let update = JobVariant::new(&agent, ¶ms).as_update(); @@ -298,7 +300,8 @@ mod test { { "name": "RUST_LOG", "value": agent.log_level }, { "name": "MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "value": agent.flush_connections.to_string() }, { "name": "MIRRORD_AGENT_NFTABLES", "value": agent.nftables.to_string() }, - { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) } + { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) }, + { "name": "MIRRORD_AGENT_SUPPORT_IPV6", "value": Some(support_ipv6.to_string()) } ], "resources": // Add requests to avoid getting defaulted https://github.com/metalbear-co/mirrord/issues/579 @@ -330,12 +333,14 @@ mod test { fn targeted() -> Result<(), Box> { let mut config_context = ConfigContext::default(); let agent = AgentFileConfig::default().generate_config(&mut config_context)?; + let support_ipv6 = false; let params = ContainerParams { name: "foobar".to_string(), port: 3000, gid: 13, tls_cert: None, pod_ips: None, + support_ipv6, }; let update = JobTargetedVariant::new( @@ -432,7 +437,8 @@ mod test { { "name": "RUST_LOG", "value": agent.log_level }, { "name": "MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "value": agent.flush_connections.to_string() }, { "name": "MIRRORD_AGENT_NFTABLES", "value": agent.nftables.to_string() }, - { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) } + { "name": "MIRRORD_AGENT_JSON_LOG", "value": Some(agent.json_log.to_string()) }, + { "name": "MIRRORD_AGENT_SUPPORT_IPV6", "value": Some(support_ipv6.to_string()) } ], "resources": // Add requests to avoid getting defaulted https://github.com/metalbear-co/mirrord/issues/579 { diff --git a/mirrord/kube/src/api/container/util.rs b/mirrord/kube/src/api/container/util.rs index 77f917378ce..23fd752181b 100644 --- a/mirrord/kube/src/api/container/util.rs +++ b/mirrord/kube/src/api/container/util.rs @@ -4,7 +4,7 @@ use futures::{AsyncBufReadExt, TryStreamExt}; use k8s_openapi::api::core::v1::{EnvVar, Pod, Toleration}; use kube::{api::LogParams, Api}; use mirrord_config::agent::{AgentConfig, LinuxCapability}; -use mirrord_protocol::{AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; +use mirrord_protocol::{AGENT_IPV6_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; use regex::Regex; use tracing::warn; @@ -59,6 +59,7 @@ pub(super) fn agent_env(agent: &AgentConfig, params: &&ContainerParams) -> Vec, + support_ipv6: bool, ) -> Result<(ContainerParams, Option), KubeApiError> { let runtime_data = match target.path.as_ref().unwrap_or(&Target::Targetless) { Target::Targetless => None, @@ -187,7 +188,7 @@ impl KubernetesAPI { .join(",") }); - let params = ContainerParams::new(tls_cert, pod_ips); + let params = ContainerParams::new(tls_cert, pod_ips, support_ipv6); Ok((params, runtime_data)) } @@ -209,7 +210,12 @@ impl KubernetesAPI { where P: Progress + Send + Sync, { - let (params, runtime_data) = self.create_agent_params(target, tls_cert).await?; + let support_ipv6 = config + .map(|layer_conf| layer_conf.feature.network.ipv6) + .unwrap_or_default(); + let (params, runtime_data) = self + .create_agent_params(target, tls_cert, support_ipv6) + .await?; if let Some(RuntimeData { guessed_container: true, container_name, diff --git a/mirrord/kube/src/api/runtime.rs b/mirrord/kube/src/api/runtime.rs index a77afd1b5b5..a431a3b4c03 100644 --- a/mirrord/kube/src/api/runtime.rs +++ b/mirrord/kube/src/api/runtime.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeMap, convert::Infallible, fmt::{self, Display, Formatter}, - net::Ipv4Addr, + net::IpAddr, ops::FromResidual, str::FromStr, }; @@ -71,7 +71,7 @@ impl Display for ContainerRuntime { #[derive(Debug)] pub struct RuntimeData { pub pod_name: String, - pub pod_ips: Vec, + pub pod_ips: Vec, pub pod_namespace: Option, pub node_name: String, pub container_id: String, @@ -128,9 +128,9 @@ impl RuntimeData { .filter_map(|pod_ip| { pod_ip .ip - .parse::() + .parse::() .inspect_err(|e| { - tracing::warn!("failed to parse pod IP {ip}: {e:?}", ip = pod_ip.ip); + tracing::warn!("failed to parse pod IP {ip}: {e:#?}", ip = pod_ip.ip); }) .ok() }) diff --git a/mirrord/layer/src/socket/ops.rs b/mirrord/layer/src/socket/ops.rs index 543a512629c..efc0095c8a4 100644 --- a/mirrord/layer/src/socket/ops.rs +++ b/mirrord/layer/src/socket/ops.rs @@ -4,6 +4,7 @@ use std::{ collections::HashMap, io, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, TcpStream}, + ops::Not, os::{ fd::{BorrowedFd, FromRawFd, IntoRawFd}, unix::io::RawFd, @@ -129,7 +130,7 @@ pub(super) fn socket(domain: c_int, type_: c_int, protocol: c_int) -> Detour, command: Vec<&str>) { let service = service.await; - let mut process = run_exec_with_target(command, &service.target, None, None, None).await; + let mut process = + run_exec_with_target(command, &service.pod_container_target(), None, None, None).await; let res = process.wait().await; assert!(res.success()); } @@ -458,8 +485,14 @@ mod traffic_tests { pub async fn listen_localhost(#[future] service: KubeService) { let service = service.await; let node_command = vec!["node", "node-e2e/listen/test_listen_localhost.mjs"]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); } @@ -471,8 +504,14 @@ mod traffic_tests { pub async fn gethostname_remote_result(#[future] hostname_service: KubeService) { let service = hostname_service.await; let node_command = vec!["python3", "-u", "python-e2e/hostname.py"]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; assert!(res.success()); @@ -511,7 +550,9 @@ mod traffic_tests { "MIRRORD_OUTGOING_REMOTE_UNIX_STREAMS", "/app/unix-socket-server.sock", )]); - let mut process = run_exec_with_target(executable, &service.target, None, None, env).await; + let mut process = + run_exec_with_target(executable, &service.pod_container_target(), None, None, env) + .await; let res = process.wait().await; // The test application panics if it does not successfully connect to the socket, send data, @@ -534,7 +575,14 @@ mod traffic_tests { .to_string(); let executable = vec![app_path.as_str()]; - let mut process = run_exec_with_target(executable, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + executable, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.wait().await; // The test application panics if it does not successfully connect to the socket, send data, @@ -551,8 +599,14 @@ mod traffic_tests { "node", "node-e2e/outgoing/test_outgoing_traffic_many_requests.mjs", ]; - let mut process = - run_exec_with_target(node_command, &service.target, None, None, None).await; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + None, + ) + .await; let res = process.child.wait().await.unwrap(); assert!(res.success()); @@ -571,7 +625,7 @@ mod traffic_tests { let mirrord_args = vec!["--no-outgoing"]; let mut process = run_exec_with_target( node_command, - &service.target, + &service.pod_container_target(), None, Some(mirrord_args), None, diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 518aa0bc13e..31b5f668a2b 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -9,7 +9,10 @@ mod steal_tests { }; use futures_util::{SinkExt, StreamExt}; - use kube::Client; + use hyper::{body, client::conn, Request, StatusCode}; + use hyper_util::rt::TokioIo; + use k8s_openapi::api::core::v1::Pod; + use kube::{Api, Client}; use reqwest::{header::HeaderMap, Url}; use rstest::*; use tokio::time::sleep; @@ -19,13 +22,13 @@ mod steal_tests { }; use crate::utils::{ - config_dir, get_service_host_and_port, get_service_url, http2_service, kube_client, - send_request, send_requests, service, tcp_echo_service, websocket_service, Application, - KubeService, + config_dir, get_service_host_and_port, get_service_url, http2_service, + ipv6::{ipv6_service, portforward_http_requests}, + kube_client, send_request, send_requests, service, tcp_echo_service, websocket_service, + Application, KubeService, }; #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] - #[cfg(target_os = "linux")] #[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[timeout(Duration::from_secs(240))] @@ -49,7 +52,12 @@ mod steal_tests { } let mut process = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + None, + ) .await; process @@ -63,6 +71,47 @@ mod steal_tests { application.assert(&process).await; } + #[ignore] // Needs special cluster setup, so ignore by default. + #[rstest] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[timeout(Duration::from_secs(240))] + async fn steal_http_ipv6_traffic( + #[future] ipv6_service: KubeService, + #[future] kube_client: Client, + ) { + let application = Application::PythonFastApiHTTPIPv6; + let service = ipv6_service.await; + let kube_client = kube_client.await; + + let mut flags = vec!["--steal"]; + + if cfg!(feature = "ephemeral") { + flags.extend(["-e"].into_iter()); + } + + let mut process = application + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + Some(vec![("MIRRORD_INCOMING_ENABLE_IPV6", "true")]), + ) + .await; + + process + .wait_for_line(Duration::from_secs(40), "daemon subscribed") + .await; + + let api = Api::::namespaced(kube_client.clone(), &service.namespace); + portforward_http_requests(&api, service).await; + + tokio::time::timeout(Duration::from_secs(40), process.wait()) + .await + .unwrap(); + + application.assert(&process).await; + } + #[cfg_attr(not(any(feature = "ephemeral", feature = "job")), ignore)] #[cfg(target_os = "linux")] #[rstest] @@ -89,7 +138,7 @@ mod steal_tests { let mut process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_AGENT_STEALER_FLUSH_CONNECTIONS", "true")]), @@ -125,7 +174,12 @@ mod steal_tests { } let mut process = application - .run(&service.target, Some(&service.namespace), Some(flags), None) + .run( + &service.pod_container_target(), + Some(&service.namespace), + Some(flags), + None, + ) .await; // Verify that we hooked the socket operations and the agent started stealing. @@ -208,7 +262,7 @@ mod steal_tests { let mut process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![ @@ -288,7 +342,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -329,7 +383,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -370,7 +424,7 @@ mod steal_tests { let mut client = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), None, Some(vec![("MIRRORD_CONFIG_FILE", config_path.to_str().unwrap())]), @@ -423,7 +477,7 @@ mod steal_tests { let mut mirrored_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -494,7 +548,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -559,7 +613,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -629,7 +683,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(flags), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), @@ -708,7 +762,7 @@ mod steal_tests { let mut mirrorded_process = application .run( - &service.target, + &service.pod_container_target(), Some(&service.namespace), Some(vec!["--steal"]), Some(vec![("MIRRORD_HTTP_HEADER_FILTER", "x-filter: yes")]), diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 538a4d753e6..3b596efa357 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -4,7 +4,7 @@ use std::{ collections::HashMap, fmt::Debug, - net::Ipv4Addr, + net::IpAddr, ops::Not, path::PathBuf, process::{ExitStatus, Stdio}, @@ -39,6 +39,7 @@ use tokio::{ task::JoinHandle, }; +pub(crate) mod ipv6; pub mod sqs_resources; const TEXT: &str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."; @@ -90,6 +91,7 @@ fn format_time() -> String { pub enum Application { PythonFlaskHTTP, PythonFastApiHTTP, + PythonFastApiHTTPIPv6, NodeHTTP, NodeHTTP2, Go21HTTP, @@ -399,6 +401,15 @@ impl Application { "app_fastapi:app", ] } + Application::PythonFastApiHTTPIPv6 => { + vec![ + "uvicorn", + "--port=80", + "--host=::", + "--app-dir=./python-e2e/", + "app_fastapi:app", + ] + } Application::PythonCloseSocket => { vec!["python3", "-u", "python-e2e/close_socket.py"] } @@ -447,7 +458,7 @@ impl Application { } pub async fn assert(&self, process: &TestProcess) { - if let Application::PythonFastApiHTTP = self { + if matches!(self, Self::PythonFastApiHTTP | Self::PythonFastApiHTTPIPv6) { process.assert_log_level(true, "ERROR").await; process.assert_log_level(false, "ERROR").await; process.assert_log_level(true, "CRITICAL").await; @@ -728,15 +739,19 @@ impl Drop for ResourceGuard { pub struct KubeService { pub name: String, pub namespace: String, - pub target: String, guards: Vec, namespace_guard: Option, + pub pod_name: String, } impl KubeService { pub fn deployment_target(&self) -> String { format!("deployment/{}", self.name) } + + pub fn pod_container_target(&self) -> String { + format!("pod/{}/container/{CONTAINER_NAME}", self.pod_name) + } } impl Drop for KubeService { @@ -817,6 +832,17 @@ fn deployment_from_json(name: &str, image: &str, env: Value) -> Deployment { .expect("Failed creating `deployment` from json spec!") } +/// Change the `ipFamilies` and `ipFamilyPolicy` fields to make the service IPv6-only. +/// +/// # Panics +/// +/// Will panic if the given service does not have a spec. +fn set_ipv6_only(service: &mut Service) { + let spec = service.spec.as_mut().unwrap(); + spec.ip_families = Some(vec!["IPv6".to_string()]); + spec.ip_family_policy = Some("SingleStack".to_string()); +} + fn service_from_json(name: &str, service_type: &str) -> Service { serde_json::from_value(json!({ "apiVersion": "v1", @@ -1065,6 +1091,7 @@ pub async fn service( randomize_name, kube_client.await, default_env(), + false, ) .await } @@ -1093,6 +1120,7 @@ pub async fn service_with_env( randomize_name, kube_client, env, + false, ) .await } @@ -1108,6 +1136,7 @@ pub async fn service_with_env( /// This behavior can be changed, see [`PRESERVE_FAILED_ENV_NAME`]. /// * `randomize_name` - whether a random suffix should be added to the end of the resource names /// * `env` - `Value`, should be `Value::Array` of kubernetes container env var definitions. +#[allow(clippy::too_many_arguments)] async fn internal_service( namespace: &str, service_type: &str, @@ -1116,6 +1145,7 @@ async fn internal_service( randomize_name: bool, kube_client: Client, env: Value, + ipv6_only: bool, ) -> KubeService { let delete_after_fail = std::env::var_os(PRESERVE_FAILED_ENV_NAME).is_none(); @@ -1182,7 +1212,10 @@ async fn internal_service( watch_resource_exists(&deployment_api, &name).await; // `Service` - let service = service_from_json(&name, service_type); + let mut service = service_from_json(&name, service_type); + if ipv6_only { + set_ipv6_only(&mut service); + } let service_guard = ResourceGuard::create( service_api.clone(), name.clone(), @@ -1193,13 +1226,13 @@ async fn internal_service( .unwrap(); watch_resource_exists(&service_api, "default").await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1211,7 +1244,7 @@ async fn internal_service( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), + pod_name, guards: vec![pod_guard, service_guard], namespace_guard, } @@ -1304,13 +1337,13 @@ pub async fn service_for_mirrord_ls( .unwrap(); watch_resource_exists(&service_api, "default").await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1322,7 +1355,7 @@ pub async fn service_for_mirrord_ls( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), + pod_name, guards: vec![pod_guard, service_guard], namespace_guard, } @@ -1458,13 +1491,13 @@ pub async fn service_for_mirrord_ls( .unwrap(); watch_resource_exists(&job_api, &name).await; - let target = get_instance_name::(kube_client.clone(), &name, namespace) + let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) .await .unwrap(); let pod_api: Api = Api::namespaced(kube_client.clone(), namespace); - await_condition(pod_api, &target, is_pod_running()) + await_condition(pod_api, &pod_name, is_pod_running()) .await .unwrap(); @@ -1476,7 +1509,6 @@ pub async fn service_for_mirrord_ls( KubeService { name, namespace: namespace.to_string(), - target: format!("pod/{target}/container/{CONTAINER_NAME}"), guards: vec![ pod_guard, service_guard, @@ -1485,6 +1517,7 @@ pub async fn service_for_mirrord_ls( job_guard, ], namespace_guard, + pod_name, } } @@ -1607,12 +1640,15 @@ async fn get_pod_or_node_host(kube_client: Client, name: &str, namespace: &str) .next() .and_then(|pod| pod.status) .and_then(|status| status.host_ip) - .and_then(|ip| { - ip.parse::() - .unwrap() - .is_private() - .not() - .then_some(ip) + .filter(|ip| { + // use this IP only if it's a public one. + match ip.parse::().unwrap() { + IpAddr::V4(ip4) => ip4.is_private(), + IpAddr::V6(ip6) => { + ip6.is_unicast_link_local() || ip6.is_unique_local() || ip6.is_loopback() + } + } + .not() }) .unwrap_or_else(resolve_node_host) } diff --git a/tests/src/utils/ipv6.rs b/tests/src/utils/ipv6.rs new file mode 100644 index 00000000000..4b766e7f42d --- /dev/null +++ b/tests/src/utils/ipv6.rs @@ -0,0 +1,100 @@ +#![cfg(test)] + +use http_body_util::{BodyExt, Empty}; +use hyper::{ + client::{conn, conn::http1::SendRequest}, + Request, +}; +use k8s_openapi::api::core::v1::Pod; +use kube::{Api, Client}; +use rstest::fixture; + +use crate::utils::{internal_service, kube_client, KubeService}; + +/// Create a new [`KubeService`] and related Kubernetes resources. The resources will be deleted +/// when the returned service is dropped, unless it is dropped during panic. +/// This behavior can be changed, see +/// [`PRESERVE_FAILED_ENV_NAME`](crate::utils::PRESERVE_FAILED_ENV_NAME). +/// +/// * `randomize_name` - whether a random suffix should be added to the end of the resource names +#[fixture] +pub async fn ipv6_service( + #[default("default")] namespace: &str, + #[default("NodePort")] service_type: &str, + #[default("ghcr.io/metalbear-co/mirrord-pytest:latest")] image: &str, + #[default("http-echo")] service_name: &str, + #[default(true)] randomize_name: bool, + #[future] kube_client: Client, +) -> KubeService { + internal_service( + namespace, + service_type, + image, + service_name, + randomize_name, + kube_client.await, + serde_json::json!([ + { + "name": "HOST", + "value": "::" + } + ]), + true, + ) + .await +} + +/// Send an HTTP request using the referenced `request_sender`, with the provided `method`, +/// then verify a success status code, and a response body that is the used method. +/// +/// # Panics +/// - If the request cannot be sent. +/// - If the response's code is not OK +/// - If the response's body is not the method's name. +pub async fn send_request_with_method( + method: &str, + request_sender: &mut SendRequest>, +) { + let req = Request::builder() + .method(method) + .header("Host", "::") + .body(Empty::::new()) + .unwrap(); + + println!("Request: {:?}", req); + + let res = request_sender.send_request(req).await.unwrap(); + println!("Response: {:?}", res); + assert_eq!(res.status(), hyper::StatusCode::OK); + let bytes = res.collect().await.unwrap().to_bytes(); + let response_string = String::from_utf8(bytes.to_vec()).unwrap(); + assert_eq!(response_string, method); +} + +/// Create a portforward to the pod of the test service, and send HTTP requests over it. +/// Send four HTTP request (GET, POST, PUT, DELETE), using the referenced `request_sender`, with the +/// provided `method`, verify OK status, and a response body that is the used method. +/// +/// # Panics +/// - If a request cannot be sent. +/// - If a response's code is not OK +/// - If a response's body is not the method's name. +pub async fn portforward_http_requests(api: &Api, service: KubeService) { + let mut portforwarder = api + .portforward(&service.pod_name, &[80]) + .await + .expect("Failed to start portforward to test pod"); + + let stream = portforwarder.take_stream(80).unwrap(); + let stream = hyper_util::rt::TokioIo::new(stream); + + let (mut request_sender, connection) = conn::http1::handshake(stream).await.unwrap(); + tokio::spawn(async move { + if let Err(err) = connection.await { + eprintln!("Error in connection from test function to deployed test app {err:#?}"); + } + }); + for method in ["GET", "POST", "PUT", "DELETE"] { + send_request_with_method(method, &mut request_sender).await; + } +} From 007597cd38d523a159b5e611c2ec27f36d1b5fcf Mon Sep 17 00:00:00 2001 From: meowjesty <43983236+meowjesty@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:34:22 -0300 Subject: [PATCH 08/23] Fix fs policy test. (#3010) * Fix fs policy test. * changelog * no changelog --- changelog.d/+104-policy-fs.added.md | 2 +- tests/src/operator/policies/fs.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog.d/+104-policy-fs.added.md b/changelog.d/+104-policy-fs.added.md index 1ad43a13736..312cf56f3fa 100644 --- a/changelog.d/+104-policy-fs.added.md +++ b/changelog.d/+104-policy-fs.added.md @@ -1 +1 @@ -Add policy to control file ops. +Add policy to control file ops from the operator. diff --git a/tests/src/operator/policies/fs.rs b/tests/src/operator/policies/fs.rs index afa2cdf62bc..3feae1d514c 100644 --- a/tests/src/operator/policies/fs.rs +++ b/tests/src/operator/policies/fs.rs @@ -26,12 +26,12 @@ async fn fs_service(#[future] kube_client: kube::Client) -> KubeService { #[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[timeout(Duration::from_secs(60))] -pub async fn create_cluster_fs_policy_and_try_file_operations( - #[future] service: KubeService, +pub async fn create_namespaced_fs_policy_and_try_file_open( + #[future] fs_service: KubeService, #[future] kube_client: kube::Client, ) { let kube_client = kube_client.await; - let service = service.await; + let service = fs_service.await; // Create policy, delete it when test exits. let _policy_guard = PolicyGuard::namespaced( From 7c945412fbfdb5857b13445383ff6477eb884e12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:08:48 +0100 Subject: [PATCH 09/23] Pin cargo chef (#3014) * Pinned cargo-chef to 0.1.68 * Changelog --- changelog.d/+pinned-cargo-chef.internal.md | 1 + mirrord/agent/Dockerfile | 3 ++- mirrord/cli/Dockerfile | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/+pinned-cargo-chef.internal.md diff --git a/changelog.d/+pinned-cargo-chef.internal.md b/changelog.d/+pinned-cargo-chef.internal.md new file mode 100644 index 00000000000..8b18e6071d2 --- /dev/null +++ b/changelog.d/+pinned-cargo-chef.internal.md @@ -0,0 +1 @@ +Pinned `cargo-chef` version to `0.1.68` in the dockerfiles. \ No newline at end of file diff --git a/mirrord/agent/Dockerfile b/mirrord/agent/Dockerfile index 83c97871cae..e9de8df84d6 100644 --- a/mirrord/agent/Dockerfile +++ b/mirrord/agent/Dockerfile @@ -8,7 +8,8 @@ RUN ./platform.sh # this takes around 1 minute since libgit2 is slow https://github.com/rust-lang/cargo/issues/9167 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -RUN cargo install cargo-chef +# cargo-chef 0.1.69 breaks the build +RUN cargo install cargo-chef@0.1.68 FROM chef AS planner diff --git a/mirrord/cli/Dockerfile b/mirrord/cli/Dockerfile index 81e37142055..144f4a0e162 100644 --- a/mirrord/cli/Dockerfile +++ b/mirrord/cli/Dockerfile @@ -8,7 +8,8 @@ RUN ./platform.sh # this takes around 1 minute since libgit2 is slow https://github.com/rust-lang/cargo/issues/9167 ENV CARGO_NET_GIT_FETCH_WITH_CLI=true -RUN cargo install cargo-chef +# cargo-chef 0.1.69 breaks the build +RUN cargo install cargo-chef@0.1.68 FROM chef AS planner From 6668e3ccac9236486634a161844cf486b46dba14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:48:20 +0100 Subject: [PATCH 10/23] Fixed fs policy E2E test (#3012) * Fixed case of FsPolicy fields, improved docs * Fixed FsPolicy test and improved test utils * Changelog * Fixed case of envpolicy field --- changelog.d/+fs-policy-test.internal.md | 1 + mirrord/operator/src/crd/policy.rs | 21 +- .../fspolicy/test_operator_fs_policy.mjs | 69 ++---- tests/src/operator/policies.rs | 22 +- tests/src/operator/policies/fs.rs | 43 ++-- tests/src/utils.rs | 213 +++++++++--------- 6 files changed, 172 insertions(+), 197 deletions(-) create mode 100644 changelog.d/+fs-policy-test.internal.md diff --git a/changelog.d/+fs-policy-test.internal.md b/changelog.d/+fs-policy-test.internal.md new file mode 100644 index 00000000000..2ebd288613f --- /dev/null +++ b/changelog.d/+fs-policy-test.internal.md @@ -0,0 +1 @@ +Fixed fs policy E2E test. diff --git a/mirrord/operator/src/crd/policy.rs b/mirrord/operator/src/crd/policy.rs index e236164da98..cf712606d3c 100644 --- a/mirrord/operator/src/crd/policy.rs +++ b/mirrord/operator/src/crd/policy.rs @@ -104,7 +104,7 @@ pub struct MirrordClusterPolicySpec { /// Policy for controlling environment variables access from mirrord instances. #[derive(Clone, Default, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] pub struct EnvPolicy { /// List of environment variables that should be excluded when using mirrord. /// @@ -123,20 +123,29 @@ pub struct EnvPolicy { /// Allows the operator control over remote file ops behaviour, overriding what the user has set in /// their mirrord config file, if it matches something in one of the lists (regex sets) of this /// struct. +/// +/// If the file path matches regexes in multiple sets, priority is as follows: +/// 1. `local` +/// 2. `notFound` +/// 3. `readOnly` #[derive(Clone, Default, Debug, Deserialize, Eq, PartialEq, Serialize, JsonSchema)] -#[serde(rename_all = "kebab-case")] +#[serde(rename_all = "camelCase")] pub struct FsPolicy { - /// The file can only be opened in read-only mode, otherwise the operator returns an IO error. + /// Files that cannot be opened for writing. + /// + /// Opening the file for writing is rejected with an IO error. #[serde(default)] pub read_only: HashSet, - /// The file cannot be opened in the remote target. + /// Files that cannot be opened at all. /// - /// `open` calls that match this are forced to be opened in the local user's machine. + /// Opening the file will be rejected and mirrord will open the file locally instead. #[serde(default)] pub local: HashSet, - /// Any file that matches this returns a file not found error from the operator. + /// Files that cannot be opened at all. + /// + /// Opening the file is rejected with an IO error. #[serde(default)] pub not_found: HashSet, } diff --git a/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs b/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs index 8e58bf52cec..af84f17ee50 100644 --- a/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs +++ b/tests/node-e2e/fspolicy/test_operator_fs_policy.mjs @@ -1,54 +1,19 @@ import fs from 'fs'; -fs.open("/app/file.local", (fail, fd) => { - console.log(`open file.local ${fd}`); - if (fd) { - console.log(`SUCCESS /app/file.local ${fd}`); - } - - if (fail) { - console.error(`FAIL /app/file.local ${fail}`); - } -}); - -fs.open("/app/file.not-found", (fail, fd) => { - console.log(`open file.not-found ${fd}`); - if (fd) { - console.log(`SUCCESS /app/file.not-found ${fd}`); - } - - if (fail) { - console.error(`FAIL /app/file.not-found ${fail}`); - } -}); - -fs.open("/app/file.read-only", (fail, fd) => { - if (fd) { - console.log(`SUCCESS /app/file.read-only ${fd}`); - } - - if (fail) { - console.error(`FAIL /app/file.read-only ${fail}`); - } -}); - -fs.open("/app/file.read-only", "r+", (fail, fd) => { - if (fd) { - console.log(`SUCCESS r+ /app/file.read-only ${fd}`); - } - - if (fail) { - console.error(`FAIL r+ /app/file.read-only ${fail}`); - } -}); - -fs.open("/app/file.read-write", "r+", (fail, fd) => { - if (fd) { - console.log(`SUCCESS /app/file.read-write ${fd}`); - } - - if (fail) { - console.error(`FAIL /app/file.read-write ${fail}`); - } -}); - +function test_open(path, mode) { + fs.open(path, mode, (fail, fd) => { + if (fd) { + console.log(`SUCCESS ${mode} ${path} ${fd}`); + } + + if (fail) { + console.log(`FAIL ${mode} ${path} ${fail}`); + } + }); +} + +test_open("/app/file.local", "r"); +test_open("/app/file.not-found", "r"); +test_open("/app/file.read-only", "r"); +test_open("/app/file.read-only", "r+"); +test_open("/app/file.read-write", "r+"); diff --git a/tests/src/operator/policies.rs b/tests/src/operator/policies.rs index 3684a97fc20..2a0a8465bbe 100644 --- a/tests/src/operator/policies.rs +++ b/tests/src/operator/policies.rs @@ -33,28 +33,18 @@ impl PolicyGuard { ) -> Self { let policy_api: Api = Api::namespaced(kube_client.clone(), namespace); PolicyGuard { - _inner: ResourceGuard::create( - policy_api, - policy.metadata.name.clone().unwrap(), - policy, - true, - ) - .await - .expect("Could not create policy in E2E test."), + _inner: ResourceGuard::create(policy_api, policy, true) + .await + .expect("Could not create policy in E2E test."), } } pub async fn clusterwide(kube_client: kube::Client, policy: &MirrordClusterPolicy) -> Self { let policy_api: Api = Api::all(kube_client.clone()); PolicyGuard { - _inner: ResourceGuard::create( - policy_api, - policy.metadata.name.clone().unwrap(), - policy, - true, - ) - .await - .expect("Could not create policy in E2E test."), + _inner: ResourceGuard::create(policy_api, policy, true) + .await + .expect("Could not create policy in E2E test."), } } } diff --git a/tests/src/operator/policies/fs.rs b/tests/src/operator/policies/fs.rs index 3feae1d514c..79a1b7e7202 100644 --- a/tests/src/operator/policies/fs.rs +++ b/tests/src/operator/policies/fs.rs @@ -39,14 +39,14 @@ pub async fn create_namespaced_fs_policy_and_try_file_open( &MirrordPolicy::new( "e2e-test-fs-policy-with-path-pattern", MirrordPolicySpec { - target_path: Some("fs_policy_e2e-test-*".into()), + target_path: Some("*fs-policy-e2e-test-*".into()), selector: None, block: Default::default(), env: Default::default(), fs: FsPolicy { - read_only: HashSet::from_iter(vec!["file.read-only".to_string()]), - local: HashSet::from_iter(vec!["file.local".to_string()]), - not_found: HashSet::from_iter(vec!["file.not-found".to_string()]), + read_only: HashSet::from_iter(vec!["file\\.read-only".to_string()]), + local: HashSet::from_iter(vec!["file\\.local".to_string()]), + not_found: HashSet::from_iter(vec!["file\\.not-found".to_string()]), }, }, ), @@ -68,20 +68,25 @@ pub async fn create_namespaced_fs_policy_and_try_file_open( test_process.wait_assert_success().await; - test_process - .assert_stderr_contains("FAIL /app/file.local") - .await; - test_process - .assert_stderr_contains("FAIL /app/file.not-found") - .await; - test_process - .assert_stderr_contains("FAIL r+ /app/file.read-only") - .await; + let stdout = test_process.get_stdout().await; - test_process - .assert_stdout_contains("SUCCESS /app/file.read-only") - .await; - test_process - .assert_stdout_contains("SUCCESS /app/file.read-write") - .await; + let reading_local_failed = stdout.contains("FAIL r /app/file.local"); + let reading_not_found_failed = stdout.contains("FAIL r /app/file.not-found"); + let reading_read_only_succeeded = stdout.contains("SUCCESS r /app/file.read-only"); + let writing_read_only_failed = stdout.contains("FAIL r+ /app/file.read-only"); + let writing_read_write_succeeded = stdout.contains("SUCCESS r+ /app/file.read-write"); + + assert!( + reading_local_failed + && reading_not_found_failed + && reading_read_only_succeeded + && writing_read_only_failed + && writing_read_write_succeeded, + "some file operations did not finish as expected:\n + \treading_local_failed={reading_local_failed}\n + \treading_not_found_failed={reading_not_found_failed}\n + \treading_read_only_succeeded={reading_read_only_succeeded} \n + \twriting_read_only_failed={writing_read_only_failed}\n + \twriting_read_write_succeeded={writing_read_write_succeeded}", + ) } diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 3b596efa357..bf8e0d1a79b 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -6,6 +6,7 @@ use std::{ fmt::Debug, net::IpAddr, ops::Not, + os::unix::process::ExitStatusExt, path::PathBuf, process::{ExitStatus, Stdio}, sync::{Arc, Once}, @@ -24,7 +25,7 @@ use kube::{ api::{DeleteParams, ListParams, PostParams, WatchParams}, core::WatchEvent, runtime::wait::{await_condition, conditions::is_pod_running}, - Api, Client, Config, Error, + Api, Client, Config, Error, Resource, }; use rand::{distributions::Alphanumeric, Rng}; use reqwest::{RequestBuilder, StatusCode}; @@ -155,68 +156,111 @@ impl TestProcess { pub async fn assert_log_level(&self, stderr: bool, level: &str) { if stderr { - assert!(!self.stderr_data.read().await.contains(level)); + assert!( + self.stderr_data.read().await.contains(level).not(), + "application stderr should not contain `{level}`" + ); } else { - assert!(!self.stdout_data.read().await.contains(level)); + assert!( + self.stdout_data.read().await.contains(level).not(), + "application stdout should not contain `{level}`" + ); } } pub async fn assert_python_fileops_stderr(&self) { - assert!(!self.stderr_data.read().await.contains("FAILED")); + assert!( + self.stderr_data.read().await.contains("FAILED").not(), + "application stderr should not contain `FAILED`" + ); } pub async fn wait_assert_success(&mut self) { let output = self.wait().await; - assert!(output.success()); + assert!( + output.success(), + "application unexpectedly failed: exit code {:?}, signal code {:?}", + output.code(), + output.signal(), + ); } pub async fn wait_assert_fail(&mut self) { let output = self.wait().await; - assert!(!output.success()); + assert!( + output.success().not(), + "application unexpectedly succeeded: exit code {:?}, signal code {:?}", + output.code(), + output.signal() + ); } pub async fn assert_stdout_contains(&self, string: &str) { - assert!(self.get_stdout().await.contains(string)); + assert!( + self.get_stdout().await.contains(string), + "application stdout should contain `{string}`", + ); } pub async fn assert_stdout_doesnt_contain(&self, string: &str) { - assert!(!self.get_stdout().await.contains(string)); + assert!( + self.get_stdout().await.contains(string).not(), + "application stdout should not contain `{string}`", + ); } pub async fn assert_stderr_contains(&self, string: &str) { - assert!(self.get_stderr().await.contains(string)); + assert!( + self.get_stderr().await.contains(string), + "application stderr should contain `{string}`", + ); } pub async fn assert_stderr_doesnt_contain(&self, string: &str) { - assert!(!self.get_stderr().await.contains(string)); + assert!( + self.get_stderr().await.contains(string).not(), + "application stderr should not contain `{string}`", + ); } pub async fn assert_no_error_in_stdout(&self) { - assert!(!self - .error_capture - .is_match(&self.stdout_data.read().await) - .unwrap()); + assert!( + self.error_capture + .is_match(&self.stdout_data.read().await) + .unwrap() + .not(), + "application stdout contains an error" + ); } pub async fn assert_no_error_in_stderr(&self) { - assert!(!self - .error_capture - .is_match(&self.stderr_data.read().await) - .unwrap()); + assert!( + self.error_capture + .is_match(&self.stderr_data.read().await) + .unwrap() + .not(), + "application stderr contains an error" + ); } pub async fn assert_no_warn_in_stdout(&self) { - assert!(!self - .warn_capture - .is_match(&self.stdout_data.read().await) - .unwrap()); + assert!( + self.warn_capture + .is_match(&self.stdout_data.read().await) + .unwrap() + .not(), + "application stdout contains a warning" + ); } pub async fn assert_no_warn_in_stderr(&self) { - assert!(!self - .warn_capture - .is_match(&self.stderr_data.read().await) - .unwrap()); + assert!( + self.warn_capture + .is_match(&self.stderr_data.read().await) + .unwrap() + .not(), + "application stderr contains a warning" + ); } pub async fn wait_for_line(&self, timeout: Duration, line: &str) { @@ -673,23 +717,27 @@ pub(crate) struct ResourceGuard { impl ResourceGuard { /// Create a kube resource and spawn a task to delete it when this guard is dropped. /// Return [`Error`] if creating the resource failed. - pub async fn create( + pub async fn create< + K: Resource + Debug + Clone + DeserializeOwned + Serialize + 'static, + >( api: Api, - name: String, data: &K, delete_on_fail: bool, ) -> Result { + let name = data.meta().name.clone().unwrap(); + println!("Creating {} `{name}`: {data:?}", K::kind(&())); api.create(&PostParams::default(), data).await?; + println!("Created {} `{name}`", K::kind(&())); let deleter = async move { - println!("Deleting resource `{name}`"); + println!("Deleting {} `{name}`", K::kind(&())); let delete_params = DeleteParams { grace_period_seconds: Some(0), ..Default::default() }; let res = api.delete(&name, &delete_params).await; if let Err(e) = res { - println!("Failed to delete resource `{name}`: {e:?}"); + println!("Failed to delete {} `{name}`: {e:?}", K::kind(&())); } }; @@ -1171,7 +1219,7 @@ async fn internal_service( }; println!( - "{} creating service {name:?} in namespace {namespace:?}", + "{} creating service {name} in namespace {namespace}", format_time() ); @@ -1189,7 +1237,6 @@ async fn internal_service( // Create namespace and wrap it in ResourceGuard if it does not yet exist. let namespace_guard = ResourceGuard::create( namespace_api.clone(), - namespace.to_string(), &namespace_resource, delete_after_fail, ) @@ -1201,14 +1248,9 @@ async fn internal_service( // `Deployment` let deployment = deployment_from_json(&name, image, env); - let pod_guard = ResourceGuard::create( - deployment_api.clone(), - name.to_string(), - &deployment, - delete_after_fail, - ) - .await - .unwrap(); + let pod_guard = ResourceGuard::create(deployment_api.clone(), &deployment, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&deployment_api, &name).await; // `Service` @@ -1216,14 +1258,9 @@ async fn internal_service( if ipv6_only { set_ipv6_only(&mut service); } - let service_guard = ResourceGuard::create( - service_api.clone(), - name.clone(), - &service, - delete_after_fail, - ) - .await - .unwrap(); + let service_guard = ResourceGuard::create(service_api.clone(), &service, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&service_api, "default").await; let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) @@ -1237,8 +1274,8 @@ async fn internal_service( .unwrap(); println!( - "{:?} done creating service {name:?} in namespace {namespace:?}", - Utc::now() + "{} done creating service {name} in namespace {namespace}", + format_time(), ); KubeService { @@ -1303,7 +1340,6 @@ pub async fn service_for_mirrord_ls( // Create namespace and wrap it in ResourceGuard if it does not yet exist. let namespace_guard = ResourceGuard::create( namespace_api.clone(), - namespace.to_string(), &namespace_resource, delete_after_fail, ) @@ -1315,26 +1351,16 @@ pub async fn service_for_mirrord_ls( // `Deployment` let deployment = deployment_from_json(&name, image, default_env()); - let pod_guard = ResourceGuard::create( - deployment_api.clone(), - name.to_string(), - &deployment, - delete_after_fail, - ) - .await - .unwrap(); + let pod_guard = ResourceGuard::create(deployment_api.clone(), &deployment, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&deployment_api, &name).await; // `Service` let service = service_from_json(&name, service_type); - let service_guard = ResourceGuard::create( - service_api.clone(), - name.clone(), - &service, - delete_after_fail, - ) - .await - .unwrap(); + let service_guard = ResourceGuard::create(service_api.clone(), &service, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&service_api, "default").await; let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) @@ -1425,7 +1451,6 @@ pub async fn service_for_mirrord_ls( // Create namespace and wrap it in ResourceGuard if it does not yet exist. let namespace_guard = ResourceGuard::create( namespace_api.clone(), - namespace.to_string(), &namespace_resource, delete_after_fail, ) @@ -1437,58 +1462,38 @@ pub async fn service_for_mirrord_ls( // `Deployment` let deployment = deployment_from_json(&name, image, default_env()); - let pod_guard = ResourceGuard::create( - deployment_api.clone(), - name.to_string(), - &deployment, - delete_after_fail, - ) - .await - .unwrap(); + let pod_guard = ResourceGuard::create(deployment_api.clone(), &deployment, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&deployment_api, &name).await; // `Service` let service = service_from_json(&name, service_type); - let service_guard = ResourceGuard::create( - service_api.clone(), - name.clone(), - &service, - delete_after_fail, - ) - .await - .unwrap(); + let service_guard = ResourceGuard::create(service_api.clone(), &service, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&service_api, "default").await; // `StatefulSet` let stateful_set = stateful_set_from_json(&name, image); - let stateful_set_guard = ResourceGuard::create( - stateful_set_api.clone(), - name.to_string(), - &stateful_set, - delete_after_fail, - ) - .await - .unwrap(); + let stateful_set_guard = + ResourceGuard::create(stateful_set_api.clone(), &stateful_set, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&stateful_set_api, &name).await; // `CronJob` let cron_job = cron_job_from_json(&name, image); - let cron_job_guard = ResourceGuard::create( - cron_job_api.clone(), - name.to_string(), - &cron_job, - delete_after_fail, - ) - .await - .unwrap(); + let cron_job_guard = ResourceGuard::create(cron_job_api.clone(), &cron_job, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&cron_job_api, &name).await; // `Job` let job = job_from_json(&name, image); - let job_guard = - ResourceGuard::create(job_api.clone(), name.to_string(), &job, delete_after_fail) - .await - .unwrap(); + let job_guard = ResourceGuard::create(job_api.clone(), &job, delete_after_fail) + .await + .unwrap(); watch_resource_exists(&job_api, &name).await; let pod_name = get_instance_name::(kube_client.clone(), &name, namespace) From 1aca6426fe07de97058d607a4cc207717282fbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Tue, 14 Jan 2025 19:38:43 +0100 Subject: [PATCH 11/23] 3.129.0 (#3019) --- CHANGELOG.md | 43 ++++++++++++++ Cargo.lock | 56 +++++++++---------- Cargo.toml | 2 +- .../+103-policy-env-vars-exclude.added.md | 1 - changelog.d/+104-policy-fs.added.md | 1 - changelog.d/+fs-policy-test.internal.md | 1 - changelog.d/+http-filter-docs.changed.md | 1 - .../+mirrord-policy-rejection.fixed.md | 1 - changelog.d/+pinned-cargo-chef.internal.md | 1 - changelog.d/2843.internal.md | 1 - changelog.d/2868.changed.md | 1 - changelog.d/2956.added.md | 1 - changelog.d/2986.changed.md | 1 - changelog.d/2988.fixed.md | 1 - changelog.d/2992.fixed.md | 1 - changelog.d/2999.added.md | 1 - changelog.d/3004.changed.md | 1 - 17 files changed, 72 insertions(+), 43 deletions(-) delete mode 100644 changelog.d/+103-policy-env-vars-exclude.added.md delete mode 100644 changelog.d/+104-policy-fs.added.md delete mode 100644 changelog.d/+fs-policy-test.internal.md delete mode 100644 changelog.d/+http-filter-docs.changed.md delete mode 100644 changelog.d/+mirrord-policy-rejection.fixed.md delete mode 100644 changelog.d/+pinned-cargo-chef.internal.md delete mode 100644 changelog.d/2843.internal.md delete mode 100644 changelog.d/2868.changed.md delete mode 100644 changelog.d/2956.added.md delete mode 100644 changelog.d/2986.changed.md delete mode 100644 changelog.d/2988.fixed.md delete mode 100644 changelog.d/2992.fixed.md delete mode 100644 changelog.d/2999.added.md delete mode 100644 changelog.d/3004.changed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c52d3d77a21..87bf68e74a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,49 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [3.129.0](https://github.com/metalbear-co/mirrord/tree/3.129.0) - 2025-01-14 + + +### Added + +- Support for stealing incoming connections that are over IPv6. + [#2956](https://github.com/metalbear-co/mirrord/issues/2956) +- mirrord policy to control file ops from the operator. +- mirrord policy to restrict fetching remote environment variables. + + +### Changed + +- Updated how intproxy is outputing logfile when using container mode, now logs + will be written on host machine. + [#2868](https://github.com/metalbear-co/mirrord/issues/2868) +- Changed log level for debugger ports detection. + [#2986](https://github.com/metalbear-co/mirrord/issues/2986) +- Readonly file buffering is not enabled by default to improve performance + [#3004](https://github.com/metalbear-co/mirrord/issues/3004) +- Extended docs for HTTP filter in the mirrord config. + + +### Fixed + +- Fixed panic when Go >=1.23.3 verifies pidfd support on Linux. + [#2988](https://github.com/metalbear-co/mirrord/issues/2988) +- Fix misleading agent IO operation error that always mentioned getaddrinfo. + [#2992](https://github.com/metalbear-co/mirrord/issues/2992) +- Fixed a bug where port mirroring block (due to active mirrord policies) would + terminate the mirrord session. + + +### Internal + +- Added lint for unused crate dependencies. + [#2843](https://github.com/metalbear-co/mirrord/issues/2843) +- Fixed fs policy E2E test. +- Pinned `cargo-chef` version to `0.1.68` in the dockerfiles. +- Added available namespaces to `mirrord ls` output. New output format is + enabled with a flag in an environment variable. + [#2999](https://github.com/metalbear-co/mirrord/issues/2999) + ## [3.128.0](https://github.com/metalbear-co/mirrord/tree/3.128.0) - 2024-12-19 diff --git a/Cargo.lock b/Cargo.lock index 89528a03ca1..941d424a4c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2400,7 +2400,7 @@ dependencies = [ [[package]] name = "fileops" -version = "3.128.0" +version = "3.129.0" dependencies = [ "libc", ] @@ -3518,7 +3518,7 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "issue1317" -version = "3.128.0" +version = "3.129.0" dependencies = [ "actix-web", "env_logger 0.11.6", @@ -3528,7 +3528,7 @@ dependencies = [ [[package]] name = "issue1776" -version = "3.128.0" +version = "3.129.0" dependencies = [ "errno 0.3.10", "libc", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "issue1776portnot53" -version = "3.128.0" +version = "3.129.0" dependencies = [ "libc", "socket2", @@ -3545,14 +3545,14 @@ dependencies = [ [[package]] name = "issue1899" -version = "3.128.0" +version = "3.129.0" dependencies = [ "libc", ] [[package]] name = "issue2001" -version = "3.128.0" +version = "3.129.0" dependencies = [ "libc", ] @@ -3873,7 +3873,7 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "listen_ports" -version = "3.128.0" +version = "3.129.0" [[package]] name = "litemap" @@ -4114,7 +4114,7 @@ dependencies = [ [[package]] name = "mirrord" -version = "3.128.0" +version = "3.129.0" dependencies = [ "actix-codec", "clap", @@ -4170,7 +4170,7 @@ dependencies = [ [[package]] name = "mirrord-agent" -version = "3.128.0" +version = "3.129.0" dependencies = [ "actix-codec", "async-trait", @@ -4225,7 +4225,7 @@ dependencies = [ [[package]] name = "mirrord-analytics" -version = "3.128.0" +version = "3.129.0" dependencies = [ "assert-json-diff", "base64 0.22.1", @@ -4239,7 +4239,7 @@ dependencies = [ [[package]] name = "mirrord-auth" -version = "3.128.0" +version = "3.129.0" dependencies = [ "bcder", "chrono", @@ -4260,7 +4260,7 @@ dependencies = [ [[package]] name = "mirrord-config" -version = "3.128.0" +version = "3.129.0" dependencies = [ "bimap", "bitflags 2.6.0", @@ -4283,7 +4283,7 @@ dependencies = [ [[package]] name = "mirrord-config-derive" -version = "3.128.0" +version = "3.129.0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -4293,7 +4293,7 @@ dependencies = [ [[package]] name = "mirrord-console" -version = "3.128.0" +version = "3.129.0" dependencies = [ "bincode", "drain", @@ -4309,7 +4309,7 @@ dependencies = [ [[package]] name = "mirrord-intproxy" -version = "3.128.0" +version = "3.129.0" dependencies = [ "bytes", "exponential-backoff", @@ -4340,7 +4340,7 @@ dependencies = [ [[package]] name = "mirrord-intproxy-protocol" -version = "3.128.0" +version = "3.129.0" dependencies = [ "bincode", "mirrord-protocol", @@ -4350,7 +4350,7 @@ dependencies = [ [[package]] name = "mirrord-kube" -version = "3.128.0" +version = "3.129.0" dependencies = [ "actix-codec", "async-stream", @@ -4374,7 +4374,7 @@ dependencies = [ [[package]] name = "mirrord-layer" -version = "3.128.0" +version = "3.129.0" dependencies = [ "actix-codec", "base64 0.22.1", @@ -4417,7 +4417,7 @@ dependencies = [ [[package]] name = "mirrord-layer-macro" -version = "3.128.0" +version = "3.129.0" dependencies = [ "proc-macro2", "quote", @@ -4426,7 +4426,7 @@ dependencies = [ [[package]] name = "mirrord-macros" -version = "3.128.0" +version = "3.129.0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -4436,7 +4436,7 @@ dependencies = [ [[package]] name = "mirrord-operator" -version = "3.128.0" +version = "3.129.0" dependencies = [ "base64 0.22.1", "bincode", @@ -4469,7 +4469,7 @@ dependencies = [ [[package]] name = "mirrord-progress" -version = "3.128.0" +version = "3.129.0" dependencies = [ "enum_dispatch", "indicatif", @@ -4504,7 +4504,7 @@ dependencies = [ [[package]] name = "mirrord-sip" -version = "3.128.0" +version = "3.129.0" dependencies = [ "apple-codesign", "object 0.36.7", @@ -4517,7 +4517,7 @@ dependencies = [ [[package]] name = "mirrord-vpn" -version = "3.128.0" +version = "3.129.0" dependencies = [ "futures", "ipnet", @@ -4865,7 +4865,7 @@ dependencies = [ [[package]] name = "outgoing" -version = "3.128.0" +version = "3.129.0" [[package]] name = "outref" @@ -5926,14 +5926,14 @@ dependencies = [ [[package]] name = "rust-bypassed-unix-socket" -version = "3.128.0" +version = "3.129.0" dependencies = [ "tokio", ] [[package]] name = "rust-e2e-fileops" -version = "3.128.0" +version = "3.129.0" dependencies = [ "libc", ] @@ -5949,7 +5949,7 @@ dependencies = [ [[package]] name = "rust-unix-socket-client" -version = "3.128.0" +version = "3.129.0" dependencies = [ "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index da8a4e788d0..9a2b2a42cae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ resolver = "2" # latest commits on rustls suppress certificate verification [workspace.package] -version = "3.128.0" +version = "3.129.0" edition = "2021" license = "MIT" readme = "README.md" diff --git a/changelog.d/+103-policy-env-vars-exclude.added.md b/changelog.d/+103-policy-env-vars-exclude.added.md deleted file mode 100644 index 2e1bde95ebd..00000000000 --- a/changelog.d/+103-policy-env-vars-exclude.added.md +++ /dev/null @@ -1 +0,0 @@ -Add policy to exclude env vars. diff --git a/changelog.d/+104-policy-fs.added.md b/changelog.d/+104-policy-fs.added.md deleted file mode 100644 index 312cf56f3fa..00000000000 --- a/changelog.d/+104-policy-fs.added.md +++ /dev/null @@ -1 +0,0 @@ -Add policy to control file ops from the operator. diff --git a/changelog.d/+fs-policy-test.internal.md b/changelog.d/+fs-policy-test.internal.md deleted file mode 100644 index 2ebd288613f..00000000000 --- a/changelog.d/+fs-policy-test.internal.md +++ /dev/null @@ -1 +0,0 @@ -Fixed fs policy E2E test. diff --git a/changelog.d/+http-filter-docs.changed.md b/changelog.d/+http-filter-docs.changed.md deleted file mode 100644 index 75a6917214a..00000000000 --- a/changelog.d/+http-filter-docs.changed.md +++ /dev/null @@ -1 +0,0 @@ -Extended docs for HTTP filter in the mirrord config. \ No newline at end of file diff --git a/changelog.d/+mirrord-policy-rejection.fixed.md b/changelog.d/+mirrord-policy-rejection.fixed.md deleted file mode 100644 index ad7415c5411..00000000000 --- a/changelog.d/+mirrord-policy-rejection.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed a bug where port mirroring block (due to active mirrord policies) would terminate the mirrord session. diff --git a/changelog.d/+pinned-cargo-chef.internal.md b/changelog.d/+pinned-cargo-chef.internal.md deleted file mode 100644 index 8b18e6071d2..00000000000 --- a/changelog.d/+pinned-cargo-chef.internal.md +++ /dev/null @@ -1 +0,0 @@ -Pinned `cargo-chef` version to `0.1.68` in the dockerfiles. \ No newline at end of file diff --git a/changelog.d/2843.internal.md b/changelog.d/2843.internal.md deleted file mode 100644 index c3587426952..00000000000 --- a/changelog.d/2843.internal.md +++ /dev/null @@ -1 +0,0 @@ -Added lint for unused crate dependencies. \ No newline at end of file diff --git a/changelog.d/2868.changed.md b/changelog.d/2868.changed.md deleted file mode 100644 index f24bdc1461c..00000000000 --- a/changelog.d/2868.changed.md +++ /dev/null @@ -1 +0,0 @@ -Updated how intproxy is outputing logfile when using container mode, now logs will be written on host machine. diff --git a/changelog.d/2956.added.md b/changelog.d/2956.added.md deleted file mode 100644 index 5cadfd68a24..00000000000 --- a/changelog.d/2956.added.md +++ /dev/null @@ -1 +0,0 @@ -Support for stealing incoming connections that are over IPv6. diff --git a/changelog.d/2986.changed.md b/changelog.d/2986.changed.md deleted file mode 100644 index f17b5621854..00000000000 --- a/changelog.d/2986.changed.md +++ /dev/null @@ -1 +0,0 @@ -Changed log level for debugger ports detection. \ No newline at end of file diff --git a/changelog.d/2988.fixed.md b/changelog.d/2988.fixed.md deleted file mode 100644 index 95b7074d056..00000000000 --- a/changelog.d/2988.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixed panic when Go >=1.23.3 verifies pidfd support on Linux. \ No newline at end of file diff --git a/changelog.d/2992.fixed.md b/changelog.d/2992.fixed.md deleted file mode 100644 index 0e907384f90..00000000000 --- a/changelog.d/2992.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix misleading agent IO operation error that always mentioned getaddrinfo. diff --git a/changelog.d/2999.added.md b/changelog.d/2999.added.md deleted file mode 100644 index ac9b5676f4d..00000000000 --- a/changelog.d/2999.added.md +++ /dev/null @@ -1 +0,0 @@ -Added available namespaces to `mirrord ls` output. New output format is enabled with `--rich` flag. diff --git a/changelog.d/3004.changed.md b/changelog.d/3004.changed.md deleted file mode 100644 index 4a6367bf967..00000000000 --- a/changelog.d/3004.changed.md +++ /dev/null @@ -1 +0,0 @@ -use file buffering by default to improve performance From 36bfd3c946bf0b691fed1468667f819d702e257c Mon Sep 17 00:00:00 2001 From: Gemma <58080601+gememma@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:34:54 +0000 Subject: [PATCH 12/23] Prevent config rendering on every process (#3011) * Add detecting encoded config from env var * [WIP] check for empty env var value * Skip test_error config field when deserializing * Add unit test to check if encoding and decoding matches the original config * Changelog * Fix clippy * Remove direct interactions with env vars for fields in config, use update_env() instead * Various * Add method to forcefully recalculate config instead of taking from existing env * Revert to using custom ser/deser on BiMap in config * Doc comments --- Cargo.lock | 1 + changelog.d/2936.fixed.md | 1 + mirrord/cli/src/container.rs | 2 + mirrord/cli/src/execution.rs | 8 +- mirrord/cli/src/extension.rs | 6 +- mirrord/cli/src/external_proxy.rs | 2 +- mirrord/cli/src/internal_proxy.rs | 2 +- mirrord/cli/src/main.rs | 15 +- mirrord/config/Cargo.toml | 7 +- mirrord/config/src/agent.rs | 6 +- mirrord/config/src/config.rs | 6 + mirrord/config/src/container.rs | 4 +- mirrord/config/src/experimental.rs | 4 +- mirrord/config/src/external_proxy.rs | 4 +- mirrord/config/src/feature.rs | 4 +- mirrord/config/src/feature/copy_target.rs | 2 +- mirrord/config/src/feature/env.rs | 4 +- mirrord/config/src/feature/fs/advanced.rs | 4 +- mirrord/config/src/feature/network.rs | 4 +- mirrord/config/src/feature/network/dns.rs | 2 +- .../config/src/feature/network/incoming.rs | 27 +++- .../feature/network/incoming/http_filter.rs | 2 +- .../config/src/feature/network/outgoing.rs | 2 +- mirrord/config/src/internal_proxy.rs | 4 +- mirrord/config/src/lib.rs | 136 +++++++++++++++++- 25 files changed, 210 insertions(+), 49 deletions(-) create mode 100644 changelog.d/2936.fixed.md diff --git a/Cargo.lock b/Cargo.lock index 941d424a4c7..fd5a54ee1e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4262,6 +4262,7 @@ dependencies = [ name = "mirrord-config" version = "3.129.0" dependencies = [ + "base64 0.22.1", "bimap", "bitflags 2.6.0", "fancy-regex", diff --git a/changelog.d/2936.fixed.md b/changelog.d/2936.fixed.md new file mode 100644 index 00000000000..e4dfd4792a6 --- /dev/null +++ b/changelog.d/2936.fixed.md @@ -0,0 +1 @@ +Stopped mirrord entering a crash loop when trying to load into some processes like VSCode's `watchdog.js` when the user config contained a call to `get_env()`, which occurred due to missing env - the config is now only rendered once and set into an env var. \ No newline at end of file diff --git a/mirrord/cli/src/container.rs b/mirrord/cli/src/container.rs index aa396a0162f..a2659190e0d 100644 --- a/mirrord/cli/src/container.rs +++ b/mirrord/cli/src/container.rs @@ -333,6 +333,7 @@ pub(crate) async fn container_command( (CONTAINER_EXECUTION_KIND as u32).to_string(), ); + // LayerConfig must be created after setting relevant env vars let (mut config, mut analytics) = create_config_and_analytics(&mut progress, watch)?; let (_internal_proxy_tls_guards, _external_proxy_tls_guards) = @@ -427,6 +428,7 @@ pub(crate) async fn container_ext_command( env.insert("MIRRORD_IMPERSONATED_TARGET".into(), target.to_string()); } + // LayerConfig must be created after setting relevant env vars let (mut config, mut analytics) = create_config_and_analytics(&mut progress, watch)?; let (_internal_proxy_tls_guards, _external_proxy_tls_guards) = diff --git a/mirrord/cli/src/execution.rs b/mirrord/cli/src/execution.rs index 7629b466b20..5378f83d5b6 100644 --- a/mirrord/cli/src/execution.rs +++ b/mirrord/cli/src/execution.rs @@ -177,7 +177,7 @@ impl MirrordExecution { /// [`tokio::time::sleep`] or [`tokio::task::yield_now`] after calling this function. #[tracing::instrument(level = Level::TRACE, skip_all)] pub(crate) async fn start

( - config: &LayerConfig, + config: &mut LayerConfig, // We only need the executable on macos, for SIP handling. #[cfg(target_os = "macos")] executable: Option<&str>, progress: &mut P, @@ -289,10 +289,8 @@ impl MirrordExecution { })?; // Provide details for layer to connect to agent via internal proxy - env_vars.insert( - MIRRORD_CONNECT_TCP_ENV.to_string(), - format!("127.0.0.1:{}", address.port()), - ); + config.connect_tcp = Some(format!("127.0.0.1:{}", address.port())); + config.update_env_var()?; // Fixes // by disabling the fork safety check in the Objective-C runtime. diff --git a/mirrord/cli/src/extension.rs b/mirrord/cli/src/extension.rs index f8d4a0da482..005491a9aed 100644 --- a/mirrord/cli/src/extension.rs +++ b/mirrord/cli/src/extension.rs @@ -10,7 +10,7 @@ use crate::{config::ExtensionExecArgs, error::CliError, execution::MirrordExecut async fn mirrord_exec

( #[cfg(target_os = "macos")] executable: Option<&str>, env: HashMap, - config: LayerConfig, + mut config: LayerConfig, mut progress: P, analytics: &mut AnalyticsReporter, ) -> CliResult<()> @@ -21,9 +21,9 @@ where // or run tasks before actually launching. #[cfg(target_os = "macos")] let mut execution_info = - MirrordExecution::start(&config, executable, &mut progress, analytics).await?; + MirrordExecution::start(&mut config, executable, &mut progress, analytics).await?; #[cfg(not(target_os = "macos"))] - let mut execution_info = MirrordExecution::start(&config, &mut progress, analytics).await?; + let mut execution_info = MirrordExecution::start(&mut config, &mut progress, analytics).await?; // We don't execute so set envs aren't passed, so we need to add config file and target to // env. diff --git a/mirrord/cli/src/external_proxy.rs b/mirrord/cli/src/external_proxy.rs index a6ab4bb8674..34bf46b82b8 100644 --- a/mirrord/cli/src/external_proxy.rs +++ b/mirrord/cli/src/external_proxy.rs @@ -60,7 +60,7 @@ fn print_addr(listener: &TcpListener) -> io::Result<()> { } pub async fn proxy(listen_port: u16, watch: drain::Watch) -> CliResult<()> { - let config = LayerConfig::from_env()?; + let config = LayerConfig::recalculate_from_env()?; init_extproxy_tracing_registry(&config)?; tracing::info!(?config, "external_proxy starting"); diff --git a/mirrord/cli/src/internal_proxy.rs b/mirrord/cli/src/internal_proxy.rs index b1b9a20b72a..e8210962e84 100644 --- a/mirrord/cli/src/internal_proxy.rs +++ b/mirrord/cli/src/internal_proxy.rs @@ -50,7 +50,7 @@ pub(crate) async fn proxy( listen_port: u16, watch: drain::Watch, ) -> CliResult<(), InternalProxyError> { - let config = LayerConfig::from_env()?; + let config = LayerConfig::recalculate_from_env()?; init_intproxy_tracing_registry(&config)?; tracing::info!(?config, "internal_proxy starting"); diff --git a/mirrord/cli/src/main.rs b/mirrord/cli/src/main.rs index 4631af499dc..2ca498c83fc 100644 --- a/mirrord/cli/src/main.rs +++ b/mirrord/cli/src/main.rs @@ -64,7 +64,7 @@ pub(crate) use error::{CliError, CliResult}; use verify_config::verify_config; async fn exec_process

( - config: LayerConfig, + mut config: LayerConfig, args: &ExecArgs, progress: &P, analytics: &mut AnalyticsReporter, @@ -75,10 +75,15 @@ where let mut sub_progress = progress.subtask("preparing to launch process"); #[cfg(target_os = "macos")] - let execution_info = - MirrordExecution::start(&config, Some(&args.binary), &mut sub_progress, analytics).await?; + let execution_info = MirrordExecution::start( + &mut config, + Some(&args.binary), + &mut sub_progress, + analytics, + ) + .await?; #[cfg(not(target_os = "macos"))] - let execution_info = MirrordExecution::start(&config, &mut sub_progress, analytics).await?; + let execution_info = MirrordExecution::start(&mut config, &mut sub_progress, analytics).await?; // This is not being yielded, as this is not proper async, something along those lines. // We need an `await` somewhere in this function to drive our socket IO that happens @@ -352,6 +357,7 @@ async fn exec(args: &ExecArgs, watch: drain::Watch) -> CliResult<()> { std::env::set_var(name, value); } + // LayerConfig must be created after setting relevant env vars let (config, mut context) = LayerConfig::from_env_with_warnings()?; let mut analytics = AnalyticsReporter::only_error(config.telemetry, Default::default(), watch); @@ -472,6 +478,7 @@ async fn port_forward(args: &PortForwardArgs, watch: drain::Watch) -> CliResult< std::env::set_var("MIRRORD_CONFIG_FILE", config_file); } + // LayerConfig must be created after setting relevant env vars let (config, mut context) = LayerConfig::from_env_with_warnings()?; let mut analytics = AnalyticsReporter::new(config.telemetry, ExecutionKind::PortForward, watch); diff --git a/mirrord/config/Cargo.toml b/mirrord/config/Cargo.toml index e17985aa0cd..a2fd265c274 100644 --- a/mirrord/config/Cargo.toml +++ b/mirrord/config/Cargo.toml @@ -17,8 +17,8 @@ edition.workspace = true workspace = true [dependencies] -mirrord-config-derive = { path = "./derive"} -mirrord-analytics = { path = "../analytics"} +mirrord-config-derive = { path = "./derive" } +mirrord-analytics = { path = "../analytics" } serde.workspace = true serde_json.workspace = true @@ -27,13 +27,14 @@ tracing.workspace = true serde_yaml.workspace = true toml = "0.8" schemars.workspace = true -bimap = "0.6" +bimap = { version = "0.6" } nom = "7.1" ipnet.workspace = true bitflags = "2" k8s-openapi = { workspace = true, features = ["schemars", "earliest"] } tera = "1" fancy-regex.workspace = true +base64.workspace = true [dev-dependencies] rstest.workspace = true diff --git a/mirrord/config/src/agent.rs b/mirrord/config/src/agent.rs index b82c45a7177..3dff5adfefc 100644 --- a/mirrord/config/src/agent.rs +++ b/mirrord/config/src/agent.rs @@ -67,7 +67,7 @@ impl fmt::Display for LinuxCapability { /// } /// } /// ``` -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "AgentFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq"))] pub struct AgentConfig { @@ -354,7 +354,7 @@ pub struct AgentConfig { /// Create an agent that returns an error after accepting the first client. For testing /// purposes. Only supported with job agents (not with ephemeral agents). #[cfg(all(debug_assertions, not(test)))] // not(test) so that it's not included in the schema json. - #[serde(skip_serializing)] + #[serde(skip)] #[config(env = "MIRRORD_AGENT_TEST_ERROR", default = false, unstable)] pub test_error: bool, } @@ -477,7 +477,7 @@ impl AgentFileConfig { } } -#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] #[config(derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct AgentDnsConfig { diff --git a/mirrord/config/src/config.rs b/mirrord/config/src/config.rs index c0176c6f66b..9fa4bdd168a 100644 --- a/mirrord/config/src/config.rs +++ b/mirrord/config/src/config.rs @@ -86,6 +86,12 @@ pub enum ConfigError { value: String, fail: Box, }, + + #[error("mirrord-config: decoding resolved config from env var failed with `{0}`")] + EnvVarDecodeError(String), + + #[error("mirrord-config: encoding resolved config failed with `{0}`")] + EnvVarEncodeError(String), } impl From for ConfigError { diff --git a/mirrord/config/src/container.rs b/mirrord/config/src/container.rs index 5f9e16f7447..c48fec4a34d 100644 --- a/mirrord/config/src/container.rs +++ b/mirrord/config/src/container.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::config::source::MirrordConfigSource; @@ -12,7 +12,7 @@ static DEFAULT_CLI_IMAGE: &str = concat!( ); /// Unstable: `mirrord container` command specific config. -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "ContainerFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq"))] pub struct ContainerConfig { diff --git a/mirrord/config/src/experimental.rs b/mirrord/config/src/experimental.rs index 302cb14fc27..29af8e2f145 100644 --- a/mirrord/config/src/experimental.rs +++ b/mirrord/config/src/experimental.rs @@ -1,13 +1,13 @@ use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::config::source::MirrordConfigSource; /// mirrord Experimental features. /// This shouldn't be used unless someone from MetalBear/mirrord tells you to. -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "ExperimentalFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct ExperimentalConfig { diff --git a/mirrord/config/src/external_proxy.rs b/mirrord/config/src/external_proxy.rs index 2ce284b6b7b..0f1c80f5e0a 100644 --- a/mirrord/config/src/external_proxy.rs +++ b/mirrord/config/src/external_proxy.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::config::source::MirrordConfigSource; @@ -23,7 +23,7 @@ pub static MIRRORD_EXTERNAL_TLS_KEY_ENV: &str = "MIRRORD_EXTERNAL_TLS_KEY"; /// } /// } /// ``` -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "ExternalProxyFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq"))] pub struct ExternalProxyConfig { diff --git a/mirrord/config/src/feature.rs b/mirrord/config/src/feature.rs index 317d590e0c8..41fecca7002 100644 --- a/mirrord/config/src/feature.rs +++ b/mirrord/config/src/feature.rs @@ -1,7 +1,7 @@ use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use self::{copy_target::CopyTargetConfig, env::EnvConfig, fs::FsConfig, network::NetworkConfig}; use crate::{config::source::MirrordConfigSource, feature::split_queues::SplitQueuesConfig}; @@ -64,7 +64,7 @@ pub mod split_queues; /// } /// } /// ``` -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "FeatureFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct FeatureConfig { diff --git a/mirrord/config/src/feature/copy_target.rs b/mirrord/config/src/feature/copy_target.rs index c5ed004ebb4..8a72326d52a 100644 --- a/mirrord/config/src/feature/copy_target.rs +++ b/mirrord/config/src/feature/copy_target.rs @@ -75,7 +75,7 @@ impl FromMirrordConfig for CopyTargetConfig { /// } /// } /// ``` -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct CopyTargetConfig { pub enabled: bool, diff --git a/mirrord/config/src/feature/env.rs b/mirrord/config/src/feature/env.rs index 339b0c16e78..700b31f0667 100644 --- a/mirrord/config/src/feature/env.rs +++ b/mirrord/config/src/feature/env.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, path::PathBuf}; use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::{ config::{from_env::FromEnv, source::MirrordConfigSource, ConfigContext, Result}, @@ -47,7 +47,7 @@ pub const MIRRORD_OVERRIDE_ENV_FILE_ENV: &str = "MIRRORD_OVERRIDE_ENV_VARS_FILE" /// } /// } /// ``` -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "EnvFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct EnvConfig { diff --git a/mirrord/config/src/feature/fs/advanced.rs b/mirrord/config/src/feature/fs/advanced.rs index da1df66c00f..40c0e21c926 100644 --- a/mirrord/config/src/feature/fs/advanced.rs +++ b/mirrord/config/src/feature/fs/advanced.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use mirrord_analytics::{AnalyticValue, CollectAnalytics}; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use super::{FsModeConfig, FsUserConfig}; use crate::{ @@ -80,7 +80,7 @@ use crate::{ /// } /// } /// ``` -#[derive(MirrordConfig, Default, Clone, PartialEq, Eq, Debug, Serialize)] +#[derive(MirrordConfig, Default, Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] #[config( map_to = "AdvancedFsUserConfig", derive = "PartialEq,Eq,JsonSchema", diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index 227bd82a915..1d86071e32e 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -2,7 +2,7 @@ use dns::{DnsConfig, DnsFileConfig}; use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use self::{incoming::*, outgoing::*}; use crate::{ @@ -54,7 +54,7 @@ pub mod outgoing; /// } /// } /// ``` -#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] #[config(map_to = "NetworkFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct NetworkConfig { diff --git a/mirrord/config/src/feature/network/dns.rs b/mirrord/config/src/feature/network/dns.rs index fd0df6411b6..9371a2ee1e8 100644 --- a/mirrord/config/src/feature/network/dns.rs +++ b/mirrord/config/src/feature/network/dns.rs @@ -87,7 +87,7 @@ pub enum DnsFilterConfig { /// `read_only: ["/etc/resolv.conf"]`. /// - DNS filter currently works only with frameworks that use `getaddrinfo`/`gethostbyname` /// functions. -#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] #[config(map_to = "DnsFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct DnsConfig { diff --git a/mirrord/config/src/feature/network/incoming.rs b/mirrord/config/src/feature/network/incoming.rs index fddd46260d5..857fb0179d3 100644 --- a/mirrord/config/src/feature/network/incoming.rs +++ b/mirrord/config/src/feature/network/incoming.rs @@ -310,6 +310,7 @@ fn serialize_bi_map(map: &BiMap, serializer: S) -> Result(deserializer: D) -> Result, D::Error> +where + D: de::Deserializer<'de>, +{ + // NB: this deserialises the BiMap from a vec + let vec: Vec<(u16, u16)> = Vec::deserialize(deserializer)?; + + let mut elements = BiMap::new(); + vec.iter().for_each(|(key, value)| { + elements.insert(*key, *value); + }); + Ok(elements) +} + /// Controls the incoming TCP traffic feature. /// /// See the incoming [reference](https://mirrord.dev/docs/reference/traffic/#incoming) for more @@ -387,7 +402,7 @@ where /// } /// } /// ``` -#[derive(Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] pub struct IncomingConfig { /// #### feature.network.incoming.port_mapping {#feature-network-incoming-port_mapping} /// @@ -396,7 +411,10 @@ pub struct IncomingConfig { /// This is useful when you want to mirror/steal a port to a different port on the remote /// machine. For example, your local process listens on port `9333` and the container listens /// on port `80`. You'd use `[[9333, 80]]` - #[serde(serialize_with = "serialize_bi_map")] + #[serde( + serialize_with = "serialize_bi_map", + deserialize_with = "deserialize_bi_map" + )] pub port_mapping: BiMap, /// #### feature.network.incoming.ignore_localhost {#feature-network-incoming-ignore_localhost} @@ -434,7 +452,10 @@ pub struct IncomingConfig { /// you probably can't listen on `80` without sudo, so you can use `[[80, 4480]]` /// then access it on `4480` while getting traffic from remote `80`. /// The value of `port_mapping` doesn't affect this. - #[serde(serialize_with = "serialize_bi_map")] + #[serde( + serialize_with = "serialize_bi_map", + deserialize_with = "deserialize_bi_map" + )] pub listen_ports: BiMap, /// #### feature.network.incoming.on_concurrent_steal {#feature-network-incoming-on_concurrent_steal} diff --git a/mirrord/config/src/feature/network/incoming/http_filter.rs b/mirrord/config/src/feature/network/incoming/http_filter.rs index 343e850b75d..c68e288fa58 100644 --- a/mirrord/config/src/feature/network/incoming/http_filter.rs +++ b/mirrord/config/src/feature/network/incoming/http_filter.rs @@ -79,7 +79,7 @@ use crate::{ /// { "header": "^x-debug-session: 121212$" } /// ] ///} -#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] #[config(map_to = "HttpFilterFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct HttpFilterConfig { diff --git a/mirrord/config/src/feature/network/outgoing.rs b/mirrord/config/src/feature/network/outgoing.rs index 8ff2a84ce21..c0878a412a9 100644 --- a/mirrord/config/src/feature/network/outgoing.rs +++ b/mirrord/config/src/feature/network/outgoing.rs @@ -89,7 +89,7 @@ pub enum OutgoingFilterConfig { /// } /// } /// ``` -#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Default, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] #[config(map_to = "OutgoingFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq, Eq"))] pub struct OutgoingConfig { diff --git a/mirrord/config/src/internal_proxy.rs b/mirrord/config/src/internal_proxy.rs index dd28f2f4d9e..ba5b9483479 100644 --- a/mirrord/config/src/internal_proxy.rs +++ b/mirrord/config/src/internal_proxy.rs @@ -2,7 +2,7 @@ use std::{net::SocketAddr, path::PathBuf}; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use crate::config::source::MirrordConfigSource; @@ -26,7 +26,7 @@ pub static MIRRORD_INTPROXY_CLIENT_TLS_KEY_ENV: &str = "MIRRORD_INTPROXY_CLIENT_ /// } /// } /// ``` -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "InternalProxyFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq"))] pub struct InternalProxyConfig { diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index 055adac98d2..8414bd742df 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -25,13 +25,14 @@ use std::{ path::Path, }; +use base64::prelude::*; use config::{ConfigContext, ConfigError, MirrordConfig}; use experimental::ExperimentalConfig; use feature::{env::mapper::EnvVarsRemapper, network::outgoing::OutgoingFilterConfig}; use mirrord_analytics::CollectAnalytics; use mirrord_config_derive::MirrordConfig; use schemars::JsonSchema; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use target::Target; use tera::Tera; use tracing::warn; @@ -45,6 +46,9 @@ use crate::{ /// Env variable to load config from file (json, yaml and toml supported). pub static MIRRORD_CONFIG_FILE_ENV: &str = "MIRRORD_CONFIG_FILE"; +/// Env variable to load config from an already resolved base64 encoding. +pub static MIRRORD_RESOLVED_CONFIG_ENV: &str = "MIRRORD_RESOLVED_CONFIG"; + /// mirrord allows for a high degree of customization when it comes to which features you want to /// enable, and how they should function. /// @@ -174,7 +178,7 @@ pub static MIRRORD_CONFIG_FILE_ENV: &str = "MIRRORD_CONFIG_FILE"; /// ``` /// /// # Options {#root-options} -#[derive(MirrordConfig, Clone, Debug, Serialize)] +#[derive(MirrordConfig, Clone, Debug, Serialize, Deserialize, PartialEq)] #[config(map_to = "LayerFileConfig", derive = "JsonSchema")] #[cfg_attr(test, config(derive = "PartialEq"))] pub struct LayerConfig { @@ -327,15 +331,55 @@ pub struct LayerConfig { } impl LayerConfig { + /// Given an encoded complete config from the [`MIRRORD_RESOLVED_CONFIG_ENV`] + /// env var, attempt to decode it into [`LayerConfig`]. + /// Intended to avoid re-resolving the config in every process mirrord is loaded into. + fn from_env_var(encoded_value: String) -> Result { + let decoded = BASE64_STANDARD + .decode(encoded_value) + .map_err(|error| ConfigError::EnvVarDecodeError(error.to_string()))?; + let serialized = std::str::from_utf8(&decoded) + .map_err(|error| ConfigError::EnvVarDecodeError(error.to_string()))?; + Ok(serde_json::from_str::(serialized)?) + } + + /// Given a [`LayerConfig`], serialise it and convert to base 64 so it can be + /// set into [`MIRRORD_RESOLVED_CONFIG_ENV`]. + fn to_env_var(&self) -> Result { + let serialized = serde_json::to_string(self) + .map_err(|error| ConfigError::EnvVarEncodeError(error.to_string()))?; + Ok(BASE64_STANDARD.encode(serialized)) + } + + /// Given the encoded config as a string, set it into [`MIRRORD_RESOLVED_CONFIG_ENV`]. + /// Must be used when updating [`LayerConfig`] after creation in order for the config + /// in env to reflect the change. + pub fn update_env_var(&self) -> Result<(), ConfigError> { + std::env::set_var(MIRRORD_RESOLVED_CONFIG_ENV, self.to_env_var()?); + Ok(()) + } + /// Generate a config from the environment variables and/or a config file. /// On success, returns the config and a vec of warnings. /// To be used from CLI to verify config and print warnings pub fn from_env_with_warnings() -> Result<(Self, ConfigContext), ConfigError> { let mut cfg_context = ConfigContext::default(); - if let Ok(path) = std::env::var(MIRRORD_CONFIG_FILE_ENV) { - LayerFileConfig::from_path(path)?.generate_config(&mut cfg_context) - } else { - LayerFileConfig::default().generate_config(&mut cfg_context) + + match std::env::var(MIRRORD_RESOLVED_CONFIG_ENV) { + Ok(value) if !value.is_empty() => LayerConfig::from_env_var(value), + _ => { + // the resolved config is not present in env, so resolve it and then set into env + // var + let config = if let Ok(path) = std::env::var(MIRRORD_CONFIG_FILE_ENV) { + LayerFileConfig::from_path(path)?.generate_config(&mut cfg_context) + } else { + LayerFileConfig::default().generate_config(&mut cfg_context) + }?; + + // serialise the config and encode as base64 + config.update_env_var()?; + Ok(config) + } } .map(|config| (config, cfg_context)) } @@ -347,6 +391,17 @@ impl LayerConfig { Self::from_env_with_warnings().map(|(config, _)| config) } + /// forcefully recalculate the config using [`Self::from_env_with_warnings()`] + pub fn recalculate_from_env_with_warnings() -> Result<(Self, ConfigContext), ConfigError> { + std::env::remove_var(MIRRORD_RESOLVED_CONFIG_ENV); + Self::from_env_with_warnings() + } + + /// forcefully recalculate the config using [`Self::from_env_with_warnings()`] without warnings + pub fn recalculate_from_env() -> Result { + Self::recalculate_from_env_with_warnings().map(|(config, _)| config) + } + /// Verify that there are no conflicting settings. /// /// We don't call it from `from_env` since we want to verify it only once (from cli) @@ -986,4 +1041,73 @@ mod tests { assert_eq!(existing_content.replace("\r\n", "\n"), compare_content); } + + /// related to issue #2936: https://github.com/metalbear-co/mirrord/issues/2936 + /// checks that resolved config written to [`MIRRORD_RESOLVED_CONFIG_ENV`] can be + /// transformed back into a [`LayerConfig`] + #[test] + fn encode_and_decode_default_config() { + let mut cfg_context = ConfigContext::default(); + let resolved_config = LayerFileConfig::default() + .generate_config(&mut cfg_context) + .expect("Default config should be generated from default 'LayerFileConfig'"); + + let encoded = resolved_config.to_env_var().unwrap(); + let decoded = LayerConfig::from_env_var(encoded).unwrap(); + + assert_eq!(decoded, resolved_config); + } + + #[test] + fn encode_and_decode_advanced_config() { + let mut cfg_context = ConfigContext::default(); + + // this config includes template variables, so it needs to be rendered first + let mut template_engine = Tera::default(); + template_engine + .add_raw_template("main", get_advanced_config().as_str()) + .unwrap(); + let rendered = template_engine + .render("main", &tera::Context::new()) + .expect("Tera should render JSON config file contents"); + let resolved_config = ConfigType::Json + .parse(rendered.as_str()) + .generate_config(&mut cfg_context) + .expect("Layer config should be generated from JSON config file contents"); + + let encoded = resolved_config.to_env_var().unwrap(); + let decoded = LayerConfig::from_env_var(encoded).unwrap(); + + assert_eq!(decoded, resolved_config); + } + + fn get_advanced_config() -> String { + r#" + { + "accept_invalid_certificates": false, + "target": { + "path": "pod/test-service-abcdefg-abcd", + "namespace": "default" + }, + "feature": { + "env": true, + "fs": "write", + "network": { + "dns": false, + "incoming": { + "mode": "steal", + "http_filter": { + "header_filter": "x-intercept: {{ get_env(name="USER") }}" + } + }, + "outgoing": { + "tcp": true, + "udp": false + } + } + } + } + "# + .to_string() + } } From db8288180c08300a170fd8f313bc43e3319435a9 Mon Sep 17 00:00:00 2001 From: meowjesty <43983236+meowjesty@users.noreply.github.com> Date: Mon, 20 Jan 2025 17:59:34 -0300 Subject: [PATCH 13/23] Update configuration.md and improve config env.mapping list. (#3020) * Update configuration.md and improve config env.mapping list. * schema * try new line * changelog --- changelog.d/+improve-env-config-docs.changed.md | 1 + mirrord-schema.json | 2 +- mirrord/config/configuration.md | 8 +++++--- mirrord/config/src/feature/env.rs | 8 +++++--- 4 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 changelog.d/+improve-env-config-docs.changed.md diff --git a/changelog.d/+improve-env-config-docs.changed.md b/changelog.d/+improve-env-config-docs.changed.md new file mode 100644 index 00000000000..79e19a7ced7 --- /dev/null +++ b/changelog.d/+improve-env-config-docs.changed.md @@ -0,0 +1 @@ +Update configuration.md and improve config env.mapping list. diff --git a/mirrord-schema.json b/mirrord-schema.json index 46306a4ed28..5beadff3c58 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -752,7 +752,7 @@ }, "mapping": { "title": "feature.env.mapping {#feature-env-mapping}", - "description": "Specify map of patterns that if matched will replace the value according to specification.\n\n*Capture groups are allowed.*\n\nExample: ```json { \".+_TIMEOUT\": \"10000\" \"LOG_.+_VERBOSITY\": \"debug\" \"(\\w+)_(\\d+)\": \"magic-value\" } ```\n\nWill do the next replacements for environment variables that match:\n\n`CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000` `LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug` `DATA_1234: common-value` => `DATA_1234: magic-value`", + "description": "Specify map of patterns that if matched will replace the value according to specification.\n\n*Capture groups are allowed.*\n\nExample: ```json { \".+_TIMEOUT\": \"10000\" \"LOG_.+_VERBOSITY\": \"debug\" \"(\\w+)_(\\d+)\": \"magic-value\" } ```\n\nWill do the next replacements for environment variables that match:\n\n* `CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000`\n\n* `LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug`\n\n* `DATA_1234: common-value` => `DATA_1234: magic-value`", "type": [ "object", "null" diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md index 3a3f8b4fa57..8e8b9ea6aee 100644 --- a/mirrord/config/configuration.md +++ b/mirrord/config/configuration.md @@ -729,9 +729,11 @@ Example: Will do the next replacements for environment variables that match: -`CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000` -`LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug` -`DATA_1234: common-value` => `DATA_1234: magic-value` +* `CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000` + +* `LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug` + +* `DATA_1234: common-value` => `DATA_1234: magic-value` ### feature.env.override {#feature-env-override} diff --git a/mirrord/config/src/feature/env.rs b/mirrord/config/src/feature/env.rs index 700b31f0667..1763a960b22 100644 --- a/mirrord/config/src/feature/env.rs +++ b/mirrord/config/src/feature/env.rs @@ -134,9 +134,11 @@ pub struct EnvConfig { /// /// Will do the next replacements for environment variables that match: /// - /// `CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000` - /// `LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug` - /// `DATA_1234: common-value` => `DATA_1234: magic-value` + /// * `CONNECTION_TIMEOUT: 500` => `CONNECTION_TIMEOUT: 10000` + /// + /// * `LOG_FILE_VERBOSITY: info` => `LOG_FILE_VERBOSITY: debug` + /// + /// * `DATA_1234: common-value` => `DATA_1234: magic-value` pub mapping: Option>, } From 95c79bdb2edb8ca4dc39d4af152442d91d328ed7 Mon Sep 17 00:00:00 2001 From: Aviram Hassan Date: Tue, 21 Jan 2025 15:35:44 +0200 Subject: [PATCH 14/23] use older builder base image for aarch64 to support centos-7 libc (#3025) * use older builder base image for aarch64 to support centos-7 libc * .. --- Cross.toml | 5 ++--- changelog.d/3024.fixed.md | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog.d/3024.fixed.md diff --git a/Cross.toml b/Cross.toml index 8339c1d1fd2..dfca795a2e4 100644 --- a/Cross.toml +++ b/Cross.toml @@ -5,8 +5,7 @@ passthrough = [ # Dockerfile used for building mirrord-layer for x64 with very old libc # this to support centos7 or Amazon Linux 2. [target.x86_64-unknown-linux-gnu] -image = "ghcr.io/metalbear-co/ci-layer-build:latest" +image = "ghcr.io/metalbear-co/ci-layer-build:8ca4a4e9757a5749c384c57c3435e56c2b442458e14e4cb7ecbaec4d557f5d69" -# 0.2.5 doesn't work (lacks clang?) [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" \ No newline at end of file +image = "ghcr.io/metalbear-co/ci-layer-build-aarch64:4c99d799d297ddec935914907b94d899bd0a2349155cec934492ef19a69ddbf0" \ No newline at end of file diff --git a/changelog.d/3024.fixed.md b/changelog.d/3024.fixed.md new file mode 100644 index 00000000000..c584f5269ed --- /dev/null +++ b/changelog.d/3024.fixed.md @@ -0,0 +1 @@ +use older builder base image for aarch64 to support centos-7 libc From 78f9570777012f5acd80d2163e472275f329bb17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:19:54 +0100 Subject: [PATCH 15/23] Added 'info' log level (#3026) --- Cargo.lock | 2 +- changelog.d/+added-log-leve.internal.md | 1 + mirrord/cli/src/execution.rs | 3 ++- mirrord/cli/src/port_forward.rs | 2 ++ mirrord/intproxy/src/lib.rs | 1 + mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/codec.rs | 6 ++++++ mirrord/vpn/src/agent.rs | 3 +++ 8 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 changelog.d/+added-log-leve.internal.md diff --git a/Cargo.lock b/Cargo.lock index fd5a54ee1e7..548a59c3242 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4480,7 +4480,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.13.3" +version = "1.13.4" dependencies = [ "actix-codec", "bincode", diff --git a/changelog.d/+added-log-leve.internal.md b/changelog.d/+added-log-leve.internal.md new file mode 100644 index 00000000000..604957c5641 --- /dev/null +++ b/changelog.d/+added-log-leve.internal.md @@ -0,0 +1 @@ +Extended `mirrord-protocol` with info logs from the agent. \ No newline at end of file diff --git a/mirrord/cli/src/execution.rs b/mirrord/cli/src/execution.rs index 5378f83d5b6..5d3e9657e88 100644 --- a/mirrord/cli/src/execution.rs +++ b/mirrord/cli/src/execution.rs @@ -27,7 +27,7 @@ use tokio::{ sync::mpsc::{self, UnboundedReceiver}, }; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, trace, warn, Level}; +use tracing::{debug, error, info, trace, warn, Level}; #[cfg(all(target_os = "macos", target_arch = "aarch64"))] use crate::extract::extract_arm64; @@ -550,6 +550,7 @@ impl MirrordExecution { match msg.level { LogLevel::Error => error!("Agent log: {}", msg.message), LogLevel::Warn => warn!("Agent log: {}", msg.message), + LogLevel::Info => info!("Agent log: {}", msg.message), } continue; diff --git a/mirrord/cli/src/port_forward.rs b/mirrord/cli/src/port_forward.rs index 002a904ae1b..0f701eb9890 100644 --- a/mirrord/cli/src/port_forward.rs +++ b/mirrord/cli/src/port_forward.rs @@ -315,6 +315,7 @@ impl PortForwarder { DaemonMessage::LogMessage(log_message) => match log_message.level { LogLevel::Warn => tracing::warn!("agent log: {}", log_message.message), LogLevel::Error => tracing::error!("agent log: {}", log_message.message), + LogLevel::Info => tracing::info!("agent log: {}", log_message.message), }, DaemonMessage::Close(error) => { return Err(PortForwardError::AgentError(error)); @@ -556,6 +557,7 @@ impl ReversePortForwarder { DaemonMessage::LogMessage(log_message) => match log_message.level { LogLevel::Warn => tracing::warn!("agent log: {}", log_message.message), LogLevel::Error => tracing::error!("agent log: {}", log_message.message), + LogLevel::Info => tracing::info!("agent log: {}", log_message.message), }, DaemonMessage::Close(error) => { return Err(PortForwardError::AgentError(error)); diff --git a/mirrord/intproxy/src/lib.rs b/mirrord/intproxy/src/lib.rs index 7dd396c9eb0..7dee93344bf 100644 --- a/mirrord/intproxy/src/lib.rs +++ b/mirrord/intproxy/src/lib.rs @@ -329,6 +329,7 @@ impl IntProxy { DaemonMessage::LogMessage(log) => match log.level { LogLevel::Error => tracing::error!("agent log: {}", log.message), LogLevel::Warn => tracing::warn!("agent log: {}", log.message), + LogLevel::Info => tracing::info!("agent log: {}", log.message), }, DaemonMessage::GetEnvVarsResponse(res) => { self.task_txs diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 7d787e1e5c6..e6c9980c15b 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.13.3" +version = "1.13.4" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/codec.rs b/mirrord/protocol/src/codec.rs index 4071018fe6a..81e79bf5db7 100644 --- a/mirrord/protocol/src/codec.rs +++ b/mirrord/protocol/src/codec.rs @@ -24,10 +24,16 @@ use crate::{ ResponseError, }; +/// Minimal mirrord-protocol version that that allows [`LogLevel::Info`]. +pub static INFO_LOG_VERSION: LazyLock = + LazyLock::new(|| ">=1.13.4".parse().expect("Bad Identifier")); + #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone, Copy)] pub enum LogLevel { Warn, Error, + /// Supported from [`INFO_LOG_VERSION`]. + Info, } #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] diff --git a/mirrord/vpn/src/agent.rs b/mirrord/vpn/src/agent.rs index 5b7c6cb5f9e..3f145c421a0 100644 --- a/mirrord/vpn/src/agent.rs +++ b/mirrord/vpn/src/agent.rs @@ -136,6 +136,9 @@ impl Stream for VpnAgent { LogLevel::Warn => { tracing::warn!(message = %message.message, "agent sent warn message") } + LogLevel::Info => { + tracing::info!(message = %message.message, "agent sent info message") + } } self.poll_next(cx) From 2ee5a2c5357b6cf4496ca30af91834435aea37d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:19:22 +0100 Subject: [PATCH 16/23] Fix hanging requests with filtered steal (#3016) * Integration test * Moved MetadataStore to a separate module * bind_similar -> BoundTcpSocket * BodyExt -> BatchedBody, reworked trait * local HTTP handling rework * Remove obsolete stuff from mirrord-protocol * Moved HttpResponseFallback to the agent * Moved frame senders to InterceptorHandle * HttpResponseReaders in IncomingProxy * Clippy * Better tracing * HttpResponseReader logic split into methods * unless_bus_closed * I hate this * HttpGateway tests * ClientStore test * Changed implementation of ClientStore shared state * Some docs * Docs * Removed obsolete integration test - replaced before with a unit test * More ClientStore tracing * Less spammy debug for InProxyTaskMessage * Clippy and docs * More tracing * Clippy * Fixed TcpProxyTask * More IncomingProxy docs * macos tests fixed * Fixed reverseportforwarder and its tests * Upgrade fixed * Extended changelog * Frames doc * Helper function for BatchedBody * auto_responder -> unwrap instead of is_err + break * ClientStore unwrap -> expect * Comments for ClientStore cleanup_task * Closed doc * TcpStealApi::response_body_tx doc * Removed expect from client_store::cleanup_task * Doc lint * Update mirrord/intproxy/src/background_tasks.rs Co-authored-by: meowjesty <43983236+meowjesty@users.noreply.github.com> * Update mirrord/intproxy/src/proxies/incoming.rs Co-authored-by: meowjesty <43983236+meowjesty@users.noreply.github.com> * error -> unreachable * pub(crate) for Closed * Update mirrord/intproxy/src/proxies/incoming.rs Co-authored-by: meowjesty <43983236+meowjesty@users.noreply.github.com> * not war * More doccc * self_address -> self * rephrased error messages * instrument on LocalHttpClient::new * More docs on clone for StreamingBody * docc * docsss * Doc fixed * Doc fixed * more instrument * in whole -> without a filter * moar doccc * TPC -> TCP * added ignore to doctest --------- Co-authored-by: meowjesty <43983236+meowjesty@users.noreply.github.com> --- Cargo.lock | 6 +- changelog.d/3013.fixed.md | 2 + mirrord/agent/src/steal.rs | 4 +- mirrord/agent/src/steal/api.rs | 23 +- mirrord/agent/src/steal/connection.rs | 54 +- mirrord/agent/src/steal/http.rs | 7 +- .../agent/src/steal/http/response_fallback.rs | 58 + mirrord/cli/src/port_forward.rs | 660 ++++------ mirrord/intproxy/Cargo.toml | 4 - mirrord/intproxy/src/background_tasks.rs | 66 +- mirrord/intproxy/src/proxies/incoming.rs | 1101 +++++++++-------- .../src/proxies/incoming/bound_socket.rs | 46 + mirrord/intproxy/src/proxies/incoming/http.rs | 263 +++- .../src/proxies/incoming/http/client_store.rs | 232 ++++ .../proxies/incoming/http/response_mode.rs | 31 + .../proxies/incoming/http/streaming_body.rs | 140 +++ .../src/proxies/incoming/http_gateway.rs | 943 ++++++++++++++ .../src/proxies/incoming/interceptor.rs | 909 -------------- .../src/proxies/incoming/metadata_store.rs | 48 + .../proxies/incoming/port_subscription_ext.rs | 15 +- .../intproxy/src/proxies/incoming/tasks.rs | 113 ++ .../src/proxies/incoming/tcp_proxy.rs | 198 +++ mirrord/layer/tests/common/mod.rs | 32 +- mirrord/layer/tests/fileops.rs | 24 +- mirrord/layer/tests/http_mirroring.rs | 2 + mirrord/protocol/Cargo.toml | 3 +- mirrord/protocol/src/batched_body.rs | 86 ++ mirrord/protocol/src/body_chunks.rs | 71 -- mirrord/protocol/src/lib.rs | 2 +- mirrord/protocol/src/tcp.rs | 709 ++--------- tests/src/traffic/steal.rs | 28 +- 31 files changed, 3185 insertions(+), 2695 deletions(-) create mode 100644 changelog.d/3013.fixed.md create mode 100644 mirrord/agent/src/steal/http/response_fallback.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/bound_socket.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/http/client_store.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/http/response_mode.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/http/streaming_body.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/http_gateway.rs delete mode 100644 mirrord/intproxy/src/proxies/incoming/interceptor.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/metadata_store.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/tasks.rs create mode 100644 mirrord/intproxy/src/proxies/incoming/tcp_proxy.rs create mode 100644 mirrord/protocol/src/batched_body.rs delete mode 100644 mirrord/protocol/src/body_chunks.rs diff --git a/Cargo.lock b/Cargo.lock index 548a59c3242..b6e72453953 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4314,8 +4314,6 @@ version = "3.129.0" dependencies = [ "bytes", "exponential-backoff", - "futures", - "h2 0.4.7", "http-body-util", "hyper 1.5.2", "hyper-util", @@ -4326,7 +4324,6 @@ dependencies = [ "mirrord-operator", "mirrord-protocol", "rand", - "reqwest 0.12.12", "rstest", "rustls 0.23.20", "rustls-pemfile 2.2.0", @@ -4486,6 +4483,7 @@ dependencies = [ "bincode", "bytes", "fancy-regex", + "futures", "hickory-proto", "hickory-resolver", "http-body-util", @@ -4498,8 +4496,6 @@ dependencies = [ "serde", "socket2", "thiserror 2.0.9", - "tokio", - "tokio-stream", "tracing", ] diff --git a/changelog.d/3013.fixed.md b/changelog.d/3013.fixed.md new file mode 100644 index 00000000000..811ab816b8c --- /dev/null +++ b/changelog.d/3013.fixed.md @@ -0,0 +1,2 @@ +Fixed an issue where HTTP requests stolen with a filter would hang with a single-threaded local HTTP server. +Improved handling of incoming connections on the local machine (e.g introduces reuse of local HTTP connections). diff --git a/mirrord/agent/src/steal.rs b/mirrord/agent/src/steal.rs index 399c0597d4e..a425748a0d8 100644 --- a/mirrord/agent/src/steal.rs +++ b/mirrord/agent/src/steal.rs @@ -1,5 +1,5 @@ use mirrord_protocol::{ - tcp::{DaemonTcp, HttpResponseFallback, StealType, TcpData}, + tcp::{DaemonTcp, StealType, TcpData}, ConnectionId, Port, }; use tokio::sync::mpsc::Sender; @@ -17,6 +17,8 @@ mod subscriptions; pub(crate) use api::TcpStealerApi; pub(crate) use connection::TcpConnectionStealer; +use self::http::HttpResponseFallback; + /// Commands from the agent that are passed down to the stealer worker, through [`TcpStealerApi`]. /// /// These are the operations that the agent receives from the layer to make the _steal_ feature diff --git a/mirrord/agent/src/steal/api.rs b/mirrord/agent/src/steal/api.rs index a6ec1d8d1f7..15d2f265ba7 100644 --- a/mirrord/agent/src/steal/api.rs +++ b/mirrord/agent/src/steal/api.rs @@ -1,24 +1,23 @@ -use std::collections::HashMap; +use std::{collections::HashMap, convert::Infallible}; use bytes::Bytes; use hyper::body::Frame; use mirrord_protocol::{ - tcp::{ - ChunkedResponse, DaemonTcp, HttpResponse, HttpResponseFallback, InternalHttpResponse, - LayerTcpSteal, ReceiverStreamBody, TcpData, - }, + tcp::{ChunkedResponse, DaemonTcp, HttpResponse, InternalHttpResponse, LayerTcpSteal, TcpData}, RequestId, }; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio_stream::wrappers::ReceiverStream; -use super::*; +use super::{http::ReceiverStreamBody, *}; use crate::{ error::{AgentError, Result}, util::ClientId, watched_task::TaskStatus, }; +type ResponseBodyTx = Sender, Infallible>>; + /// Bridges the communication between the agent and the [`TcpConnectionStealer`] task. /// There is an API instance for each connected layer ("client"). All API instances send commands /// On the same stealer command channel, where the layer-independent stealer listens to them. @@ -40,7 +39,15 @@ pub(crate) struct TcpStealerApi { /// View on the stealer task's status. task_status: TaskStatus, - response_body_txs: HashMap<(ConnectionId, RequestId), Sender>>>, + /// [`Sender`]s that allow us to provide body [`Frame`]s of responses to filtered HTTP + /// requests. + /// + /// With [`LayerTcpSteal::HttpResponseChunked`], response bodies come from the client + /// in a series of [`ChunkedResponse::Body`] messages. + /// + /// Thus, we use [`ReceiverStreamBody`] for [`Response`](hyper::Response)'s body type and + /// pipe the [`Frame`]s through an [`mpsc::channel`]. + response_body_txs: HashMap<(ConnectionId, RequestId), ResponseBodyTx>, } impl TcpStealerApi { @@ -196,7 +203,7 @@ impl TcpStealerApi { let key = (response.connection_id, response.request_id); self.response_body_txs.insert(key, tx.clone()); - self.http_response(HttpResponseFallback::Streamed(http_response, None)) + self.http_response(HttpResponseFallback::Streamed(http_response)) .await?; for frame in response.internal_response.body { diff --git a/mirrord/agent/src/steal/connection.rs b/mirrord/agent/src/steal/connection.rs index 37435176b8b..5e4b6b1219a 100644 --- a/mirrord/agent/src/steal/connection.rs +++ b/mirrord/agent/src/steal/connection.rs @@ -12,12 +12,11 @@ use hyper::{ http::{header::UPGRADE, request::Parts}, }; use mirrord_protocol::{ - body_chunks::{BodyExt as _, Frames}, + batched_body::{BatchedBody, Frames}, tcp::{ ChunkedHttpBody, ChunkedHttpError, ChunkedRequest, DaemonTcp, HttpRequest, - HttpResponseFallback, InternalHttpBody, InternalHttpBodyFrame, InternalHttpRequest, - StealType, TcpClose, TcpData, HTTP_CHUNKED_REQUEST_VERSION, HTTP_FILTERED_UPGRADE_VERSION, - HTTP_FRAMED_VERSION, + InternalHttpBody, InternalHttpBodyFrame, InternalHttpRequest, StealType, TcpClose, TcpData, + HTTP_CHUNKED_REQUEST_VERSION, HTTP_FILTERED_UPGRADE_VERSION, HTTP_FRAMED_VERSION, }, ConnectionId, Port, RemoteError::{BadHttpFilterExRegex, BadHttpFilterRegex}, @@ -31,6 +30,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use tracing::warn; +use super::http::HttpResponseFallback; use crate::{ error::{AgentError, Result}, steal::{ @@ -173,7 +173,7 @@ impl Client { }, mut body, ) = request.request.into_parts(); - match body.next_frames(true).await { + match body.ready_frames() { Err(..) => return, // We don't check is_last here since loop will finish when body.next_frames() // returns None @@ -205,7 +205,7 @@ impl Client { } loop { - match body.next_frames(false).await { + match body.next_frames().await { Ok(Frames { frames, is_last }) => { let frames = frames .into_iter() @@ -599,40 +599,16 @@ impl TcpConnectionStealer { async fn send_http_response(&mut self, client_id: ClientId, response: HttpResponseFallback) { let connection_id = response.connection_id(); let request_id = response.request_id(); - - match response.into_hyper::() { - Ok(response) => { - self.connections - .send( - connection_id, - ConnectionMessageIn::Response { - client_id, - request_id, - response, - }, - ) - .await; - } - Err(error) => { - tracing::warn!( - ?error, - connection_id, - request_id, + self.connections + .send( + connection_id, + ConnectionMessageIn::Response { client_id, - "Failed to transform client message into a hyper response", - ); - - self.connections - .send( - connection_id, - ConnectionMessageIn::ResponseFailed { - client_id, - request_id, - }, - ) - .await; - } - } + request_id, + response: response.into_hyper::(), + }, + ) + .await; } /// Handles [`Command`]s that were received by [`TcpConnectionStealer::command_rx`]. diff --git a/mirrord/agent/src/steal/http.rs b/mirrord/agent/src/steal/http.rs index 159d9c9aac8..cad0308bc96 100644 --- a/mirrord/agent/src/steal/http.rs +++ b/mirrord/agent/src/steal/http.rs @@ -3,11 +3,12 @@ use crate::http::HttpVersion; mod filter; +mod response_fallback; mod reversible_stream; -pub use filter::HttpFilter; - -pub(crate) use self::reversible_stream::ReversibleStream; +pub(crate) use filter::HttpFilter; +pub(crate) use response_fallback::{HttpResponseFallback, ReceiverStreamBody}; +pub(crate) use reversible_stream::ReversibleStream; /// Handy alias due to [`ReversibleStream`] being generic, avoiding value mismatches. pub(crate) type DefaultReversibleStream = ReversibleStream<{ HttpVersion::MINIMAL_HEADER_SIZE }>; diff --git a/mirrord/agent/src/steal/http/response_fallback.rs b/mirrord/agent/src/steal/http/response_fallback.rs new file mode 100644 index 00000000000..2124ec41a57 --- /dev/null +++ b/mirrord/agent/src/steal/http/response_fallback.rs @@ -0,0 +1,58 @@ +use std::convert::Infallible; + +use bytes::Bytes; +use http_body_util::{combinators::BoxBody, BodyExt, Full, StreamBody}; +use hyper::{body::Frame, Response}; +use mirrord_protocol::{ + tcp::{HttpResponse, InternalHttpBody}, + ConnectionId, RequestId, +}; +use tokio_stream::wrappers::ReceiverStream; + +pub type ReceiverStreamBody = StreamBody, Infallible>>>; + +#[derive(Debug)] +pub enum HttpResponseFallback { + Framed(HttpResponse), + Fallback(HttpResponse>), + Streamed(HttpResponse), +} + +impl HttpResponseFallback { + pub fn connection_id(&self) -> ConnectionId { + match self { + HttpResponseFallback::Framed(req) => req.connection_id, + HttpResponseFallback::Fallback(req) => req.connection_id, + HttpResponseFallback::Streamed(req) => req.connection_id, + } + } + + pub fn request_id(&self) -> RequestId { + match self { + HttpResponseFallback::Framed(req) => req.request_id, + HttpResponseFallback::Fallback(req) => req.request_id, + HttpResponseFallback::Streamed(req) => req.request_id, + } + } + + pub fn into_hyper(self) -> Response> { + match self { + HttpResponseFallback::Framed(req) => req + .internal_response + .map_body(|body| body.map_err(|_| unreachable!()).boxed()) + .into(), + HttpResponseFallback::Fallback(req) => req + .internal_response + .map_body(|body| { + Full::new(Bytes::from_owner(body)) + .map_err(|_| unreachable!()) + .boxed() + }) + .into(), + HttpResponseFallback::Streamed(req) => req + .internal_response + .map_body(|body| body.map_err(|_| unreachable!()).boxed()) + .into(), + } + } +} diff --git a/mirrord/cli/src/port_forward.rs b/mirrord/cli/src/port_forward.rs index 0f701eb9890..220dc1c337e 100644 --- a/mirrord/cli/src/port_forward.rs +++ b/mirrord/cli/src/port_forward.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet, VecDeque}, - net::{IpAddr, SocketAddr}, + net::{IpAddr, Ipv4Addr, SocketAddr}, time::{Duration, Instant}, }; @@ -11,12 +11,8 @@ use mirrord_config::feature::network::incoming::{ }; use mirrord_intproxy::{ background_tasks::{BackgroundTasks, TaskError, TaskSender, TaskUpdate}, - error::IntProxyError, - main_tasks::{MainTaskId, ProxyMessage, ToLayer}, - proxies::incoming::{ - port_subscription_ext::PortSubscriptionExt, IncomingProxy, IncomingProxyError, - IncomingProxyMessage, - }, + main_tasks::{ProxyMessage, ToLayer}, + proxies::incoming::{IncomingProxy, IncomingProxyError, IncomingProxyMessage}, }; use mirrord_intproxy_protocol::{ IncomingRequest, IncomingResponse, LayerId, PortSubscribe, PortSubscription, @@ -28,7 +24,7 @@ use mirrord_protocol::{ tcp::{DaemonTcpOutgoing, LayerTcpOutgoing}, LayerClose, LayerConnect, LayerWrite, SocketAddress, }, - tcp::{Filter, HttpFilter, LayerTcp, LayerTcpSteal, StealType}, + tcp::{Filter, HttpFilter, StealType}, ClientMessage, ConnectionId, DaemonMessage, LogLevel, Port, CLIENT_READY_FOR_LOGS, }; use thiserror::Error; @@ -427,44 +423,57 @@ impl PortForwarder { } pub struct ReversePortForwarder { - /// details for traffic mirroring or stealing - incoming_mode: IncomingMode, /// communicates with the agent (only TCP supported). agent_connection: AgentConnection, - /// associates destination ports with local ports. - mappings: HashMap, - /// background task (uses IncomingProxy to communicate with layer) - background_tasks: BackgroundTasks, + /// background task (uses [`IncomingProxy`] to communicate with layer) + background_tasks: BackgroundTasks<(), ProxyMessage, IncomingProxyError>, /// incoming proxy background task tx incoming_proxy: TaskSender, - - /// true if Ping has been sent to agent. + /// `true` if [`ClientMessage::Ping`] has been sent to agent and we're waiting for the the + /// [`DaemonMessage::Pong`] waiting_for_pong: bool, ping_pong_timeout: Instant, } impl ReversePortForwarder { pub(crate) async fn new( - agent_connection: AgentConnection, + mut agent_connection: AgentConnection, mappings: HashMap, network_config: IncomingConfig, ) -> Result { - // setup IncomingProxy - let mut background_tasks: BackgroundTasks = + let mut background_tasks: BackgroundTasks<(), ProxyMessage, IncomingProxyError> = Default::default(); - let incoming = - background_tasks.register(IncomingProxy::default(), MainTaskId::IncomingProxy, 512); - // construct IncomingMode from config file + let incoming = background_tasks.register(IncomingProxy::default(), (), 512); + + agent_connection + .sender + .send(ClientMessage::SwitchProtocolVersion( + mirrord_protocol::VERSION.clone(), + )) + .await?; + let protocol_version = match agent_connection.receiver.recv().await { + Some(DaemonMessage::SwitchProtocolVersionResponse(version)) => version, + _ => return Err(PortForwardError::AgentConnectionFailed), + }; + + if CLIENT_READY_FOR_LOGS.matches(&protocol_version) { + agent_connection + .sender + .send(ClientMessage::ReadyForLogs) + .await?; + } + + incoming + .send(IncomingProxyMessage::AgentProtocolVersion(protocol_version)) + .await; + let incoming_mode = IncomingMode::new(&network_config); for (i, (&remote, &local)) in mappings.iter().enumerate() { - // send subscription to incoming proxy let subscription = incoming_mode.subscription(remote); let message_id = i as u64; let layer_id = LayerId(1); let req = IncomingRequest::PortSubscribe(PortSubscribe { - listening_on: format!("127.0.0.1:{local}") - .parse() - .expect("Error parsing socket address"), + listening_on: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), local), subscription, }); incoming @@ -475,9 +484,7 @@ impl ReversePortForwarder { } Ok(Self { - incoming_mode, agent_connection, - mappings, background_tasks, incoming_proxy: incoming, waiting_for_pong: false, @@ -486,31 +493,6 @@ impl ReversePortForwarder { } pub(crate) async fn run(&mut self) -> Result<(), PortForwardError> { - // setup agent connection - self.agent_connection - .sender - .send(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )) - .await?; - match self.agent_connection.receiver.recv().await { - Some(DaemonMessage::SwitchProtocolVersionResponse(version)) - if CLIENT_READY_FOR_LOGS.matches(&version) => - { - self.agent_connection - .sender - .send(ClientMessage::ReadyForLogs) - .await?; - } - _ => return Err(PortForwardError::AgentConnectionFailed), - } - - for remote_port in self.mappings.keys() { - let subscription = self.incoming_mode.subscription(*remote_port); - let msg = subscription.agent_subscribe(); - self.agent_connection.sender.send(msg).await? - } - loop { select! { _ = tokio::time::sleep_until(self.ping_pong_timeout.into()) => { @@ -531,8 +513,8 @@ impl ReversePortForwarder { }, }, - Some((task_id, update)) = self.background_tasks.next() => { - self.handle_msg_from_local(task_id, update).await? + Some((_, update)) = self.background_tasks.next() => { + self.handle_msg_from_local(update).await? }, } } @@ -565,8 +547,8 @@ impl ReversePortForwarder { DaemonMessage::Pong if self.waiting_for_pong => { self.waiting_for_pong = false; } + // Includes unexpected DaemonMessage::Pong other => { - // includes unexepcted DaemonMessage::Pong return Err(PortForwardError::AgentError(format!( "unexpected message from agent: {other:?}" ))); @@ -579,20 +561,11 @@ impl ReversePortForwarder { #[tracing::instrument(level = Level::TRACE, skip(self), err)] async fn handle_msg_from_local( &mut self, - task_id: MainTaskId, - update: TaskUpdate, + update: TaskUpdate, ) -> Result<(), PortForwardError> { - match (task_id, update) { - (MainTaskId::IncomingProxy, TaskUpdate::Message(message)) => match message { + match update { + TaskUpdate::Message(message) => match message { ProxyMessage::ToAgent(message) => { - if matches!( - message, - ClientMessage::TcpSteal(LayerTcpSteal::PortSubscribe(_)) - | ClientMessage::Tcp(LayerTcp::PortSubscribe(_)) - ) { - // suppress additional subscription requests - return Ok(()); - } self.agent_connection.sender.send(message).await?; } ProxyMessage::ToLayer(ToLayer { @@ -600,9 +573,7 @@ impl ReversePortForwarder { .. }) => { if let Err(error) = res { - return Err(PortForwardError::from(IntProxyError::from( - IncomingProxyError::SubscriptionFailed(error), - ))); + return Err(IncomingProxyError::SubscriptionFailed(error).into()); } } other => { @@ -611,21 +582,20 @@ impl ReversePortForwarder { ) } }, - (MainTaskId::IncomingProxy, TaskUpdate::Finished(result)) => match result { + + TaskUpdate::Finished(result) => match result { Ok(()) => { - tracing::error!("incoming proxy task finished unexpectedly"); - return Err(IntProxyError::TaskExit(task_id).into()); + unreachable!( + "IncomingProxy should not finish, task sender is alive in this struct" + ); } Err(TaskError::Error(e)) => { - tracing::error!("incoming proxy task failed: {e}"); return Err(e.into()); } Err(TaskError::Panic) => { - tracing::error!("incoming proxy task panicked"); - return Err(IntProxyError::TaskPanic(task_id).into()); + return Err(PortForwardError::IncomingProxyPanicked); } }, - _ => unreachable!("other task types are never used in port forwarding"), } Ok(()) @@ -976,15 +946,17 @@ pub enum PortForwardError { #[error("multiple port forwarding mappings found for desination port `{0:?}`")] ReversePortMapSetupError(RemotePort), - // running errors #[error("agent closed connection with error: `{0}`")] AgentError(String), #[error("connection with the agent failed")] AgentConnectionFailed, - #[error("error from Incoming Proxy task")] - IncomingProxyError(IntProxyError), + #[error("error from the IncomingProxy task: {0}")] + IncomingProxyError(#[from] IncomingProxyError), + + #[error("IncomingProxy task panicked")] + IncomingProxyPanicked, #[error("TcpListener operation failed with error: `{0}`")] TcpListenerError(std::io::Error), @@ -1005,12 +977,6 @@ impl From> for PortForwardError { } } -impl From for PortForwardError { - fn from(value: IntProxyError) -> Self { - Self::IncomingProxyError(value) - } -} - #[cfg(test)] mod test { use std::{ @@ -1026,9 +992,9 @@ mod test { DaemonConnect, DaemonRead, LayerConnect, LayerWrite, SocketAddress, }, tcp::{ - DaemonTcp, Filter, HttpRequest, HttpResponse, InternalHttpRequest, - InternalHttpResponse, LayerTcp, LayerTcpSteal, NewTcpConnection, StealType, TcpClose, - TcpData, + DaemonTcp, Filter, HttpRequest, HttpResponse, InternalHttpBody, InternalHttpBodyFrame, + InternalHttpRequest, InternalHttpResponse, LayerTcp, LayerTcpSteal, NewTcpConnection, + StealType, TcpClose, TcpData, }, ClientMessage, DaemonMessage, }; @@ -1046,90 +1012,142 @@ mod test { RemoteAddr, }; + /// Connects [`ReversePortForwarder`] with test code with [`ClientMessage`] and + /// [`DaemonMessage`] channels. Runs a background [`tokio::task`] that auto responds to + /// standard [`mirrord_protocol`] messages (e.g [`ClientMessage::Ping`]). + struct TestAgentConnection { + daemon_msg_tx: mpsc::Sender, + client_msg_rx: mpsc::Receiver, + } + + impl TestAgentConnection { + fn new() -> (Self, AgentConnection) { + let (daemon_to_forwarder, daemon_from_forwarder) = mpsc::channel::(8); + let (client_task_to_test, client_task_from_test) = mpsc::channel::(8); + let (client_forwarder_to_task, client_task_from_forwarder) = + mpsc::channel::(8); + + tokio::spawn(Self::auto_responder( + client_task_from_forwarder, + client_task_to_test, + daemon_to_forwarder.clone(), + )); + + ( + Self { + daemon_msg_tx: daemon_to_forwarder, + client_msg_rx: client_task_from_test, + }, + AgentConnection { + sender: client_forwarder_to_task, + receiver: daemon_from_forwarder, + }, + ) + } + + /// Sends the [`DaemonMessage`] to the [`ReversePortForwarder`]. + async fn send(&self, message: DaemonMessage) { + self.daemon_msg_tx.send(message).await.unwrap(); + } + + /// Receives a [`ClientMessage`] from the [`ReversePortForwarder`]. + /// + /// Some standard messages are handled internally and are never returned: + /// 1. [`ClientMessage::Ping`] + /// 2. [`ClientMessage::SwitchProtocolVersion`] + /// 3. [`ClientMessage::ReadyForLogs`] + async fn recv(&mut self) -> ClientMessage { + self.client_msg_rx.recv().await.unwrap() + } + + async fn auto_responder( + mut rx: mpsc::Receiver, + tx_to_test_code: mpsc::Sender, + tx_to_port_forwarder: mpsc::Sender, + ) { + loop { + let Some(message) = rx.recv().await else { + break; + }; + + match message { + ClientMessage::Ping => { + tx_to_port_forwarder + .send(DaemonMessage::Pong) + .await + .unwrap(); + } + ClientMessage::ReadyForLogs => {} + ClientMessage::SwitchProtocolVersion(version) => { + tx_to_port_forwarder + .send(DaemonMessage::SwitchProtocolVersionResponse( + std::cmp::min(&version, &*mirrord_protocol::VERSION).clone(), + )) + .await + .unwrap(); + } + other => tx_to_test_code.send(other).await.unwrap(), + } + } + } + } + #[tokio::test] async fn single_port_forwarding() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_destination = listener.local_addr().unwrap(); drop(listener); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); + let (mut test_connection, agent_connection) = TestAgentConnection::new(); - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; - let remote_destination = (RemoteAddr::Ip("152.37.40.40".parse().unwrap()), 3038); + let remote_ip = "152.37.40.40".parse::().unwrap(); + let remote_destination = (RemoteAddr::Ip(remote_ip), 3038); let mappings = HashMap::from([(local_destination, remote_destination.clone())]); - tokio::spawn(async move { - let mut port_forwarder = PortForwarder::new(agent_connection, mappings) - .await - .unwrap(); - port_forwarder.run().await.unwrap() - }); - - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) + // Prepare listeners before sending work to the background task. + let mut port_forwarder = PortForwarder::new(agent_connection, mappings) .await .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); + tokio::spawn(async move { port_forwarder.run().await.unwrap() }); - // send data to socket + // Connect to PortForwarders listener and send some data to trigger remote connection + // request. let mut stream = TcpStream::connect(local_destination).await.unwrap(); stream.write_all(b"data-my-beloved").await.unwrap(); - // expect Connect on client_msg_rx - let remote_address = SocketAddress::Ip("152.37.40.40:3038".parse().unwrap()); + // Expect a connection request + let remote_address = SocketAddress::Ip(SocketAddr::new(remote_ip.into(), 3038)); let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Connect(LayerConnect { remote_address: remote_address.clone(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected,); // reply with successful on daemon_msg_tx - daemon_msg_tx + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Connect(Ok( DaemonConnect { connection_id: 1, - remote_address: remote_address.clone(), - local_address: remote_address, + remote_address, + local_address: "1.2.3.4:2137".parse::().unwrap().into(), }, )))) - .await - .unwrap(); + .await; let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Write(LayerWrite { connection_id: 1, bytes: b"data-my-beloved".to_vec(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected); // send response data from agent on daemon_msg_tx - daemon_msg_tx + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Read(Ok( DaemonRead { connection_id: 1, bytes: b"reply-my-beloved".to_vec(), }, )))) - .await - .unwrap(); + .await; // check data arrives at local let mut buf = [0; 16]; @@ -1149,38 +1167,17 @@ mod test { let local_destination_2 = listener.local_addr().unwrap(); drop(listener); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); - - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; + let (mut test_connection, agent_connection) = TestAgentConnection::new(); let mappings = HashMap::from([ (local_destination_1, remote_destination_1.clone()), (local_destination_2, remote_destination_2.clone()), ]); - tokio::spawn(async move { - let mut port_forwarder = PortForwarder::new(agent_connection, mappings) - .await - .unwrap(); - port_forwarder.run().await.unwrap() - }); - - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) + // Prepare listeners before sending work to the background task. + let mut port_forwarder = PortForwarder::new(agent_connection, mappings) .await .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); + tokio::spawn(async move { port_forwarder.run().await.unwrap() }); // send data to first socket let mut stream_1 = TcpStream::connect(local_destination_1).await.unwrap(); @@ -1196,11 +1193,7 @@ mod test { let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Connect(LayerConnect { remote_address: remote_address_1.clone(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected); // send data to second socket let mut stream_2 = TcpStream::connect(local_destination_2).await.unwrap(); @@ -1214,14 +1207,10 @@ mod test { let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Connect(LayerConnect { remote_address: remote_address_2.clone(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected); // reply with successful on each daemon_msg_tx - daemon_msg_tx + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Connect(Ok( DaemonConnect { connection_id: 1, @@ -1229,9 +1218,8 @@ mod test { local_address: remote_address_1, }, )))) - .await - .unwrap(); - daemon_msg_tx + .await; + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Connect(Ok( DaemonConnect { connection_id: 2, @@ -1239,49 +1227,38 @@ mod test { local_address: remote_address_2, }, )))) - .await - .unwrap(); + .await; // expect data to be received let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Write(LayerWrite { connection_id: 1, bytes: b"data-from-1".to_vec(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected); let expected = ClientMessage::TcpOutgoing(LayerTcpOutgoing::Write(LayerWrite { connection_id: 2, bytes: b"data-from-2".to_vec(), })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected); + assert_eq!(test_connection.recv().await, expected); // send each data response from agent on daemon_msg_tx - daemon_msg_tx + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Read(Ok( DaemonRead { connection_id: 1, bytes: b"reply-to-1".to_vec(), }, )))) - .await - .unwrap(); - daemon_msg_tx + .await; + test_connection .send(DaemonMessage::TcpOutgoing(DaemonTcpOutgoing::Read(Ok( DaemonRead { connection_id: 2, bytes: b"reply-to-2".to_vec(), }, )))) - .await - .unwrap(); + .await; // check data arrives at each local addr let mut buf = [0; 10]; @@ -1299,54 +1276,33 @@ mod test { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_destination = listener.local_addr().unwrap(); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); - - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; let remote_address = IpAddr::from("152.37.40.40".parse::().unwrap()); let destination_port = 3038; let mappings = HashMap::from([(destination_port, local_destination.port())]); let network_config = IncomingConfig::default(); + let (mut test_connection, agent_connection) = TestAgentConnection::new(); + tokio::spawn(async move { - let mut port_forwarder = - ReversePortForwarder::new(agent_connection, mappings, network_config) - .await - .unwrap(); - port_forwarder.run().await.unwrap() + ReversePortForwarder::new(agent_connection, mappings, network_config) + .await + .unwrap() + .run() + .await + .unwrap() }); - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) - .await - .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); - // expect port subscription for remote port and send subscribe result - let expected = Some(ClientMessage::Tcp(LayerTcp::PortSubscribe( - destination_port, - ))); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx + let expected = ClientMessage::Tcp(LayerTcp::PortSubscribe(destination_port)); + assert_eq!(test_connection.recv().await, expected); + test_connection .send(DaemonMessage::Tcp(DaemonTcp::SubscribeResult(Ok( destination_port, )))) - .await - .unwrap(); + .await; // send new connection from agent and some data - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::NewConnection( NewTcpConnection { connection_id: 1, @@ -1356,17 +1312,15 @@ mod test { local_address: local_destination.ip(), }, ))) - .await - .unwrap(); + .await; let mut stream = listener.accept().await.unwrap().0; - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Data(TcpData { connection_id: 1, bytes: b"data-my-beloved".to_vec(), }))) - .await - .unwrap(); + .await; // check data arrives at local let mut buf = [0; 15]; @@ -1374,12 +1328,11 @@ mod test { assert_eq!(buf, b"data-my-beloved".as_ref()); // ensure graceful behaviour on close - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Close(TcpClose { connection_id: 1, }))) - .await - .unwrap(); + .await; } #[rstest] @@ -1389,13 +1342,6 @@ mod test { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_destination = listener.local_addr().unwrap(); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); - - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; let remote_address = IpAddr::from("152.37.40.40".parse::().unwrap()); let destination_port = 3038; let mappings = HashMap::from([(destination_port, local_destination.port())]); @@ -1404,62 +1350,47 @@ mod test { ..Default::default() }; + let (mut test_connection, agent_connection) = TestAgentConnection::new(); tokio::spawn(async move { - let mut port_forwarder = - ReversePortForwarder::new(agent_connection, mappings, network_config) - .await - .unwrap(); - port_forwarder.run().await.unwrap() + ReversePortForwarder::new(agent_connection, mappings, network_config) + .await + .unwrap() + .run() + .await + .unwrap() }); - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) - .await - .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); - // expect port subscription for remote port and send subscribe result - let expected = Some(ClientMessage::TcpSteal(LayerTcpSteal::PortSubscribe( - StealType::All(destination_port), + let expected = ClientMessage::TcpSteal(LayerTcpSteal::PortSubscribe(StealType::All( + destination_port, ))); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::Tcp(DaemonTcp::SubscribeResult(Ok( + assert_eq!(test_connection.recv().await, expected); + test_connection + .send(DaemonMessage::TcpSteal(DaemonTcp::SubscribeResult(Ok( destination_port, )))) - .await - .unwrap(); + .await; // send new connection from agent and some data - daemon_msg_tx - .send(DaemonMessage::Tcp(DaemonTcp::NewConnection( + test_connection + .send(DaemonMessage::TcpSteal(DaemonTcp::NewConnection( NewTcpConnection { connection_id: 1, remote_address, destination_port, - source_port: local_destination.port(), - local_address: local_destination.ip(), + source_port: 2137, + local_address: "1.2.3.4".parse().unwrap(), }, ))) - .await - .unwrap(); + .await; let mut stream = listener.accept().await.unwrap().0; - daemon_msg_tx + test_connection .send(DaemonMessage::TcpSteal(DaemonTcp::Data(TcpData { connection_id: 1, bytes: b"data-my-beloved".to_vec(), }))) - .await - .unwrap(); + .await; // check data arrives at local let mut buf = [0; 15]; @@ -1468,12 +1399,8 @@ mod test { // check for response from local stream.write_all(b"reply-my-beloved").await.unwrap(); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; assert_eq!( - message, + test_connection.recv().await, ClientMessage::TcpSteal(LayerTcpSteal::Data(TcpData { connection_id: 1, bytes: b"reply-my-beloved".to_vec() @@ -1481,12 +1408,11 @@ mod test { ); // ensure graceful behaviour on close - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Close(TcpClose { connection_id: 1, }))) - .await - .unwrap(); + .await; } #[rstest] @@ -1499,13 +1425,6 @@ mod test { let local_destination_1 = listener_1.local_addr().unwrap(); let local_destination_2 = listener_2.local_addr().unwrap(); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); - - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; let remote_address = IpAddr::from("152.37.40.40".parse::().unwrap()); let destination_port_1 = 3038; let destination_port_2 = 4048; @@ -1515,6 +1434,7 @@ mod test { ]); let network_config = IncomingConfig::default(); + let (mut test_connection, agent_connection) = TestAgentConnection::new(); tokio::spawn(async move { let mut port_forwarder = ReversePortForwarder::new(agent_connection, mappings, network_config) @@ -1523,48 +1443,29 @@ mod test { port_forwarder.run().await.unwrap() }); - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) - .await - .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); - // expect port subscription for each remote port and send subscribe result // matches! used because order may be random for _ in 0..2 { - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; + let message = test_connection.recv().await; assert!( matches!(message, ClientMessage::Tcp(LayerTcp::PortSubscribe(_))), "expected ClientMessage::Tcp(LayerTcp::PortSubscribe(_), received {message:?}" ); } - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::SubscribeResult(Ok( destination_port_1, )))) - .await - .unwrap(); - daemon_msg_tx + .await; + test_connection .send(DaemonMessage::Tcp(DaemonTcp::SubscribeResult(Ok( destination_port_2, )))) - .await - .unwrap(); + .await; // send new connections from agent and some data - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::NewConnection( NewTcpConnection { connection_id: 1, @@ -1574,11 +1475,10 @@ mod test { local_address: local_destination_1.ip(), }, ))) - .await - .unwrap(); + .await; let mut stream_1 = listener_1.accept().await.unwrap().0; - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::NewConnection( NewTcpConnection { connection_id: 2, @@ -1588,25 +1488,22 @@ mod test { local_address: local_destination_2.ip(), }, ))) - .await - .unwrap(); + .await; let mut stream_2 = listener_2.accept().await.unwrap().0; - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Data(TcpData { connection_id: 1, bytes: b"connection-1-my-beloved".to_vec(), }))) - .await - .unwrap(); + .await; - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Data(TcpData { connection_id: 2, bytes: b"connection-2-my-beloved".to_vec(), }))) - .await - .unwrap(); + .await; // check data arrives at local let mut buf = [0; 23]; @@ -1618,19 +1515,17 @@ mod test { assert_eq!(buf, b"connection-2-my-beloved".as_ref()); // ensure graceful behaviour on close - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Close(TcpClose { connection_id: 1, }))) - .await - .unwrap(); + .await; - daemon_msg_tx + test_connection .send(DaemonMessage::Tcp(DaemonTcp::Close(TcpClose { connection_id: 2, }))) - .await - .unwrap(); + .await; } #[rstest] @@ -1642,14 +1537,6 @@ mod test { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let local_destination = listener.local_addr().unwrap(); - let (daemon_msg_tx, daemon_msg_rx) = mpsc::channel::(12); - let (client_msg_tx, mut client_msg_rx) = mpsc::channel::(12); - - let agent_connection = AgentConnection { - sender: client_msg_tx, - receiver: daemon_msg_rx, - }; - let remote_address = IpAddr::from("152.37.40.40".parse::().unwrap()); let destination_port = 8080; let mappings = HashMap::from([(destination_port, local_destination.port())]); let mut network_config = IncomingConfig { @@ -1658,6 +1545,8 @@ mod test { }; network_config.http_filter.header_filter = Some("header: value".to_string()); + let (mut test_connection, agent_connection) = TestAgentConnection::new(); + tokio::spawn(async move { let mut port_forwarder = ReversePortForwarder::new(agent_connection, mappings, network_config) @@ -1666,27 +1555,8 @@ mod test { port_forwarder.run().await.unwrap() }); - // expect handshake procedure - let expected = Some(ClientMessage::SwitchProtocolVersion( - mirrord_protocol::VERSION.clone(), - )); - assert_eq!(client_msg_rx.recv().await, expected); - daemon_msg_tx - .send(DaemonMessage::SwitchProtocolVersionResponse( - mirrord_protocol::VERSION.clone(), - )) - .await - .unwrap(); - let expected = Some(ClientMessage::ReadyForLogs); - assert_eq!(client_msg_rx.recv().await, expected); - - // expect port subscription for remote port and send subscribe result - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; assert_eq!( - message, + test_connection.recv().await, ClientMessage::TcpSteal(LayerTcpSteal::PortSubscribe(StealType::FilteredHttpEx( destination_port, mirrord_protocol::tcp::HttpFilter::Header( @@ -1694,27 +1564,11 @@ mod test { ) ),)) ); - daemon_msg_tx - .send(DaemonMessage::Tcp(DaemonTcp::SubscribeResult(Ok( + test_connection + .send(DaemonMessage::TcpSteal(DaemonTcp::SubscribeResult(Ok( destination_port, )))) - .await - .unwrap(); - - // send new connection from agent and some data - daemon_msg_tx - .send(DaemonMessage::TcpSteal(DaemonTcp::NewConnection( - NewTcpConnection { - connection_id: 1, - remote_address, - destination_port, - source_port: local_destination.port(), - local_address: local_destination.ip(), - }, - ))) - .await - .unwrap(); - let mut stream = listener.accept().await.unwrap().0; + .await; // send data from agent with correct header let mut headers = HeaderMap::new(); @@ -1726,23 +1580,22 @@ mod test { version: Version::HTTP_11, body: vec![], }; - daemon_msg_tx + test_connection .send(DaemonMessage::TcpSteal(DaemonTcp::HttpRequest( HttpRequest { internal_request, - connection_id: 1, - request_id: 1, - port: local_destination.port(), + connection_id: 0, + request_id: 0, + port: destination_port, }, ))) - .await - .unwrap(); + .await; + let mut stream = listener.accept().await.unwrap().0; // check data is read from stream let mut buf = [0; 15]; assert_eq!(buf, [0; 15]); stream.read_exact(&mut buf).await.unwrap(); - assert_ne!(buf, [0; 15]); // check for response from local stream @@ -1753,31 +1606,30 @@ mod test { let mut headers = HeaderMap::new(); headers.insert("content-length", "3".parse().unwrap()); let internal_response = InternalHttpResponse { - status: StatusCode::from_u16(200).unwrap(), + status: StatusCode::OK, version: Version::HTTP_11, headers, - body: b"yay".to_vec(), + body: InternalHttpBody( + [InternalHttpBodyFrame::Data(b"yay".into())] + .into_iter() + .collect(), + ), }; let expected_response = - ClientMessage::TcpSteal(LayerTcpSteal::HttpResponse(HttpResponse { - connection_id: 1, - request_id: 1, - port: local_destination.port(), + ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseFramed(HttpResponse { + connection_id: 0, + request_id: 0, + port: destination_port, internal_response, })); - let message = match client_msg_rx.recv().await.ok_or(0).unwrap() { - ClientMessage::Ping => client_msg_rx.recv().await.ok_or(0).unwrap(), - other => other, - }; - assert_eq!(message, expected_response); + assert_eq!(test_connection.recv().await, expected_response); // ensure graceful behaviour on close - daemon_msg_tx - .send(DaemonMessage::Tcp(DaemonTcp::Close(TcpClose { - connection_id: 1, + test_connection + .send(DaemonMessage::TcpSteal(DaemonTcp::Close(TcpClose { + connection_id: 0, }))) - .await - .unwrap(); + .await; } } diff --git a/mirrord/intproxy/Cargo.toml b/mirrord/intproxy/Cargo.toml index ede6a260c02..8b39387ec59 100644 --- a/mirrord/intproxy/Cargo.toml +++ b/mirrord/intproxy/Cargo.toml @@ -33,12 +33,9 @@ tokio.workspace = true tracing.workspace = true tokio-stream.workspace = true hyper = { workspace = true, features = ["client", "http1", "http2"] } -# For checking the `RST_STREAM` error from HTTP2 stealer + filter. -h2 = "0.4" hyper-util.workspace = true http-body-util.workspace = true bytes.workspace = true -futures.workspace = true rand.workspace = true tokio-rustls.workspace = true rustls.workspace = true @@ -46,5 +43,4 @@ rustls-pemfile.workspace = true exponential-backoff = "2" [dev-dependencies] -reqwest.workspace = true rstest.workspace = true diff --git a/mirrord/intproxy/src/background_tasks.rs b/mirrord/intproxy/src/background_tasks.rs index e43c8f306b8..82e6865c67e 100644 --- a/mirrord/intproxy/src/background_tasks.rs +++ b/mirrord/intproxy/src/background_tasks.rs @@ -8,6 +8,7 @@ use std::{collections::HashMap, fmt, future::Future, hash::Hash}; +use thiserror::Error; use tokio::{ sync::mpsc::{self, Receiver, Sender}, task::JoinHandle, @@ -35,6 +36,67 @@ impl MessageBus { msg = self.rx.recv() => msg, } } + + /// Returns a [`Closed`] instance for this [`MessageBus`]. + pub(crate) fn closed(&self) -> Closed { + Closed(self.tx.clone()) + } +} + +/// A helper struct bound to some [`MessageBus`] instance. +/// +/// Used in [`BackgroundTask`]s to `.await` on [`Future`]s without lingering after their +/// [`MessageBus`] is closed. +/// +/// Its lifetime does not depend on the origin [`MessageBus`] and it does not hold any references +/// to it, so that you can use it **and** the [`MessageBus`] at the same time. +/// +/// # Usage example +/// +/// ```ignore +/// use std::convert::Infallible; +/// +/// use mirrord_intproxy::background_tasks::{BackgroundTask, Closed, MessageBus}; +/// +/// struct ExampleTask; +/// +/// impl ExampleTask { +/// /// Thanks to the usage of [`Closed`] in [`Self::run`], +/// /// this function can freely resolve [`Future`]s and use the [`MessageBus`]. +/// /// When the [`MessageBus`] is closed, the whole task will exit. +/// /// +/// /// To achieve the same without [`Closed`], you'd need to wrap each +/// /// [`Future`] resolution with [`tokio::select`]. +/// async fn do_work(&self, message_bus: &mut MessageBus) {} +/// } +/// +/// impl BackgroundTask for ExampleTask { +/// type MessageIn = Infallible; +/// type MessageOut = Infallible; +/// type Error = Infallible; +/// +/// async fn run(self, message_bus: &mut MessageBus) -> Result<(), Self::Error> { +/// let closed: Closed = message_bus.closed(); +/// closed.cancel_on_close(self.do_work(message_bus)).await; +/// Ok(()) +/// } +/// } +/// ``` +pub(crate) struct Closed(Sender); + +impl Closed { + /// Resolves the given [`Future`], unless the origin [`MessageBus`] closes first. + /// + /// # Returns + /// + /// * [`Some`] holding the future output - if the future resolved first + /// * [`None`] - if the [`MessageBus`] closed first + pub(crate) async fn cancel_on_close(&self, future: F) -> Option { + tokio::select! { + _ = self.0.closed() => None, + output = future => Some(output) + } + } } /// Common trait for all background tasks in the internal proxy. @@ -165,12 +227,14 @@ where } /// An error that can occur when executing a [`BackgroundTask`]. -#[derive(Debug)] +#[derive(Debug, Error)] #[cfg_attr(test, derive(PartialEq, Eq))] pub enum TaskError { /// An internal task error. + #[error(transparent)] Error(Err), /// A panic. + #[error("task panicked")] Panic, } diff --git a/mirrord/intproxy/src/proxies/incoming.rs b/mirrord/intproxy/src/proxies/incoming.rs index 966d1175acb..6ffa446dd4c 100644 --- a/mirrord/intproxy/src/proxies/incoming.rs +++ b/mirrord/intproxy/src/proxies/incoming.rs @@ -1,107 +1,63 @@ //! Handles the logic of the `incoming` feature. - -use std::{ - collections::{hash_map::Entry, HashMap}, - fmt, io, - net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, -}; - -use bytes::Bytes; -use futures::StreamExt; -use http::RETRY_ON_RESET_ATTEMPTS; -use http_body_util::StreamBody; -use hyper::body::Frame; +//! +//! +//! Background tasks: +//! 1. TcpProxy - always handles remote connection first. Attempts to connect a couple times. Waits +//! until connection becomes readable (is TCP) or receives an http request. +//! 2. HttpSender - + +use std::{collections::HashMap, io, net::SocketAddr}; + +use bound_socket::BoundTcpSocket; +use http::{ClientStore, ResponseMode, StreamingBody}; +use http_gateway::HttpGatewayTask; +use metadata_store::MetadataStore; use mirrord_intproxy_protocol::{ ConnMetadataRequest, ConnMetadataResponse, IncomingRequest, IncomingResponse, LayerId, - MessageId, PortSubscribe, PortSubscription, PortUnsubscribe, ProxyToLayerMessage, + MessageId, PortSubscription, ProxyToLayerMessage, }; use mirrord_protocol::{ - body_chunks::BodyExt, tcp::{ - ChunkedHttpBody, ChunkedHttpError, ChunkedRequest, ChunkedResponse, DaemonTcp, HttpRequest, - HttpRequestFallback, HttpResponse, HttpResponseFallback, InternalHttpBodyFrame, - InternalHttpRequest, InternalHttpResponse, LayerTcpSteal, NewTcpConnection, - ReceiverStreamBody, StreamingBody, TcpData, + ChunkedHttpBody, ChunkedHttpError, ChunkedRequest, DaemonTcp, HttpRequest, + InternalHttpBodyFrame, LayerTcp, LayerTcpSteal, NewTcpConnection, StealType, TcpData, }, ClientMessage, ConnectionId, RequestId, ResponseError, }; +use tasks::{HttpGatewayId, HttpOut, InProxyTask, InProxyTaskError, InProxyTaskMessage}; +use tcp_proxy::{LocalTcpConnection, TcpProxyTask}; use thiserror::Error; -use tokio::{ - net::TcpSocket, - sync::mpsc::{self, Sender}, -}; -use tokio_stream::{wrappers::ReceiverStream, StreamMap, StreamNotifyClose}; -use tracing::{debug, Level}; +use tokio::sync::mpsc; +use tracing::Level; -use self::{ - interceptor::{Interceptor, InterceptorError, MessageOut}, - port_subscription_ext::PortSubscriptionExt, - subscriptions::SubscriptionsManager, -}; +use self::subscriptions::SubscriptionsManager; use crate::{ - background_tasks::{BackgroundTask, BackgroundTasks, MessageBus, TaskSender, TaskUpdate}, + background_tasks::{ + BackgroundTask, BackgroundTasks, MessageBus, TaskError, TaskSender, TaskUpdate, + }, main_tasks::{LayerClosed, LayerForked, ToLayer}, ProxyMessage, }; +mod bound_socket; mod http; -mod interceptor; -pub mod port_subscription_ext; +mod http_gateway; +mod metadata_store; +mod port_subscription_ext; mod subscriptions; - -/// Creates and binds a new [`TcpSocket`]. -/// The socket has the same IP version and address as the given `addr`. -/// -/// # Exception -/// -/// If the given `addr` is unspecified, this function binds to localhost. -#[tracing::instrument(level = Level::TRACE, ret, err)] -fn bind_similar(addr: SocketAddr) -> io::Result { - match addr.ip() { - IpAddr::V4(Ipv4Addr::UNSPECIFIED) => { - let socket = TcpSocket::new_v4()?; - socket.bind(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))?; - Ok(socket) - } - IpAddr::V6(Ipv6Addr::UNSPECIFIED) => { - let socket = TcpSocket::new_v6()?; - socket.bind(SocketAddr::new(Ipv6Addr::LOCALHOST.into(), 0))?; - Ok(socket) - } - addr @ IpAddr::V4(..) => { - let socket = TcpSocket::new_v4()?; - socket.bind(SocketAddr::new(addr, 0))?; - Ok(socket) - } - addr @ IpAddr::V6(..) => { - let socket = TcpSocket::new_v6()?; - socket.bind(SocketAddr::new(addr, 0))?; - Ok(socket) - } - } -} - -/// Id of a single [`Interceptor`] task. Used to manage interceptor tasks with the -/// [`BackgroundTasks`] struct. -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] -pub struct InterceptorId(pub ConnectionId); - -impl fmt::Display for InterceptorId { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "incoming interceptor {}", self.0,) - } -} +mod tasks; +mod tcp_proxy; /// Errors that can occur when handling the `incoming` feature. #[derive(Error, Debug)] pub enum IncomingProxyError { - #[error(transparent)] - Io(#[from] io::Error), + #[error("failed to prepare a TCP socket: {0}")] + SocketSetupFailed(#[source] io::Error), #[error("subscribing port failed: {0}")] - SubscriptionFailed(ResponseError), + SubscriptionFailed(#[source] ResponseError), } /// Messages consumed by [`IncomingProxy`] running as a [`BackgroundTask`]. +#[derive(Debug)] pub enum IncomingProxyMessage { LayerRequest(MessageId, LayerId, IncomingRequest), LayerForked(LayerForked), @@ -112,544 +68,661 @@ pub enum IncomingProxyMessage { AgentProtocolVersion(semver::Version), } -/// Handle for an [`Interceptor`]. -struct InterceptorHandle { - /// A channel for sending messages to the [`Interceptor`] task. - tx: TaskSender, - /// Port subscription that the intercepted connection belongs to. - subscription: PortSubscription, -} - -/// Store for mapping [`Interceptor`] socket addresses to addresses of the original peers. -#[derive(Default)] -struct MetadataStore { - prepared_responses: HashMap, - expected_requests: HashMap, -} - -impl MetadataStore { - fn get(&mut self, req: ConnMetadataRequest) -> ConnMetadataResponse { - self.prepared_responses - .remove(&req) - .unwrap_or_else(|| ConnMetadataResponse { - remote_source: req.peer_address, - local_address: req.listener_address.ip(), - }) - } - - fn expect(&mut self, req: ConnMetadataRequest, from: InterceptorId, res: ConnMetadataResponse) { - self.expected_requests.insert(from, req.clone()); - self.prepared_responses.insert(req, res); - } - - fn no_longer_expect(&mut self, from: InterceptorId) { - let Some(req) = self.expected_requests.remove(&from) else { - return; - }; - self.prepared_responses.remove(&req); - } +/// Handle to a running [`HttpGatewayTask`]. +struct HttpGatewayHandle { + /// Only keeps the [`HttpGatewayTask`] alive. + _tx: TaskSender, + /// For sending request body [`Frame`](hyper::body::Frame)s. + /// + /// [`None`] if all frames were already sent. + body_tx: Option>, } /// Handles logic and state of the `incoming` feature. /// Run as a [`BackgroundTask`]. /// -/// Handles port subscriptions state of the connected layers. Utilizes multiple background tasks -/// ([`Interceptor`]s) to handle incoming connections. Each connection is managed by a single -/// [`Interceptor`], that establishes a TCP connection with the user application's port and proxies -/// data. +/// Handles port subscriptions state of the connected layers. +/// Utilizes multiple background tasks ([`TcpProxyTask`]s and [`HttpGatewayTask`]s) to handle +/// incoming connections and requests. +/// +/// # Connections mirrored or stolen without a filter /// -/// Incoming connections are created by the agent either explicitly ([`NewTcpConnection`] message) -/// or implicitly ([`HttpRequest`]). +/// Each such connection exists in two places: +/// +/// 1. Here, between the intproxy and the user application. Managed by a single [`TcpProxyTask`]. +/// 2. In the cluster, between the agent and the original TCP client. +/// +/// We are notified about such connections with the [`NewTcpConnection`] message. +/// +/// The local connection lives until the agent or the user application closes it, or a local IO +/// error occurs. When we want to close this connection, we simply drop the [`TcpProxyTask`]'s +/// [`TaskSender`]. When a local IO error occurs, the [`TcpProxyTask`] finishes with an +/// [`InProxyTaskError`]. +/// +/// # Requests stolen with a filter +/// +/// In the cluster, we have a real persistent connection between the agent and the original HTTP +/// client. From this connection, intproxy receives a subset of requests. +/// +/// Locally, we don't have a concept of a filered connection. +/// Each request is handled independently by a single [`HttpGatewayTask`]. +/// Also: +/// 1. Local HTTP connections are reused when possible. +/// 2. Unless the error is fatal, each request is retried a couple of times. +/// 3. We never send [`LayerTcpSteal::ConnectionUnsubscribe`] (due to requests being handled +/// independently). If a request fails locally, we send a +/// [`StatusCode::BAD_GATEWAY`](hyper::http::StatusCode::BAD_GATEWAY) response. +/// +/// We are notified about stolen requests with the [`HttpRequest`] messages. +/// +/// The request can be cancelled only when one of the following happen: +/// 1. The agent closes the remote connection to which this request belongs +/// 2. The agent informs us that it failed to read request body ([`ChunkedRequest::Error`]) +/// +/// When we want to cancel the request, we drop the [`HttpGatewayTask`]'s [`TaskSender`]. +/// +/// # HTTP upgrades +/// +/// An HTTP request stolen with a filter can result in an HTTP upgrade. +/// When this happens, the TCP connection is recovered and passed to a new [`TcpProxyTask`]. +/// The TCP connection is then treated as stolen without a filter. #[derive(Default)] pub struct IncomingProxy { /// Active port subscriptions for all layers. subscriptions: SubscriptionsManager, - /// [`TaskSender`]s for active [`Interceptor`]s. - interceptors: HashMap, - /// For receiving updates from [`Interceptor`]s. - background_tasks: BackgroundTasks, /// For managing intercepted connections metadata. metadata_store: MetadataStore, - /// For managing streamed [`DaemonTcp::HttpRequestChunked`] request channels. - request_body_txs: HashMap<(ConnectionId, RequestId), Sender>, - /// For managing streamed [`LayerTcpSteal::HttpResponseChunked`] response streams. - response_body_rxs: StreamMap<(ConnectionId, RequestId), StreamNotifyClose>, - /// Version of [`mirrord_protocol`] negotiated with the agent. - agent_protocol_version: Option, + /// What HTTP response flavor we produce. + response_mode: ResponseMode, + /// Cache for [`LocalHttpClient`](http::LocalHttpClient)s. + client_store: ClientStore, + /// Each mirrored remote connection is mapped to a [`TcpProxyTask`] in mirror mode. + /// + /// Each entry here maps to a connection that is in progress both locally and remotely. + mirror_tcp_proxies: HashMap>, + /// Each remote connection stolen without a filter is mapped to a [`TcpProxyTask`] in steal + /// mode. + /// + /// Each entry here maps to a connection that is in progress both locally and remotely. + steal_tcp_proxies: HashMap>, + /// Each remote HTTP request stolen with a filter is mapped to a [`HttpGatewayTask`]. + /// + /// Each entry here maps to a request that is in progress both locally and remotely. + http_gateways: HashMap>, + /// Running [`BackgroundTask`]s utilized by this proxy. + tasks: BackgroundTasks, } impl IncomingProxy { - /// Used when registering new `RawInterceptor` and `HttpInterceptor` tasks in the - /// [`BackgroundTasks`] struct. - // TODO: Update outdated documentation. RawInterceptor, HttpInterceptor do not exist + /// Used when registering new tasks in the internal [`BackgroundTasks`] instance. const CHANNEL_SIZE: usize = 512; - /// Tries to register the new subscription in the [`SubscriptionsManager`]. + /// Starts a new [`HttpGatewayTask`] to handle the given request. + /// + /// If we don't have a [`PortSubscription`] for the port, the task is not started. + /// Instead, we respond immediately to the agent. #[tracing::instrument(level = Level::TRACE, skip(self, message_bus))] - async fn handle_port_subscribe( + async fn start_http_gateway( &mut self, - message_id: MessageId, - layer_id: LayerId, - subscribe: PortSubscribe, - message_bus: &mut MessageBus, + request: HttpRequest, + body_tx: Option>, + message_bus: &MessageBus, ) { - let msg = self - .subscriptions - .layer_subscribed(layer_id, message_id, subscribe); + let subscription = self.subscriptions.get(request.port).filter(|subscription| { + matches!( + subscription.subscription, + PortSubscription::Steal( + StealType::FilteredHttp(..) | StealType::FilteredHttpEx(..) + ) + ) + }); + let Some(subscription) = subscription else { + tracing::debug!( + ?request, + "Received a new HTTP request within a stale port subscription, \ + sending an unsubscribe request or an error response." + ); + + let no_other_requests = self + .http_gateways + .get(&request.connection_id) + .map(|gateways| gateways.is_empty()) + .unwrap_or(true); + if no_other_requests { + message_bus + .send(ClientMessage::TcpSteal( + LayerTcpSteal::ConnectionUnsubscribe(request.connection_id), + )) + .await; + } else { + let response = http::mirrord_error_response( + "port no longer subscribed with an HTTP filter", + request.version(), + request.connection_id, + request.request_id, + request.port, + ); + message_bus + .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponse( + response, + ))) + .await; + } - if let Some(msg) = msg { - message_bus.send(msg).await; - } + return; + }; + + let connection_id = request.connection_id; + let request_id = request.request_id; + let id = HttpGatewayId { + connection_id, + request_id, + port: request.port, + version: request.version(), + }; + let tx = self.tasks.register( + HttpGatewayTask::new( + request, + self.client_store.clone(), + self.response_mode, + subscription.listening_on, + ), + InProxyTask::HttpGateway(id), + Self::CHANNEL_SIZE, + ); + self.http_gateways + .entry(connection_id) + .or_default() + .insert(request_id, HttpGatewayHandle { _tx: tx, body_tx }); } - /// Tries to unregister the subscription from the [`SubscriptionsManager`]. + /// Handles [`NewTcpConnection`] message from the agent, starting a new [`TcpProxyTask`]. + /// + /// If we don't have a [`PortSubscription`] for the port, the task is not started. + /// Instead, we respond immediately to the agent. #[tracing::instrument(level = Level::TRACE, skip(self, message_bus))] - async fn handle_port_unsubscribe( + async fn handle_new_connection( &mut self, - layer_id: LayerId, - request: PortUnsubscribe, + connection: NewTcpConnection, + is_steal: bool, message_bus: &mut MessageBus, - ) { - let msg = self.subscriptions.layer_unsubscribed(layer_id, request); + ) -> Result<(), IncomingProxyError> { + let NewTcpConnection { + connection_id, + remote_address, + destination_port, + source_port, + local_address, + } = connection; + + let subscription = self + .subscriptions + .get(destination_port) + .filter(|subscription| match &subscription.subscription { + PortSubscription::Mirror(..) if !is_steal => true, + PortSubscription::Steal(StealType::All(..)) if is_steal => true, + _ => false, + }); + let Some(subscription) = subscription else { + tracing::debug!( + port = destination_port, + connection_id, + "Received a new connection within a stale port subscription, sending an unsubscribe request.", + ); + + let message = if is_steal { + ClientMessage::Tcp(LayerTcp::ConnectionUnsubscribe(connection_id)) + } else { + ClientMessage::TcpSteal(LayerTcpSteal::ConnectionUnsubscribe(connection_id)) + }; + message_bus.send(message).await; + + return Ok(()); + }; + + let socket = BoundTcpSocket::bind_specified_or_localhost(subscription.listening_on.ip()) + .map_err(IncomingProxyError::SocketSetupFailed)?; + + self.metadata_store.expect( + ConnMetadataRequest { + listener_address: subscription.listening_on, + peer_address: socket + .local_addr() + .map_err(IncomingProxyError::SocketSetupFailed)?, + }, + connection_id, + ConnMetadataResponse { + remote_source: SocketAddr::new(remote_address, source_port), + local_address, + }, + ); - if let Some(msg) = msg { - message_bus.send(msg).await; + let id = if is_steal { + InProxyTask::StealTcpProxy(connection_id) + } else { + InProxyTask::MirrorTcpProxy(connection_id) + }; + let tx = self.tasks.register( + TcpProxyTask::new( + LocalTcpConnection::FromTheStart { + socket, + peer: subscription.listening_on, + }, + !is_steal, + ), + id, + Self::CHANNEL_SIZE, + ); + + if is_steal { + self.steal_tcp_proxies.insert(connection_id, tx); + } else { + self.mirror_tcp_proxies.insert(connection_id, tx); } + + Ok(()) } - /// Retrieves or creates an [`Interceptor`] for the given [`HttpRequestFallback`]. - /// The request may or may not belong to an existing connection (when stealing with an http - /// filter, connections are created implicitly). - #[tracing::instrument(level = Level::TRACE, skip(self))] - fn get_interceptor_for_http_request( + /// Handles [`ChunkedRequest`] message from the agent. + async fn handle_chunked_request( &mut self, - request: &HttpRequestFallback, - ) -> Result>, IncomingProxyError> { - let id: InterceptorId = InterceptorId(request.connection_id()); - - let interceptor = match self.interceptors.entry(id) { - Entry::Occupied(e) => e.into_mut(), - - Entry::Vacant(e) => { - let Some(subscription) = self.subscriptions.get(request.port()) else { - tracing::trace!( - "received a new connection for port {} that is no longer mirrored", - request.port(), + request: ChunkedRequest, + message_bus: &mut MessageBus, + ) { + match request { + ChunkedRequest::Start(request) => { + let (body_tx, body_rx) = mpsc::channel(128); + let request = request.map_body(|frames| StreamingBody::new(body_rx, frames)); + self.start_http_gateway(request, Some(body_tx), message_bus) + .await; + } + + ChunkedRequest::Body(ChunkedHttpBody { + frames, + is_last, + connection_id, + request_id, + }) => { + let gateway = self + .http_gateways + .get_mut(&connection_id) + .and_then(|gateways| gateways.get_mut(&request_id)); + let Some(gateway) = gateway else { + tracing::debug!( + connection_id, + request_id, + frames = ?frames, + last_body_chunk = is_last, + "Received a body chunk for a request that is no longer alive locally" ); - return Ok(None); + return; }; - let interceptor_socket = bind_similar(subscription.listening_on)?; + let Some(tx) = gateway.body_tx.as_ref() else { + tracing::debug!( + connection_id, + request_id, + frames = ?frames, + last_body_chunk = is_last, + "Received a body chunk for a request with a closed body" + ); - let interceptor = self.background_tasks.register( - Interceptor::new( - interceptor_socket, - subscription.listening_on, - self.agent_protocol_version.clone(), - ), - id, - Self::CHANNEL_SIZE, - ); + return; + }; - e.insert(InterceptorHandle { - tx: interceptor, - subscription: subscription.subscription.clone(), - }) + for frame in frames { + if let Err(err) = tx.send(frame).await { + tracing::debug!( + frame = ?err.0, + connection_id, + request_id, + "Failed to send an HTTP request body frame to the HttpGatewayTask, channel is closed" + ); + break; + } + } + + if is_last { + gateway.body_tx = None; + } } - }; - Ok(Some(&interceptor.tx)) + ChunkedRequest::Error(ChunkedHttpError { + connection_id, + request_id, + }) => { + tracing::debug!( + connection_id, + request_id, + "Received an error in an HTTP request body", + ); + + if let Some(gateways) = self.http_gateways.get_mut(&connection_id) { + gateways.remove(&request_id); + }; + } + } } /// Handles all agent messages. - #[tracing::instrument(level = Level::TRACE, skip(self, message_bus))] async fn handle_agent_message( &mut self, message: DaemonTcp, + is_steal: bool, message_bus: &mut MessageBus, ) -> Result<(), IncomingProxyError> { match message { DaemonTcp::Close(close) => { - self.interceptors - .remove(&InterceptorId(close.connection_id)); - self.request_body_txs - .retain(|(connection_id, _), _| *connection_id != close.connection_id); - let keys: Vec<(ConnectionId, RequestId)> = self - .response_body_rxs - .keys() - .filter(|key| key.0 == close.connection_id) - .cloned() - .collect(); - for key in keys.iter() { - self.response_body_rxs.remove(key); + if is_steal { + self.steal_tcp_proxies.remove(&close.connection_id); + self.http_gateways.remove(&close.connection_id); + } else { + self.mirror_tcp_proxies.remove(&close.connection_id); } } + DaemonTcp::Data(data) => { - if let Some(interceptor) = self.interceptors.get(&InterceptorId(data.connection_id)) - { - interceptor.tx.send(data.bytes).await; + let tx = if is_steal { + self.steal_tcp_proxies.get(&data.connection_id) + } else { + self.mirror_tcp_proxies.get(&data.connection_id) + }; + + if let Some(tx) = tx { + tx.send(data.bytes).await; } else { - tracing::trace!( - "received new data for connection {} that is already closed", - data.connection_id + tracing::debug!( + connection_id = data.connection_id, + bytes = data.bytes.len(), + "Received new data for a connection that does not belong to any TcpProxy task", ); } } - DaemonTcp::HttpRequest(req) => { - let req = HttpRequestFallback::Fallback(req); - let interceptor = self.get_interceptor_for_http_request(&req)?; - if let Some(interceptor) = interceptor { - interceptor.send(req).await; - } + + DaemonTcp::HttpRequest(request) => { + self.start_http_gateway(request.map_body(From::from), None, message_bus) + .await; } - DaemonTcp::HttpRequestFramed(req) => { - let req = HttpRequestFallback::Framed(req); - let interceptor = self.get_interceptor_for_http_request(&req)?; - if let Some(interceptor) = interceptor { - interceptor.send(req).await; - } + + DaemonTcp::HttpRequestFramed(request) => { + self.start_http_gateway(request.map_body(From::from), None, message_bus) + .await; } - DaemonTcp::HttpRequestChunked(req) => { - match req { - ChunkedRequest::Start(req) => { - let (tx, rx) = mpsc::channel::(128); - let http_stream = StreamingBody::new(rx); - let http_req = HttpRequest { - internal_request: InternalHttpRequest { - method: req.internal_request.method, - uri: req.internal_request.uri, - headers: req.internal_request.headers, - version: req.internal_request.version, - body: http_stream, - }, - connection_id: req.connection_id, - request_id: req.request_id, - port: req.port, - }; - let key = (http_req.connection_id, http_req.request_id); - - self.request_body_txs.insert(key, tx.clone()); - - let http_req = HttpRequestFallback::Streamed { - request: http_req, - retries: 0, - }; - let interceptor = self.get_interceptor_for_http_request(&http_req)?; - if let Some(interceptor) = interceptor { - interceptor.send(http_req).await; - } - for frame in req.internal_request.body { - if let Err(err) = tx.send(frame).await { - self.request_body_txs.remove(&key); - tracing::trace!(?err, "error while sending"); - } - } - } - ChunkedRequest::Body(body) => { - let key = &(body.connection_id, body.request_id); - let mut send_err = false; - if let Some(tx) = self.request_body_txs.get(key) { - for frame in body.frames { - if let Err(err) = tx.send(frame).await { - send_err = true; - tracing::trace!(?err, "error while sending"); - } - } - } - if send_err || body.is_last { - self.request_body_txs.remove(key); - } - } - ChunkedRequest::Error(err) => { - self.request_body_txs - .remove(&(err.connection_id, err.request_id)); - tracing::trace!(?err, "ChunkedRequest error received"); - } - }; + DaemonTcp::HttpRequestChunked(request) => { + self.handle_chunked_request(request, message_bus).await; } - DaemonTcp::NewConnection(NewTcpConnection { - connection_id, - remote_address, - destination_port, - source_port, - local_address, - }) => { - let Some(subscription) = self.subscriptions.get(destination_port) else { - tracing::trace!("received a new connection for port {destination_port} that is no longer mirrored"); - return Ok(()); - }; - let interceptor_socket = bind_similar(subscription.listening_on)?; + DaemonTcp::NewConnection(connection) => { + self.handle_new_connection(connection, is_steal, message_bus) + .await?; + } - let id = InterceptorId(connection_id); + DaemonTcp::SubscribeResult(result) => { + let msgs = self.subscriptions.agent_responded(result)?; - self.metadata_store.expect( - ConnMetadataRequest { - listener_address: subscription.listening_on, - peer_address: interceptor_socket.local_addr()?, - }, - id, - ConnMetadataResponse { - remote_source: SocketAddr::new(remote_address, source_port), - local_address, - }, - ); + for msg in msgs { + message_bus.send(msg).await; + } + } + } - let interceptor = self.background_tasks.register( - Interceptor::new( - interceptor_socket, - subscription.listening_on, - self.agent_protocol_version.clone(), - ), - id, - Self::CHANNEL_SIZE, - ); + Ok(()) + } - self.interceptors.insert( - id, - InterceptorHandle { - tx: interceptor, - subscription: subscription.subscription.clone(), - }, - ); + /// Handles all messages from this task's [`MessageBus`]. + #[tracing::instrument(level = Level::TRACE, skip(self, message_bus), err)] + async fn handle_message( + &mut self, + message: IncomingProxyMessage, + message_bus: &mut MessageBus, + ) -> Result<(), IncomingProxyError> { + match message { + IncomingProxyMessage::LayerRequest(message_id, layer_id, req) => match req { + IncomingRequest::PortSubscribe(subscribe) => { + let msg = self + .subscriptions + .layer_subscribed(layer_id, message_id, subscribe); + + if let Some(msg) = msg { + message_bus.send(msg).await; + } + } + IncomingRequest::PortUnsubscribe(unsubscribe) => { + let msg = self.subscriptions.layer_unsubscribed(layer_id, unsubscribe); + + if let Some(msg) = msg { + message_bus.send(msg).await; + } + } + IncomingRequest::ConnMetadata(req) => { + let res = self.metadata_store.get(req); + message_bus + .send(ToLayer { + message_id, + layer_id, + message: ProxyToLayerMessage::Incoming(IncomingResponse::ConnMetadata( + res, + )), + }) + .await; + } + }, + + IncomingProxyMessage::AgentMirror(msg) => { + self.handle_agent_message(msg, false, message_bus).await?; + } + + IncomingProxyMessage::AgentSteal(msg) => { + self.handle_agent_message(msg, true, message_bus).await?; } - DaemonTcp::SubscribeResult(result) => { - let msgs = self.subscriptions.agent_responded(result)?; + + IncomingProxyMessage::LayerClosed(msg) => { + let msgs = self.subscriptions.layer_closed(msg.id); for msg in msgs { message_bus.send(msg).await; } } + + IncomingProxyMessage::LayerForked(msg) => { + self.subscriptions.layer_forked(msg.parent, msg.child); + } + + IncomingProxyMessage::AgentProtocolVersion(version) => { + self.response_mode = ResponseMode::from(&version); + } } Ok(()) } - fn handle_layer_fork(&mut self, msg: LayerForked) { - let LayerForked { child, parent } = msg; - self.subscriptions.layer_forked(parent, child); - } + /// Handles all updates from [`TcpProxyTask`]s. + #[tracing::instrument(level = Level::TRACE, skip(self, message_bus))] + async fn handle_tcp_proxy_update( + &mut self, + connection_id: ConnectionId, + is_steal: bool, + update: TaskUpdate, + message_bus: &mut MessageBus, + ) { + match update { + TaskUpdate::Finished(result) => { + match result { + Err(TaskError::Error(error)) => { + tracing::warn!(connection_id, %error, is_steal, "TcpProxyTask failed"); + } + Err(TaskError::Panic) => { + tracing::error!(connection_id, is_steal, "TcpProxyTask task panicked"); + } + Ok(()) => {} + }; + + self.metadata_store.no_longer_expect(connection_id); - async fn handle_layer_close(&mut self, msg: LayerClosed, message_bus: &MessageBus) { - let msgs = self.subscriptions.layer_closed(msg.id); + if is_steal { + if self.steal_tcp_proxies.remove(&connection_id).is_some() { + message_bus + .send(ClientMessage::TcpSteal( + LayerTcpSteal::ConnectionUnsubscribe(connection_id), + )) + .await; + } + } else if self.mirror_tcp_proxies.remove(&connection_id).is_some() { + message_bus + .send(ClientMessage::Tcp(LayerTcp::ConnectionUnsubscribe( + connection_id, + ))) + .await; + } + } - for msg in msgs { - message_bus.send(msg).await; + TaskUpdate::Message(..) if !is_steal => { + unreachable!("TcpProxyTask does not produce messages in mirror mode") + } + + TaskUpdate::Message(InProxyTaskMessage::Tcp(bytes)) => { + if self.steal_tcp_proxies.contains_key(&connection_id) { + message_bus + .send(ClientMessage::TcpSteal(LayerTcpSteal::Data(TcpData { + connection_id, + bytes, + }))) + .await; + } + } + + TaskUpdate::Message(InProxyTaskMessage::Http(..)) => { + unreachable!("TcpProxyTask does not produce HTTP messages") + } } } - fn get_subscription(&self, interceptor_id: InterceptorId) -> Option<&PortSubscription> { - self.interceptors - .get(&interceptor_id) - .map(|handle| &handle.subscription) - } -} + /// Handles all updates from [`HttpGatewayTask`]s. + #[tracing::instrument(level = Level::TRACE, skip(self, message_bus))] + async fn handle_http_gateway_update( + &mut self, + id: HttpGatewayId, + update: TaskUpdate, + message_bus: &mut MessageBus, + ) { + match update { + TaskUpdate::Finished(result) => { + let respond_on_panic = self + .http_gateways + .get_mut(&id.connection_id) + .and_then(|gateways| gateways.remove(&id.request_id)) + .is_some(); + + match result { + Ok(()) => {} + Err(TaskError::Error( + InProxyTaskError::IoError(..) | InProxyTaskError::UpgradeError(..), + )) => unreachable!("HttpGatewayTask does not return any errors"), + Err(TaskError::Panic) => { + tracing::error!( + connection_id = id.connection_id, + request_id = id.request_id, + "HttpGatewayTask panicked", + ); + + if respond_on_panic { + let response = http::mirrord_error_response( + "HTTP gateway task panicked", + id.version, + id.connection_id, + id.request_id, + id.port, + ); + message_bus + .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponse( + response, + ))) + .await; + } + } + } + } -impl BackgroundTask for IncomingProxy { - type Error = IncomingProxyError; - type MessageIn = IncomingProxyMessage; - type MessageOut = ProxyMessage; + TaskUpdate::Message(InProxyTaskMessage::Http(message)) => { + let exists = self + .http_gateways + .get(&id.connection_id) + .and_then(|gateways| gateways.get(&id.request_id)) + .is_some(); + if !exists { + return; + } - #[tracing::instrument(level = Level::TRACE, skip_all, err)] - async fn run(mut self, message_bus: &mut MessageBus) -> Result<(), Self::Error> { - loop { - tokio::select! { - Some(((connection_id, request_id), stream_item)) = self.response_body_rxs.next() => match stream_item { - Some(Ok(frame)) => { - let int_frame = InternalHttpBodyFrame::from(frame); - let res = ChunkedResponse::Body(ChunkedHttpBody { - frames: vec![int_frame], - is_last: false, - connection_id, - request_id, - }); + match message { + HttpOut::ResponseBasic(response) => { message_bus - .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseChunked( - res, + .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponse( + response, ))) - .await; - }, - Some(Err(error)) => { - debug!(%error, "Error while reading streamed response body"); - let res = ChunkedResponse::Error(ChunkedHttpError {connection_id, request_id}); + .await + } + HttpOut::ResponseFramed(response) => { message_bus - .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseChunked( - res, + .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseFramed( + response, ))) - .await; - self.response_body_rxs.remove(&(connection_id, request_id)); - }, - None => { - let res = ChunkedResponse::Body(ChunkedHttpBody { - frames: vec![], - is_last: true, - connection_id, - request_id, - }); + .await + } + HttpOut::ResponseChunked(response) => { message_bus .send(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseChunked( - res, + response, ))) .await; - self.response_body_rxs.remove(&(connection_id, request_id)); } - }, + HttpOut::Upgraded(on_upgrade) => { + let proxy = self.tasks.register( + TcpProxyTask::new(LocalTcpConnection::AfterUpgrade(on_upgrade), false), + InProxyTask::StealTcpProxy(id.connection_id), + Self::CHANNEL_SIZE, + ); + self.steal_tcp_proxies.insert(id.connection_id, proxy); + } + } + } + + TaskUpdate::Message(InProxyTaskMessage::Tcp(..)) => { + unreachable!("HttpGatewayTask does not produce TCP messages") + } + } + } +} + +impl BackgroundTask for IncomingProxy { + type Error = IncomingProxyError; + type MessageIn = IncomingProxyMessage; + type MessageOut = ProxyMessage; + #[tracing::instrument(level = Level::TRACE, name = "incoming_proxy_main_loop", skip_all, err)] + async fn run(mut self, message_bus: &mut MessageBus) -> Result<(), Self::Error> { + loop { + tokio::select! { msg = message_bus.recv() => match msg { None => { tracing::trace!("message bus closed, exiting"); break Ok(()); }, - Some(IncomingProxyMessage::LayerRequest(message_id, layer_id, req)) => match req { - IncomingRequest::PortSubscribe(subscribe) => self.handle_port_subscribe(message_id, layer_id, subscribe, message_bus).await, - IncomingRequest::PortUnsubscribe(unsubscribe) => self.handle_port_unsubscribe(layer_id, unsubscribe, message_bus).await, - IncomingRequest::ConnMetadata(req) => { - let res = self.metadata_store.get(req); - message_bus.send(ToLayer { message_id, layer_id, message: ProxyToLayerMessage::Incoming(IncomingResponse::ConnMetadata(res)) }).await; - } - }, - Some(IncomingProxyMessage::AgentMirror(msg)) => { - self.handle_agent_message(msg, message_bus).await?; + Some(message) => self.handle_message(message, message_bus).await?, + }, + + Some((id, update)) = self.tasks.next() => match id { + InProxyTask::MirrorTcpProxy(connection_id) => { + self.handle_tcp_proxy_update(connection_id, false, update, message_bus).await; } - Some(IncomingProxyMessage::AgentSteal(msg)) => { - self.handle_agent_message(msg, message_bus).await?; + InProxyTask::StealTcpProxy(connection_id) => { + self.handle_tcp_proxy_update(connection_id, true, update, message_bus).await; } - Some(IncomingProxyMessage::LayerClosed(msg)) => self.handle_layer_close(msg, message_bus).await, - Some(IncomingProxyMessage::LayerForked(msg)) => self.handle_layer_fork(msg), - Some(IncomingProxyMessage::AgentProtocolVersion(version)) => { - self.agent_protocol_version.replace(version); + InProxyTask::HttpGateway(id) => { + self.handle_http_gateway_update(id, update, message_bus).await; } }, - - Some(task_update) = self.background_tasks.next() => match task_update { - (id, TaskUpdate::Finished(res)) => { - tracing::trace!("{id} finished: {res:?}"); - - self.metadata_store.no_longer_expect(id); - - let msg = self.get_subscription(id).map(|s| s.wrap_agent_unsubscribe_connection(id.0)); - if let Some(msg) = msg { - message_bus.send(msg).await; - } - - self.request_body_txs.retain(|(connection_id, _), _| *connection_id != id.0); - }, - - (id, TaskUpdate::Message(msg)) => { - let Some(PortSubscription::Steal(_)) = self.get_subscription(id) else { - continue; - }; - let msg = match msg { - MessageOut::Raw(bytes) => { - ClientMessage::TcpSteal(LayerTcpSteal::Data(TcpData { - connection_id: id.0, - bytes, - })) - }, - MessageOut::Http(HttpResponseFallback::Fallback(res)) => { - ClientMessage::TcpSteal(LayerTcpSteal::HttpResponse(res)) - }, - MessageOut::Http(HttpResponseFallback::Framed(res)) => { - ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseFramed(res)) - }, - MessageOut::Http(HttpResponseFallback::Streamed(response, request)) => { - match self.streamed_http_response(response, request).await { - Some(response) => response, - None => continue, - } - } - }; - message_bus.send(msg).await; - }, - }, - } - } - } -} - -impl IncomingProxy { - /// Sends back the streamed http response to the agent. - /// - /// If we cannot get the next frame of the streamed body, then we retry the whole - /// process, by sending the original `request` again through the http `interceptor` to - /// our hyper handler. - #[allow(clippy::type_complexity)] - #[tracing::instrument(level = Level::TRACE, skip(self), ret)] - async fn streamed_http_response( - &mut self, - mut response: HttpResponse, hyper::Error>>>>, - request: Option, - ) -> Option { - let mut body = vec![]; - let key = (response.connection_id, response.request_id); - - match response - .internal_response - .body - .next_frames(true) - .await - .map_err(InterceptorError::from) - { - Ok(frames) => { - frames - .frames - .into_iter() - .map(From::from) - .for_each(|frame| body.push(frame)); - - self.response_body_rxs - .insert(key, StreamNotifyClose::new(response.internal_response.body)); - - let internal_response = InternalHttpResponse { - status: response.internal_response.status, - version: response.internal_response.version, - headers: response.internal_response.headers, - body, - }; - let response = ChunkedResponse::Start(HttpResponse { - port: response.port, - connection_id: response.connection_id, - request_id: response.request_id, - internal_response, - }); - Some(ClientMessage::TcpSteal(LayerTcpSteal::HttpResponseChunked( - response, - ))) - } - // Retry on known errors. - Err(error @ InterceptorError::Reset) - | Err(error @ InterceptorError::ConnectionClosedTooSoon(..)) - | Err(error @ InterceptorError::IncompleteMessage(..)) => { - tracing::warn!(%error, ?request, "Failed to read first frames of streaming HTTP response"); - - let interceptor = self - .interceptors - .get(&InterceptorId(response.connection_id))?; - - if let Some(HttpRequestFallback::Streamed { request, retries }) = request - && retries < RETRY_ON_RESET_ATTEMPTS - { - tracing::trace!( - ?request, - ?retries, - "`RST_STREAM` from hyper, retrying the request." - ); - interceptor - .tx - .send(HttpRequestFallback::Streamed { - request, - retries: retries + 1, - }) - .await; - } - - None - } - Err(fail) => { - tracing::warn!(?fail, "Something went wrong, skipping this response!"); - None } } } diff --git a/mirrord/intproxy/src/proxies/incoming/bound_socket.rs b/mirrord/intproxy/src/proxies/incoming/bound_socket.rs new file mode 100644 index 00000000000..1c6cbef385a --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/bound_socket.rs @@ -0,0 +1,46 @@ +use std::{ + fmt, io, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, +}; + +use tokio::net::{TcpSocket, TcpStream}; +use tracing::Level; + +/// A TCP socket that is already bound. +/// +/// Provides a nicer [`fmt::Debug`] implementation than [`TcpSocket`]. +pub struct BoundTcpSocket(TcpSocket); + +impl BoundTcpSocket { + /// Opens a new TCP socket and binds it to the given IP address and a random port. + /// If the given IP address is not specified, binds the socket to localhost instead. + #[tracing::instrument(level = Level::TRACE, ret, err)] + pub fn bind_specified_or_localhost(ip: IpAddr) -> io::Result { + let (socket, ip) = match ip { + IpAddr::V4(Ipv4Addr::UNSPECIFIED) => (TcpSocket::new_v4()?, Ipv4Addr::LOCALHOST.into()), + IpAddr::V6(Ipv6Addr::UNSPECIFIED) => (TcpSocket::new_v6()?, Ipv6Addr::LOCALHOST.into()), + addr @ IpAddr::V4(..) => (TcpSocket::new_v4()?, addr), + addr @ IpAddr::V6(..) => (TcpSocket::new_v6()?, addr), + }; + + socket.bind(SocketAddr::new(ip, 0))?; + + Ok(Self(socket)) + } + + /// Returns the address to which this socket is bound. + pub fn local_addr(&self) -> io::Result { + self.0.local_addr() + } + + /// Makes a connection to the given peer. + pub async fn connect(self, peer: SocketAddr) -> io::Result { + self.0.connect(peer).await + } +} + +impl fmt::Debug for BoundTcpSocket { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.local_addr().fmt(f) + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/http.rs b/mirrord/intproxy/src/proxies/incoming/http.rs index d122aab53c5..a871cebc2c5 100644 --- a/mirrord/intproxy/src/proxies/incoming/http.rs +++ b/mirrord/intproxy/src/proxies/incoming/http.rs @@ -1,86 +1,257 @@ -use std::convert::Infallible; +use std::{fmt, io, net::SocketAddr, ops::Not}; -use bytes::Bytes; -use http_body_util::combinators::BoxBody; use hyper::{ body::Incoming, client::conn::{http1, http2}, - Response, Version, + Request, Response, StatusCode, Version, }; use hyper_util::rt::{TokioExecutor, TokioIo}; -use mirrord_protocol::tcp::HttpRequestFallback; +use mirrord_protocol::{ + tcp::{HttpRequest, HttpResponse, InternalHttpResponse}, + ConnectionId, Port, RequestId, +}; +use thiserror::Error; use tokio::net::TcpStream; use tracing::Level; -use super::interceptor::{InterceptorError, InterceptorResult}; +mod client_store; +mod response_mode; +mod streaming_body; -pub(super) const RETRY_ON_RESET_ATTEMPTS: u32 = 10; +pub use client_store::ClientStore; +pub use response_mode::ResponseMode; +pub use streaming_body::StreamingBody; -/// Handles the differences between hyper's HTTP/1 and HTTP/2 connections. -pub enum HttpSender { - V1(http1::SendRequest>), - V2(http2::SendRequest>), +/// An HTTP client used to pass requests to the user application. +pub struct LocalHttpClient { + /// Established HTTP connection with the user application. + sender: HttpSender, + /// Address of the user application's HTTP server. + local_server_address: SocketAddr, + /// Address of this client's TCP socket. + address: SocketAddr, } -/// Consumes the given [`TcpStream`] and performs an HTTP handshake, turning it into an HTTP -/// connection. -/// -/// # Returns -/// -/// [`HttpSender`] that can be used to send HTTP requests to the peer. -#[tracing::instrument(level = Level::TRACE, skip(target_stream), err(level = Level::WARN))] -pub async fn handshake( - version: Version, - target_stream: TcpStream, -) -> InterceptorResult { - match version { - Version::HTTP_2 => { - let (sender, connection) = - http2::handshake(TokioExecutor::default(), TokioIo::new(target_stream)).await?; - tokio::spawn(connection); - - Ok(HttpSender::V2(sender)) +impl LocalHttpClient { + /// Makes an HTTP connection with the given server and creates a new client. + #[tracing::instrument(level = Level::TRACE, err(level = Level::WARN), ret)] + pub async fn new( + local_server_address: SocketAddr, + version: Version, + ) -> Result { + let stream = TcpStream::connect(local_server_address) + .await + .map_err(LocalHttpError::ConnectTcpFailed)?; + let local_server_address = stream + .peer_addr() + .map_err(LocalHttpError::SocketSetupFailed)?; + let address = stream + .local_addr() + .map_err(LocalHttpError::SocketSetupFailed)?; + let sender = HttpSender::handshake(version, stream).await?; + + Ok(Self { + sender, + local_server_address, + address, + }) + } + + /// Send the given `request` to the user application's HTTP server. + #[tracing::instrument(level = Level::DEBUG, err(level = Level::WARN), ret)] + pub async fn send_request( + &mut self, + request: HttpRequest, + ) -> Result, LocalHttpError> { + self.sender.send_request(request).await + } + + /// Returns the address of the local server to which this client is connected. + pub fn local_server_address(&self) -> SocketAddr { + self.local_server_address + } + + pub fn handles_version(&self, version: Version) -> bool { + match (&self.sender, version) { + (_, Version::HTTP_3) => false, + (HttpSender::V2(..), Version::HTTP_2) => true, + (HttpSender::V1(..), _) => true, + (HttpSender::V2(..), _) => false, } + } +} - Version::HTTP_3 => Err(InterceptorError::UnsupportedHttpVersion(version)), +impl fmt::Debug for LocalHttpClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LocalHttpClient") + .field("local_server_address", &self.local_server_address) + .field("address", &self.address) + .field("is_http_1", &matches!(self.sender, HttpSender::V1(..))) + .finish() + } +} + +/// Errors that can occur when sending an HTTP request to the user application. +#[derive(Error, Debug)] +pub enum LocalHttpError { + #[error("failed to make an HTTP handshake with the local application's HTTP server: {0}")] + HandshakeFailed(#[source] hyper::Error), - _http_v1 => { - let (sender, connection) = http1::handshake(TokioIo::new(target_stream)).await?; + #[error("{0:?} is not supported in the local HTTP proxy")] + UnsupportedHttpVersion(Version), - tokio::spawn(connection.with_upgrades()); + #[error("failed to send the request to the local application's HTTP server: {0}")] + SendFailed(#[source] hyper::Error), - Ok(HttpSender::V1(sender)) + #[error("failed to prepare a local TCP socket: {0}")] + SocketSetupFailed(#[source] io::Error), + + #[error("failed to make a TCP connection with the local application's HTTP server: {0}")] + ConnectTcpFailed(#[source] io::Error), + + #[error("failed to read the body of the local application's HTTP server response: {0}")] + ReadBodyFailed(#[source] hyper::Error), +} + +impl LocalHttpError { + /// Checks if we can retry sending the request, given that the previous attempt resulted in this + /// error. + pub fn can_retry(&self) -> bool { + match self { + Self::SocketSetupFailed(..) | Self::UnsupportedHttpVersion(..) => false, + Self::ConnectTcpFailed(..) => true, + Self::HandshakeFailed(err) | Self::SendFailed(err) | Self::ReadBodyFailed(err) => (err + .is_parse() + || err.is_parse_status() + || err.is_parse_too_large() + || err.is_user()) + .not(), } } } +/// Produces a mirrord-specific [`StatusCode::BAD_GATEWAY`] response. +pub fn mirrord_error_response( + message: M, + version: Version, + connection_id: ConnectionId, + request_id: RequestId, + port: Port, +) -> HttpResponse> { + HttpResponse { + connection_id, + port, + request_id, + internal_response: InternalHttpResponse { + status: StatusCode::BAD_GATEWAY, + version, + headers: Default::default(), + body: format!("mirrord: {message}\n").into_bytes(), + }, + } +} + +/// Holds either [`http1::SendRequest`] or [`http2::SendRequest`] and exposes a unified interface. +enum HttpSender { + V1(http1::SendRequest), + V2(http2::SendRequest), +} + impl HttpSender { - #[tracing::instrument(level = Level::TRACE, skip(self), err(level = Level::WARN))] - pub async fn send( + /// Performs an HTTP handshake over the given [`TcpStream`]. + async fn handshake(version: Version, target_stream: TcpStream) -> Result { + let local_addr = target_stream + .local_addr() + .map_err(LocalHttpError::SocketSetupFailed)?; + let peer_addr = target_stream + .peer_addr() + .map_err(LocalHttpError::SocketSetupFailed)?; + + match version { + Version::HTTP_2 => { + let (sender, connection) = + http2::handshake(TokioExecutor::default(), TokioIo::new(target_stream)) + .await + .map_err(LocalHttpError::HandshakeFailed)?; + + tokio::spawn(async move { + match connection.await { + Ok(()) => { + tracing::trace!(%local_addr, %peer_addr, "HTTP connection with the local application finished"); + } + Err(error) => { + tracing::warn!(%error, %local_addr, %peer_addr, "HTTP connection with the local application failed"); + } + } + }); + + Ok(HttpSender::V2(sender)) + } + + Version::HTTP_3 => Err(LocalHttpError::UnsupportedHttpVersion(version)), + + _http_v1 => { + let (sender, connection) = http1::handshake(TokioIo::new(target_stream)) + .await + .map_err(LocalHttpError::HandshakeFailed)?; + + tokio::spawn(async move { + match connection.with_upgrades().await { + Ok(()) => { + tracing::trace!(%local_addr, %peer_addr, "HTTP connection with the local application finished"); + } + Err(error) => { + tracing::warn!(%error, %local_addr, %peer_addr, "HTTP connection with the local application failed"); + } + } + }); + + Ok(HttpSender::V1(sender)) + } + } + } + + /// Tries to send the given [`HttpRequest`] to the server. + async fn send_request( &mut self, - req: HttpRequestFallback, - ) -> InterceptorResult, InterceptorError> { + request: HttpRequest, + ) -> Result, LocalHttpError> { match self { Self::V1(sender) => { // Solves a "connection was not ready" client error. // https://rust-lang.github.io/wg-async/vision/submitted_stories/status_quo/barbara_tries_unix_socket.html#the-single-magical-line - sender.ready().await?; + sender.ready().await.map_err(LocalHttpError::SendFailed)?; + sender - .send_request(req.into_hyper()) + .send_request(request.internal_request.into()) .await - .map_err(Into::into) + .map_err(LocalHttpError::SendFailed) } Self::V2(sender) => { - let mut req = req.into_hyper(); + let mut hyper_request: Request<_> = request.internal_request.into(); + // fixes https://github.com/metalbear-co/mirrord/issues/2497 // inspired by https://github.com/linkerd/linkerd2-proxy/blob/c5d9f1c1e7b7dddd9d75c0d1a0dca68188f38f34/linkerd/proxy/http/src/h2.rs#L175 - if req.uri().authority().is_none() { - *req.version_mut() = hyper::http::Version::HTTP_11; + if hyper_request.uri().authority().is_none() + && hyper_request.version() != Version::HTTP_11 + { + tracing::trace!( + original_version = ?hyper_request.version(), + "Request URI has no authority, changing HTTP version to {:?}", + Version::HTTP_11, + ); + + *hyper_request.version_mut() = Version::HTTP_11; } + // Solves a "connection was not ready" client error. // https://rust-lang.github.io/wg-async/vision/submitted_stories/status_quo/barbara_tries_unix_socket.html#the-single-magical-line - sender.ready().await?; - sender.send_request(req).await.map_err(Into::into) + sender.ready().await.map_err(LocalHttpError::SendFailed)?; + + sender + .send_request(hyper_request) + .await + .map_err(LocalHttpError::SendFailed) } } } diff --git a/mirrord/intproxy/src/proxies/incoming/http/client_store.rs b/mirrord/intproxy/src/proxies/incoming/http/client_store.rs new file mode 100644 index 00000000000..69ee3ce512c --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/http/client_store.rs @@ -0,0 +1,232 @@ +use std::{ + cmp, fmt, + net::SocketAddr, + sync::{Arc, Mutex}, + time::Duration, +}; + +use hyper::Version; +use tokio::{ + sync::Notify, + time::{self, Instant}, +}; +use tracing::Level; + +use super::{LocalHttpClient, LocalHttpError}; + +/// Idle [`LocalHttpClient`] caches in [`ClientStore`]. +struct IdleLocalClient { + client: LocalHttpClient, + last_used: Instant, +} + +impl fmt::Debug for IdleLocalClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("IdleLocalClient") + .field("client", &self.client) + .field("idle_for_s", &self.last_used.elapsed().as_secs_f32()) + .finish() + } +} + +/// Cache for unused [`LocalHttpClient`]s. +/// +/// [`LocalHttpClient`] that have not been used for some time are dropped in the background by a +/// dedicated [`tokio::task`]. This timeout defaults to [`Self::IDLE_CLIENT_DEFAULT_TIMEOUT`]. +#[derive(Clone)] +pub struct ClientStore { + clients: Arc>>, + /// Used to notify other tasks when there is a new client in the store. + /// + /// Make sure to only call [`Notify::notify_waiters`] and [`Notify::notified`] when holding a + /// lock on [`Self::clients`]. Otherwise you'll have a race condition. + notify: Arc, +} + +impl Default for ClientStore { + fn default() -> Self { + Self::new_with_timeout(Self::IDLE_CLIENT_DEFAULT_TIMEOUT) + } +} + +impl ClientStore { + pub const IDLE_CLIENT_DEFAULT_TIMEOUT: Duration = Duration::from_secs(3); + + /// Creates a new store. + /// + /// The store will keep unused clients alive for at least the given time. + pub fn new_with_timeout(timeout: Duration) -> Self { + let store = Self { + clients: Default::default(), + notify: Default::default(), + }; + + tokio::spawn(cleanup_task(store.clone(), timeout)); + + store + } + + /// Reuses or creates a new [`LocalHttpClient`]. + #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::WARN))] + pub async fn get( + &self, + server_addr: SocketAddr, + version: Version, + ) -> Result { + let ready = { + let mut guard = self + .clients + .lock() + .expect("ClientStore mutex is poisoned, this is a bug"); + let position = guard.iter().position(|idle| { + idle.client.handles_version(version) + && idle.client.local_server_address() == server_addr + }); + position.map(|position| guard.swap_remove(position)) + }; + + if let Some(ready) = ready { + tracing::trace!(?ready, "Reused an idle client"); + return Ok(ready.client); + } + + let connect_task = tokio::spawn(LocalHttpClient::new(server_addr, version)); + + tokio::select! { + result = connect_task => result.expect("this task should not panic"), + ready = self.wait_for_ready(server_addr, version) => { + tracing::trace!(?ready, "Reused an idle client"); + Ok(ready) + }, + } + } + + /// Stores an unused [`LocalHttpClient`], so that it can be reused later. + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub fn push_idle(&self, client: LocalHttpClient) { + let mut guard = self + .clients + .lock() + .expect("ClientStore mutex is poisoned, this is a bug"); + guard.push(IdleLocalClient { + client, + last_used: Instant::now(), + }); + self.notify.notify_waiters(); + } + + /// Waits until there is a ready unused client. + async fn wait_for_ready(&self, server_addr: SocketAddr, version: Version) -> LocalHttpClient { + loop { + let notified = { + let mut guard = self + .clients + .lock() + .expect("ClientStore mutex is poisoned, this is a bug"); + let position = guard.iter().position(|idle| { + idle.client.handles_version(version) + && idle.client.local_server_address() == server_addr + }); + + match position { + Some(position) => return guard.swap_remove(position).client, + None => self.notify.notified(), + } + }; + + notified.await; + } + } +} + +/// Cleans up stale [`LocalHttpClient`]s from the [`ClientStore`]. +async fn cleanup_task(store: ClientStore, idle_client_timeout: Duration) { + let clients = Arc::downgrade(&store.clients); + let notify = store.notify.clone(); + std::mem::drop(store); + + loop { + let Some(clients) = clients.upgrade() else { + // Failed `upgrade` means that all `ClientStore` instances were dropped. + // This task is no longer needed. + break; + }; + + let now = Instant::now(); + let mut min_last_used = None; + let notified = { + let Ok(mut guard) = clients.lock() else { + tracing::error!("ClientStore mutex is poisoned, this is a bug"); + return; + }; + + guard.retain(|client| { + if client.last_used + idle_client_timeout > now { + // We determine how long to sleep before cleaning the store again. + min_last_used = min_last_used + .map(|previous| cmp::min(previous, client.last_used)) + .or(Some(client.last_used)); + + true + } else { + // We drop the idle clients that have gone beyond the timeout. + tracing::trace!(?client, "Dropping an idle client"); + false + } + }); + + // Acquire [`Notified`] while still holding the lock. + // Prevents missed updates. + notify.notified() + }; + + if let Some(min_last_used) = min_last_used { + time::sleep_until(min_last_used + idle_client_timeout).await; + } else { + notified.await; + } + } +} + +#[cfg(test)] +mod test { + use std::{convert::Infallible, time::Duration}; + + use bytes::Bytes; + use http_body_util::Empty; + use hyper::{ + body::Incoming, server::conn::http1, service::service_fn, Request, Response, Version, + }; + use hyper_util::rt::TokioIo; + use tokio::{net::TcpListener, time}; + + use super::ClientStore; + + /// Verifies that [`ClientStore`] cleans up unused connections. + #[tokio::test] + async fn cleans_up_unused_connections() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + tokio::spawn(async move { + let service = service_fn(|_req: Request| { + std::future::ready(Ok::<_, Infallible>(Response::new(Empty::::new()))) + }); + + let (connection, _) = listener.accept().await.unwrap(); + std::mem::drop(listener); + http1::Builder::new() + .serve_connection(TokioIo::new(connection), service) + .await + .unwrap() + }); + + let client_store = ClientStore::new_with_timeout(Duration::from_millis(10)); + let client = client_store.get(addr, Version::HTTP_11).await.unwrap(); + client_store.push_idle(client); + + time::sleep(Duration::from_millis(100)).await; + + assert!(client_store.clients.lock().unwrap().is_empty()); + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/http/response_mode.rs b/mirrord/intproxy/src/proxies/incoming/http/response_mode.rs new file mode 100644 index 00000000000..c6f4eb5a583 --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/http/response_mode.rs @@ -0,0 +1,31 @@ +use mirrord_protocol::tcp::{HTTP_CHUNKED_RESPONSE_VERSION, HTTP_FRAMED_VERSION}; + +/// Determines how [`IncomingProxy`](crate::proxies::incoming::IncomingProxy) should send HTTP +/// responses. +#[derive(Debug, Clone, Copy, Default)] +pub enum ResponseMode { + /// Agent supports + /// [`LayerTcpSteal::HttpResponseChunked`](mirrord_protocol::tcp::LayerTcpSteal::HttpResponseChunked) + /// and the previous variants. + Chunked, + /// Agent supports + /// [`LayerTcpSteal::HttpResponseFramed`](mirrord_protocol::tcp::LayerTcpSteal::HttpResponseFramed) + /// and the previous variant. + Framed, + /// Agent supports only + /// [`LayerTcpSteal::HttpResponse`](mirrord_protocol::tcp::LayerTcpSteal::HttpResponse) + #[default] + Basic, +} + +impl From<&semver::Version> for ResponseMode { + fn from(value: &semver::Version) -> Self { + if HTTP_CHUNKED_RESPONSE_VERSION.matches(value) { + Self::Chunked + } else if HTTP_FRAMED_VERSION.matches(value) { + Self::Framed + } else { + Self::Basic + } + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/http/streaming_body.rs b/mirrord/intproxy/src/proxies/incoming/http/streaming_body.rs new file mode 100644 index 00000000000..06a614f1ba4 --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/http/streaming_body.rs @@ -0,0 +1,140 @@ +use std::{ + convert::Infallible, + fmt, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll}, +}; + +use bytes::Bytes; +use hyper::body::{Body, Frame}; +use mirrord_protocol::tcp::{InternalHttpBody, InternalHttpBodyFrame}; +use tokio::sync::mpsc::{self, Receiver}; + +/// Cheaply cloneable [`Body`] implementation that reads [`Frame`]s from an [`mpsc::channel`]. +/// +/// # Clone behavior +/// +/// All instances acquired via [`Clone`] share the [`mpsc::Receiver`] and a vector of previously +/// read frames. Each instance maintains its own position in the shared vector, and a new clone +/// starts at 0. +/// +/// When polled with [`Body::poll_frame`], an instance tries to return a cached frame. +/// +/// Thanks to this, each clone returns all frames from the start when polled with +/// [`Body::poll_frame`]. As you'd expect from a cloneable [`Body`] implementation. +pub struct StreamingBody { + /// Shared with instances acquired via [`Clone`]. + /// + /// Allows the clones to access previously fetched [`Frame`]s. + shared_state: Arc, Vec)>>, + /// Index of the next frame to return from the buffer, not shared with other instances acquired + /// via [`Clone`]. + /// + /// If outside of the buffer, we need to poll the stream to get the next frame. + idx: usize, +} + +impl StreamingBody { + /// Creates a new instance of this [`Body`]. + /// + /// It will first read all frames from the vector given as `first_frames`. + /// Following frames will be fetched from the given `rx`. + pub fn new( + rx: Receiver, + first_frames: Vec, + ) -> Self { + Self { + shared_state: Arc::new(Mutex::new((rx, first_frames))), + idx: 0, + } + } +} + +impl Clone for StreamingBody { + fn clone(&self) -> Self { + Self { + shared_state: self.shared_state.clone(), + // Setting idx to 0 in order to replay the previous frames. + idx: 0, + } + } +} + +impl Body for StreamingBody { + type Data = Bytes; + + type Error = Infallible; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + let mut guard = this.shared_state.lock().unwrap(); + + if let Some(frame) = guard.1.get(this.idx) { + this.idx += 1; + return Poll::Ready(Some(Ok(frame.clone().into()))); + } + + match std::task::ready!(guard.0.poll_recv(cx)) { + None => Poll::Ready(None), + Some(frame) => { + guard.1.push(frame.clone()); + this.idx += 1; + Poll::Ready(Some(Ok(frame.into()))) + } + } + } +} + +impl Default for StreamingBody { + fn default() -> Self { + let (_, dummy_rx) = mpsc::channel(1); // `mpsc::channel` panics on capacity 0 + Self { + shared_state: Arc::new(Mutex::new((dummy_rx, Default::default()))), + idx: 0, + } + } +} + +impl From> for StreamingBody { + fn from(value: Vec) -> Self { + let (_, dummy_rx) = mpsc::channel(1); // `mpsc::channel` panics on capacity 0 + let frames = vec![InternalHttpBodyFrame::Data(value)]; + Self::new(dummy_rx, frames) + } +} + +impl From for StreamingBody { + fn from(value: InternalHttpBody) -> Self { + let (_, dummy_rx) = mpsc::channel(1); // `mpsc::channel` panics on capacity 0 + Self::new(dummy_rx, value.0.into_iter().collect()) + } +} + +impl From> for StreamingBody { + fn from(value: Receiver) -> Self { + Self::new(value, Default::default()) + } +} + +impl fmt::Debug for StreamingBody { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut s = f.debug_struct("StreamingBody"); + s.field("idx", &self.idx); + + match self.shared_state.try_lock() { + Ok(guard) => { + s.field("frame_rx_closed", &guard.0.is_closed()); + s.field("cached_frames", &guard.1); + } + Err(error) => { + s.field("lock_error", &error); + } + } + + s.finish() + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/http_gateway.rs b/mirrord/intproxy/src/proxies/incoming/http_gateway.rs new file mode 100644 index 00000000000..e7f8a7819b0 --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/http_gateway.rs @@ -0,0 +1,943 @@ +use std::{ + collections::VecDeque, + convert::Infallible, + error::Error, + fmt, + net::SocketAddr, + ops::ControlFlow, + time::{Duration, Instant}, +}; + +use exponential_backoff::Backoff; +use http_body_util::BodyExt; +use hyper::{body::Incoming, http::response::Parts, StatusCode}; +use mirrord_protocol::{ + batched_body::BatchedBody, + tcp::{ + ChunkedHttpBody, ChunkedHttpError, ChunkedResponse, HttpRequest, HttpResponse, + InternalHttpBody, InternalHttpBodyFrame, InternalHttpResponse, + }, +}; +use tokio::time; +use tracing::Level; + +use super::{ + http::{mirrord_error_response, ClientStore, LocalHttpError, ResponseMode, StreamingBody}, + tasks::{HttpOut, InProxyTaskMessage}, +}; +use crate::background_tasks::{BackgroundTask, MessageBus}; + +/// [`BackgroundTask`] used by the [`IncomingProxy`](super::IncomingProxy). +/// +/// Responsible for delivering a single HTTP request to the user application. +/// +/// Exits immediately when it's [`TaskSender`](crate::background_tasks::TaskSender) is dropped. +pub struct HttpGatewayTask { + /// Request to deliver. + request: HttpRequest, + /// Shared cache of [`LocalHttpClient`](super::http::LocalHttpClient)s. + client_store: ClientStore, + /// Determines response variant. + response_mode: ResponseMode, + /// Address of the HTTP server in the user application. + server_addr: SocketAddr, +} + +impl fmt::Debug for HttpGatewayTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("HttpGatewayTask") + .field("request", &self.request) + .field("response_mode", &self.response_mode) + .field("server_addr", &self.server_addr) + .finish() + } +} + +impl HttpGatewayTask { + /// Creates a new gateway task. + pub fn new( + request: HttpRequest, + client_store: ClientStore, + response_mode: ResponseMode, + server_addr: SocketAddr, + ) -> Self { + Self { + request, + client_store, + response_mode, + server_addr, + } + } + + /// Handles the response if we operate in [`ResponseMode::Chunked`]. + /// + /// # Returns + /// + /// * An error if we failed before sending the [`ChunkedResponse::Start`] message through the + /// [`MessageBus`] (we can still retry the request) + /// * [`ControlFlow::Break`] if we failed after sending the [`ChunkedResponse::Start`] message + /// * [`ControlFlow::Continue`] if we succeeded + async fn handle_response_chunked( + &self, + parts: Parts, + mut body: Incoming, + message_bus: &mut MessageBus, + ) -> Result, LocalHttpError> { + let frames = body + .ready_frames() + .map_err(LocalHttpError::ReadBodyFailed)?; + + if frames.is_last { + let ready_frames = frames + .frames + .into_iter() + .map(InternalHttpBodyFrame::from) + .collect::>(); + + tracing::trace!( + ?ready_frames, + "All response body frames were instantly ready, sending full response" + ); + let response = HttpResponse { + port: self.request.port, + connection_id: self.request.connection_id, + request_id: self.request.request_id, + internal_response: InternalHttpResponse { + status: parts.status, + version: parts.version, + headers: parts.headers, + body: InternalHttpBody(ready_frames), + }, + }; + message_bus.send(HttpOut::ResponseFramed(response)).await; + + return Ok(ControlFlow::Continue(())); + } + + let ready_frames = frames + .frames + .into_iter() + .map(InternalHttpBodyFrame::from) + .collect::>(); + tracing::trace!( + ?ready_frames, + "Some response body frames were instantly ready, \ + but response body may not be finished yet" + ); + + let response = HttpResponse { + port: self.request.port, + connection_id: self.request.connection_id, + request_id: self.request.request_id, + internal_response: InternalHttpResponse { + status: parts.status, + version: parts.version, + headers: parts.headers, + body: ready_frames, + }, + }; + message_bus + .send(HttpOut::ResponseChunked(ChunkedResponse::Start(response))) + .await; + + loop { + let start = Instant::now(); + match body.next_frames().await { + Ok(frames) => { + let is_last = frames.is_last; + let frames = frames + .frames + .into_iter() + .map(InternalHttpBodyFrame::from) + .collect::>(); + tracing::trace!( + ?frames, + is_last, + elapsed_ms = start.elapsed().as_millis(), + "Received a next batch of response body frames", + ); + + message_bus + .send(HttpOut::ResponseChunked(ChunkedResponse::Body( + ChunkedHttpBody { + frames, + is_last, + connection_id: self.request.connection_id, + request_id: self.request.request_id, + }, + ))) + .await; + + if is_last { + break; + } + } + + // Do not return any error here, as it would later be transformed into an error + // response. We already send the request head to the agent. + Err(error) => { + tracing::warn!( + error = ?ErrorWithSources(&error), + elapsed_ms = start.elapsed().as_millis(), + gateway = ?self, + "Failed to read next response body frames", + ); + + message_bus + .send(HttpOut::ResponseChunked(ChunkedResponse::Error( + ChunkedHttpError { + connection_id: self.request.connection_id, + request_id: self.request.request_id, + }, + ))) + .await; + + return Ok(ControlFlow::Break(())); + } + } + } + + Ok(ControlFlow::Continue(())) + } + + /// Makes an attempt to send the request and read the whole response. + /// + /// [`Err`] is handled in the caller and, if we run out of send attempts, converted to an error + /// response. Because of this, this function should not return any error that happened after + /// sending [`ChunkedResponse::Start`]. The agent would get a duplicated response. + #[tracing::instrument(level = Level::TRACE, skip_all, err(level = Level::WARN))] + async fn send_attempt(&self, message_bus: &mut MessageBus) -> Result<(), LocalHttpError> { + let mut client = self + .client_store + .get(self.server_addr, self.request.version()) + .await?; + let mut response = client.send_request(self.request.clone()).await?; + let on_upgrade = (response.status() == StatusCode::SWITCHING_PROTOCOLS).then(|| { + tracing::trace!("Detected an HTTP upgrade"); + hyper::upgrade::on(&mut response) + }); + let (parts, body) = response.into_parts(); + + let flow = match self.response_mode { + ResponseMode::Basic => { + let start = Instant::now(); + let body: Vec = body + .collect() + .await + .map_err(LocalHttpError::ReadBodyFailed)? + .to_bytes() + .into(); + tracing::trace!( + body_len = body.len(), + elapsed_ms = start.elapsed().as_millis(), + "Collected the whole response body", + ); + + let response = HttpResponse { + port: self.request.port, + connection_id: self.request.connection_id, + request_id: self.request.request_id, + internal_response: InternalHttpResponse { + status: parts.status, + version: parts.version, + headers: parts.headers, + body, + }, + }; + message_bus.send(HttpOut::ResponseBasic(response)).await; + + ControlFlow::Continue(()) + } + ResponseMode::Framed => { + let start = Instant::now(); + let body = InternalHttpBody::from_body(body) + .await + .map_err(LocalHttpError::ReadBodyFailed)?; + tracing::trace!( + ?body, + elapsed_ms = start.elapsed().as_millis(), + "Collected the whole response body", + ); + + let response = HttpResponse { + port: self.request.port, + connection_id: self.request.connection_id, + request_id: self.request.request_id, + internal_response: InternalHttpResponse { + status: parts.status, + version: parts.version, + headers: parts.headers, + body, + }, + }; + message_bus.send(HttpOut::ResponseFramed(response)).await; + + ControlFlow::Continue(()) + } + ResponseMode::Chunked => { + self.handle_response_chunked(parts, body, message_bus) + .await? + } + }; + + if flow.is_break() { + return Ok(()); + } + + if let Some(on_upgrade) = on_upgrade { + message_bus.send(HttpOut::Upgraded(on_upgrade)).await; + } else { + // If there was no upgrade and no error, the client can be reused. + self.client_store.push_idle(client); + } + + Ok(()) + } +} + +impl BackgroundTask for HttpGatewayTask { + type Error = Infallible; + type MessageIn = Infallible; + type MessageOut = InProxyTaskMessage; + + #[tracing::instrument(level = Level::TRACE, name = "http_gateway_task_main_loop", skip(message_bus))] + async fn run(self, message_bus: &mut MessageBus) -> Result<(), Self::Error> { + let mut backoffs = + Backoff::new(10, Duration::from_millis(50), Duration::from_millis(500)).into_iter(); + let guard = message_bus.closed(); + + let mut attempt = 0; + let error = loop { + attempt += 1; + tracing::trace!(attempt, "Starting send attempt"); + match guard.cancel_on_close(self.send_attempt(message_bus)).await { + None | Some(Ok(())) => return Ok(()), + Some(Err(error)) => { + let backoff = error + .can_retry() + .then(|| backoffs.next()) + .flatten() + .flatten(); + let Some(backoff) = backoff else { + tracing::warn!( + gateway = ?self, + failed_attempts = attempt, + error = ?ErrorWithSources(&error), + "Failed to send an HTTP request", + ); + + break error; + }; + + tracing::trace!( + backoff_ms = backoff.as_millis(), + failed_attempts = attempt, + error = ?ErrorWithSources(&error), + "Trying again after backoff", + ); + + if guard.cancel_on_close(time::sleep(backoff)).await.is_none() { + return Ok(()); + } + } + } + }; + + let response = mirrord_error_response( + error, + self.request.version(), + self.request.connection_id, + self.request.request_id, + self.request.port, + ); + message_bus.send(HttpOut::ResponseBasic(response)).await; + + Ok(()) + } +} + +/// Helper struct for tracing an [`Error`] along with all its sources, +/// down to the root cause. +/// +/// Might help when inspecting [`hyper`] errors. +struct ErrorWithSources<'a>(&'a dyn Error); + +impl fmt::Debug for ErrorWithSources<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut list = f.debug_list(); + list.entry(&self.0); + + let mut source = self.0.source(); + while let Some(error) = source { + list.entry(&error); + source = error.source(); + } + + list.finish() + } +} + +#[cfg(test)] +mod test { + use std::{io, sync::Arc}; + + use bytes::Bytes; + use http_body_util::{Empty, StreamBody}; + use hyper::{ + body::{Frame, Incoming}, + header::{self, HeaderValue, CONNECTION, UPGRADE}, + server::conn::http1, + service::service_fn, + upgrade::Upgraded, + Method, Request, Response, StatusCode, Version, + }; + use hyper_util::rt::TokioIo; + use mirrord_protocol::tcp::{HttpRequest, InternalHttpRequest}; + use rstest::rstest; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + sync::{mpsc, watch, Semaphore}, + task, + }; + use tokio_stream::wrappers::ReceiverStream; + + use super::*; + use crate::{ + background_tasks::{BackgroundTasks, TaskUpdate}, + proxies::incoming::{ + tcp_proxy::{LocalTcpConnection, TcpProxyTask}, + InProxyTaskError, + }, + }; + + /// Binary protocol over TCP. + /// Server first sends bytes [`INITIAL_MESSAGE`], then echoes back all received data. + const TEST_PROTO: &str = "dummyecho"; + + const INITIAL_MESSAGE: &[u8] = &[0x4a, 0x50, 0x32, 0x47, 0x4d, 0x44]; + + /// Handles requests upgrading to the [`TEST_PROTO`] protocol. + async fn upgrade_req_handler( + mut req: Request, + ) -> hyper::Result>> { + async fn dummy_echo(upgraded: Upgraded) -> io::Result<()> { + let mut upgraded = TokioIo::new(upgraded); + let mut buf = [0_u8; 64]; + + upgraded.write_all(INITIAL_MESSAGE).await?; + + loop { + let bytes_read = upgraded.read(&mut buf[..]).await?; + if bytes_read == 0 { + break; + } + + let echo_back = buf.get(0..bytes_read).unwrap(); + upgraded.write_all(echo_back).await?; + } + + Ok(()) + } + + let mut res = Response::new(Empty::new()); + + let contains_expected_upgrade = req + .headers() + .get(UPGRADE) + .filter(|proto| *proto == TEST_PROTO) + .is_some(); + if !contains_expected_upgrade { + *res.status_mut() = StatusCode::BAD_REQUEST; + return Ok(res); + } + + task::spawn(async move { + match hyper::upgrade::on(&mut req).await { + Ok(upgraded) => { + if let Err(e) = dummy_echo(upgraded).await { + eprintln!("server foobar io error: {}", e) + }; + } + Err(e) => eprintln!("upgrade error: {}", e), + } + }); + + *res.status_mut() = StatusCode::SWITCHING_PROTOCOLS; + res.headers_mut() + .insert(UPGRADE, HeaderValue::from_static(TEST_PROTO)); + res.headers_mut() + .insert(CONNECTION, HeaderValue::from_static("upgrade")); + Ok(res) + } + + /// Runs a [`hyper`] server that accepts only requests upgrading to the [`TEST_PROTO`] protocol. + async fn dummy_echo_server(listener: TcpListener, mut shutdown: watch::Receiver) { + loop { + tokio::select! { + res = listener.accept() => { + let (stream, _) = res.expect("dummy echo server failed to accept connection"); + + let mut shutdown = shutdown.clone(); + + task::spawn(async move { + let conn = http1::Builder::new().serve_connection(TokioIo::new(stream), service_fn(upgrade_req_handler)); + let mut conn = conn.with_upgrades(); + let mut conn = Pin::new(&mut conn); + + tokio::select! { + res = &mut conn => { + res.expect("dummy echo server failed to serve connection"); + } + + _ = shutdown.changed() => { + conn.graceful_shutdown(); + } + } + }); + } + + _ = shutdown.changed() => break, + } + } + } + + /// Verifies that [`HttpGatewayTask`] and [`TcpProxyTask`] together correctly handle HTTP + /// upgrades. + #[tokio::test] + async fn handles_http_upgrades() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_destination = listener.local_addr().unwrap(); + + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let server_task = task::spawn(dummy_echo_server(listener, shutdown_rx)); + + let mut tasks: BackgroundTasks = + Default::default(); + let _gateway = { + let request = HttpRequest { + connection_id: 0, + request_id: 0, + port: 80, + internal_request: InternalHttpRequest { + method: Method::GET, + uri: "dummyecho://www.mirrord.dev/".parse().unwrap(), + headers: [ + (CONNECTION, HeaderValue::from_static("upgrade")), + (UPGRADE, HeaderValue::from_static(TEST_PROTO)), + ] + .into_iter() + .collect(), + version: Version::HTTP_11, + body: Default::default(), + }, + }; + let gateway = HttpGatewayTask::new( + request, + Default::default(), + ResponseMode::Basic, + local_destination, + ); + tasks.register(gateway, 0, 8) + }; + + let message = tasks + .next() + .await + .expect("no task result") + .1 + .unwrap_message(); + match message { + InProxyTaskMessage::Http(HttpOut::ResponseBasic(res)) => { + assert_eq!( + res.internal_response.status, + StatusCode::SWITCHING_PROTOCOLS + ); + println!("Received response from the gateway: {res:?}"); + assert!(res + .internal_response + .headers + .get(CONNECTION) + .filter(|v| *v == "upgrade") + .is_some()); + assert!(res + .internal_response + .headers + .get(UPGRADE) + .filter(|v| *v == TEST_PROTO) + .is_some()); + } + other => panic!("unexpected task update: {other:?}"), + } + + let message = tasks + .next() + .await + .expect("not task result") + .1 + .unwrap_message(); + let on_upgrade = match message { + InProxyTaskMessage::Http(HttpOut::Upgraded(on_upgrade)) => on_upgrade, + other => panic!("unexpected task update: {other:?}"), + }; + let update = tasks.next().await.expect("no task result").1; + match update { + TaskUpdate::Finished(Ok(())) => {} + other => panic!("unexpected task update: {other:?}"), + } + + let proxy = tasks.register( + TcpProxyTask::new(LocalTcpConnection::AfterUpgrade(on_upgrade), false), + 1, + 8, + ); + + proxy.send(b"test test test".to_vec()).await; + + let message = tasks + .next() + .await + .expect("no task result") + .1 + .unwrap_message(); + match message { + InProxyTaskMessage::Tcp(bytes) => { + assert_eq!(bytes, INITIAL_MESSAGE); + } + _ => panic!("unexpected task update: {update:?}"), + } + + let message = tasks + .next() + .await + .expect("no task result") + .1 + .unwrap_message(); + match message { + InProxyTaskMessage::Tcp(bytes) => { + assert_eq!(bytes, b"test test test"); + } + _ => panic!("unexpected task update: {update:?}"), + } + + let _ = shutdown_tx.send(true); + server_task.await.expect("dummy echo server panicked"); + } + + /// Verifies that [`HttpGatewayTask`] produces correct variant of the [`HttpResponse`]. + /// + /// Verifies that body of + /// [`LayerTcpSteal::HttpResponseChunked`](mirrord_protocol::tcp::LayerTcpSteal::HttpResponseChunked) + /// is streamed. + #[rstest] + #[case::basic(ResponseMode::Basic)] + #[case::framed(ResponseMode::Framed)] + #[case::chunked(ResponseMode::Chunked)] + #[tokio::test] + async fn produces_correct_response_variant(#[case] response_mode: ResponseMode) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let semaphore: Arc = Arc::new(Semaphore::const_new(0)); + let semaphore_clone = semaphore.clone(); + + let conn_task = tokio::spawn(async move { + let service = service_fn(|_req: Request| { + let semaphore = semaphore_clone.clone(); + async move { + let (frame_tx, frame_rx) = mpsc::channel::>>(1); + + tokio::spawn(async move { + for _ in 0..2 { + semaphore.acquire().await.unwrap().forget(); + let _ = frame_tx + .send(Ok(Frame::data(Bytes::from_static(b"hello\n")))) + .await; + } + }); + + let body = StreamBody::new(ReceiverStream::new(frame_rx)); + let mut response = Response::new(body); + response + .headers_mut() + .insert(header::CONTENT_LENGTH, HeaderValue::from_static("12")); + + Ok::<_, Infallible>(response) + } + }); + + let (connection, _) = listener.accept().await.unwrap(); + http1::Builder::new() + .serve_connection(TokioIo::new(connection), service) + .await + .unwrap() + }); + + let request = HttpRequest { + connection_id: 0, + request_id: 0, + port: 80, + internal_request: InternalHttpRequest { + method: Method::GET, + uri: "/".parse().unwrap(), + headers: Default::default(), + version: Version::HTTP_11, + body: StreamingBody::from(vec![]), + }, + }; + + let mut tasks: BackgroundTasks<(), InProxyTaskMessage, Infallible> = Default::default(); + let _gateway = tasks.register( + HttpGatewayTask::new(request, ClientStore::default(), response_mode, addr), + (), + 8, + ); + + match response_mode { + ResponseMode::Basic => { + semaphore.add_permits(2); + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseBasic(response)) => { + assert_eq!(response.internal_response.body, b"hello\nhello\n"); + } + other => panic!("unexpected task message: {other:?}"), + } + } + + ResponseMode::Framed => { + semaphore.add_permits(2); + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseFramed(response)) => { + let mut collected = vec![]; + for frame in response.internal_response.body.0 { + match frame { + InternalHttpBodyFrame::Data(data) => collected.extend(data), + InternalHttpBodyFrame::Trailers(trailers) => { + panic!("unexpected trailing headers: {trailers:?}"); + } + } + } + + assert_eq!(collected, b"hello\nhello\n"); + } + other => panic!("unexpected task message: {other:?}"), + } + } + + ResponseMode::Chunked => { + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseChunked(ChunkedResponse::Start( + response, + ))) => { + assert!(response.internal_response.body.is_empty()); + } + other => panic!("unexpected task message: {other:?}"), + } + + semaphore.add_permits(1); + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseChunked(ChunkedResponse::Body( + body, + ))) => { + assert_eq!( + body.frames, + vec![InternalHttpBodyFrame::Data(b"hello\n".into())], + ); + assert!(!body.is_last); + } + other => panic!("unexpected task message: {other:?}"), + } + + semaphore.add_permits(1); + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseChunked(ChunkedResponse::Body( + body, + ))) => { + assert_eq!( + body.frames, + vec![InternalHttpBodyFrame::Data(b"hello\n".into())], + ); + assert!(body.is_last); + } + other => panic!("unexpected task message: {other:?}"), + } + } + } + + match tasks.next().await.unwrap().1 { + TaskUpdate::Finished(Ok(())) => {} + other => panic!("unexpected task update: {other:?}"), + } + + conn_task.await.unwrap(); + } + + /// Verifies that [`HttpGateway`] sends request body frames to the server as soon as they are + /// available. + #[tokio::test] + async fn streams_request_body_frames() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let semaphore: Arc = Arc::new(Semaphore::const_new(0)); + let semaphore_clone = semaphore.clone(); + + let conn_task = tokio::spawn(async move { + let service = service_fn(|mut req: Request| { + let semaphore = semaphore_clone.clone(); + async move { + for _ in 0..2 { + semaphore.add_permits(1); + let frame = req + .body_mut() + .frame() + .await + .unwrap() + .unwrap() + .into_data() + .unwrap(); + assert_eq!(frame, "hello\n"); + } + + Ok::<_, Infallible>(Response::new(Empty::::new())) + } + }); + + let (connection, _) = listener.accept().await.unwrap(); + http1::Builder::new() + .serve_connection(TokioIo::new(connection), service) + .await + .unwrap() + }); + + let (frame_tx, frame_rx) = mpsc::channel(1); + let body = StreamingBody::new(frame_rx, vec![]); + let mut request = HttpRequest { + connection_id: 0, + request_id: 0, + port: 80, + internal_request: InternalHttpRequest { + method: Method::GET, + uri: "/".parse().unwrap(), + headers: Default::default(), + version: Version::HTTP_11, + body, + }, + }; + request + .internal_request + .headers + .insert(header::CONTENT_LENGTH, HeaderValue::from_static("12")); + + let mut tasks: BackgroundTasks<(), InProxyTaskMessage, Infallible> = Default::default(); + let client_store = ClientStore::default(); + let _gateway = tasks.register( + HttpGatewayTask::new(request, client_store.clone(), ResponseMode::Basic, addr), + (), + 8, + ); + + for _ in 0..2 { + semaphore.acquire().await.unwrap().forget(); + frame_tx + .send(InternalHttpBodyFrame::Data(b"hello\n".into())) + .await + .unwrap(); + } + std::mem::drop(frame_tx); + + match tasks.next().await.unwrap().1.unwrap_message() { + InProxyTaskMessage::Http(HttpOut::ResponseBasic(response)) => { + assert_eq!(response.internal_response.status, StatusCode::OK); + } + other => panic!("unexpected message: {other:?}"), + } + + match tasks.next().await.unwrap().1 { + TaskUpdate::Finished(Ok(())) => {} + other => panic!("unexpected task update: {other:?}"), + } + + conn_task.await.unwrap(); + } + + /// Verifies that [`HttpGateway`] reuses already established HTTP connections. + #[tokio::test] + async fn reuses_client_connections() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + tokio::spawn(async move { + let service = service_fn(|_req: Request| { + std::future::ready(Ok::<_, Infallible>(Response::new(Empty::::new()))) + }); + + let (connection, _) = listener.accept().await.unwrap(); + std::mem::drop(listener); + http1::Builder::new() + .serve_connection(TokioIo::new(connection), service) + .await + .unwrap() + }); + + let mut request = HttpRequest { + connection_id: 0, + request_id: 0, + port: 80, + internal_request: InternalHttpRequest { + method: Method::GET, + uri: "/".parse().unwrap(), + headers: Default::default(), + version: Version::HTTP_11, + body: Default::default(), + }, + }; + request + .internal_request + .headers + .insert(header::CONNECTION, HeaderValue::from_static("keep-alive")); + + let mut tasks: BackgroundTasks = Default::default(); + let client_store = ClientStore::new_with_timeout(Duration::from_secs(1337 * 21 * 37)); + let _gateway_1 = tasks.register( + HttpGatewayTask::new( + request.clone(), + client_store.clone(), + ResponseMode::Basic, + addr, + ), + 0, + 8, + ); + let _gateway_2 = tasks.register( + HttpGatewayTask::new( + request.clone(), + client_store.clone(), + ResponseMode::Basic, + addr, + ), + 1, + 8, + ); + + let mut finished = 0; + let mut responses = 0; + + while finished < 2 && responses < 2 { + match tasks.next().await.unwrap() { + (id, TaskUpdate::Finished(Ok(()))) => { + println!("gateway {id} finished"); + finished += 1; + } + ( + id, + TaskUpdate::Message(InProxyTaskMessage::Http(HttpOut::ResponseBasic(response))), + ) => { + println!("gateway {id} returned a response"); + assert_eq!(response.internal_response.status, StatusCode::OK); + responses += 1; + } + other => panic!("unexpected task update: {other:?}"), + } + } + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/interceptor.rs b/mirrord/intproxy/src/proxies/incoming/interceptor.rs deleted file mode 100644 index 2d6486d709f..00000000000 --- a/mirrord/intproxy/src/proxies/incoming/interceptor.rs +++ /dev/null @@ -1,909 +0,0 @@ -//! [`BackgroundTask`] used by [`Incoming`](super::IncomingProxy) to manage a single -//! intercepted connection. - -use std::{ - error::Error, - io::{self, ErrorKind}, - net::SocketAddr, - time::Duration, -}; - -use bytes::BytesMut; -use exponential_backoff::Backoff; -use hyper::{upgrade::OnUpgrade, StatusCode, Version}; -use hyper_util::rt::TokioIo; -use mirrord_protocol::tcp::{ - HttpRequestFallback, HttpResponse, HttpResponseFallback, InternalHttpBody, ReceiverStreamBody, - HTTP_CHUNKED_RESPONSE_VERSION, -}; -use thiserror::Error; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{TcpSocket, TcpStream}, - time::{self, sleep}, -}; -use tracing::Level; - -use super::http::HttpSender; -use crate::{ - background_tasks::{BackgroundTask, MessageBus}, - proxies::incoming::http::RETRY_ON_RESET_ATTEMPTS, -}; - -/// Messages consumed by the [`Interceptor`] when it runs as a [`BackgroundTask`]. -pub enum MessageIn { - /// Request to be sent to the user application. - Http(HttpRequestFallback), - /// Data to be sent to the user application. - Raw(Vec), -} - -/// Messages produced by the [`Interceptor`] when it runs as a [`BackgroundTask`]. -#[derive(Debug)] -pub enum MessageOut { - /// Response received from the user application. - Http(HttpResponseFallback), - /// Data received from the user application. - Raw(Vec), -} - -impl From for MessageIn { - fn from(value: HttpRequestFallback) -> Self { - Self::Http(value) - } -} - -impl From> for MessageIn { - fn from(value: Vec) -> Self { - Self::Raw(value) - } -} - -/// Errors that can occur when [`Interceptor`] runs as a [`BackgroundTask`]. -#[derive(Error, Debug)] -pub enum InterceptorError { - /// IO failed. - #[error("io failed: {0}")] - Io(#[from] io::Error), - /// Hyper failed. - #[error("hyper failed: {0}")] - Hyper(hyper::Error), - /// The layer closed connection too soon to send a request. - #[error("connection closed too soon")] - ConnectionClosedTooSoon(HttpRequestFallback), - - #[error("incomplete message")] - IncompleteMessage(HttpRequestFallback), - - /// Received a request with an unsupported HTTP version. - #[error("{0:?} is not supported")] - UnsupportedHttpVersion(Version), - /// Occurs when [`Interceptor`] receives [`MessageIn::Raw`], but it acts as an HTTP gateway and - /// there was no HTTP upgrade. - #[error("received raw bytes, but expected an HTTP request")] - UnexpectedRawData, - /// Occurs when [`Interceptor`] receives [`MessageIn::Http`], but it acts as a TCP proxy. - #[error("received an HTTP request, but expected raw bytes")] - UnexpectedHttpRequest, - - /// We dig into the [`hyper::Error`] to try and see if it's an [`h2::Error`], checking - /// for [`h2::Error::is_reset`]. - /// - /// [`hyper::Error`] mentions that `source` is not a guaranteed thing we can check for, - /// so if you see any weird behavior, check that the [`h2`] crate is in sync with - /// whatever hyper changed (for errors). - #[error("HTTP2 `RST_STREAM` received")] - Reset, - - /// We have reached the max number of attempts that we can retry our http connection, - /// due to a `RST_STREAM`, or when the connection has been closed too soon. - #[error("HTTP2 reached the maximum amount of retries!")] - MaxRetries, -} - -impl From for InterceptorError { - fn from(hyper_fail: hyper::Error) -> Self { - if hyper_fail - .source() - .and_then(|source| source.downcast_ref::()) - .is_some_and(h2::Error::is_reset) - { - Self::Reset - } else { - Self::Hyper(hyper_fail) - } - } -} - -pub type InterceptorResult = core::result::Result; - -/// Manages a single intercepted connection. -/// Multiple instances are run as [`BackgroundTask`]s by one [`IncomingProxy`](super::IncomingProxy) -/// to manage individual connections. -/// -/// This interceptor can proxy both raw TCP data and HTTP messages in the same TCP connection. -/// When it receives [`MessageIn::Raw`], it starts acting as a simple proxy. -/// When it received [`MessageIn::Http`], it starts acting as an HTTP gateway. -pub struct Interceptor { - /// Socket that should be used to make the first connection (should already be bound). - socket: TcpSocket, - /// Address of user app's listener. - peer: SocketAddr, - /// Version of [`mirrord_protocol`] negotiated with the agent. - agent_protocol_version: Option, -} - -impl Interceptor { - /// Creates a new instance. When run, this instance will use the given `socket` (must be already - /// bound) to communicate with the given `peer`. - /// - /// # Note - /// - /// The socket can be replaced when retrying HTTP requests. - pub fn new( - socket: TcpSocket, - peer: SocketAddr, - agent_protocol_version: Option, - ) -> Self { - Self { - socket, - peer, - agent_protocol_version, - } - } -} - -impl BackgroundTask for Interceptor { - type Error = InterceptorError; - type MessageIn = MessageIn; - type MessageOut = MessageOut; - - #[tracing::instrument(level = Level::TRACE, skip_all, err)] - async fn run(self, message_bus: &mut MessageBus) -> InterceptorResult<(), Self::Error> { - let mut stream = self.socket.connect(self.peer).await?; - - // First, we determine whether this is a raw TCP connection or an HTTP connection. - // If we receive an HTTP request from our parent task, this must be an HTTP connection. - // If we receive raw bytes or our peer starts sending some data, this must be raw TCP. - let request = tokio::select! { - message = message_bus.recv() => match message { - Some(MessageIn::Raw(data)) => { - if data.is_empty() { - tracing::trace!("incoming interceptor -> agent shutdown, shutting down connection with layer"); - stream.shutdown().await?; - } else { - stream.write_all(&data).await?; - } - - return RawConnection { stream }.run(message_bus).await; - } - Some(MessageIn::Http(request)) => request, - None => return Ok(()), - }, - - result = stream.readable() => { - result?; - return RawConnection { stream }.run(message_bus).await; - } - }; - - let sender = super::http::handshake(request.version(), stream).await?; - let mut http_conn = HttpConnection { - sender, - peer: self.peer, - agent_protocol_version: self.agent_protocol_version.clone(), - }; - let (response, on_upgrade) = http_conn.send(request).await.inspect_err(|fail| { - tracing::error!(?fail, "Failed getting a filtered http response!") - })?; - message_bus.send(MessageOut::Http(response)).await; - - let raw = if let Some(on_upgrade) = on_upgrade { - let upgraded = on_upgrade.await?; - let parts = upgraded - .downcast::>() - .expect("IO type is known"); - if !parts.read_buf.is_empty() { - message_bus - .send(MessageOut::Raw(parts.read_buf.into())) - .await; - } - - Some(RawConnection { - stream: parts.io.into_inner(), - }) - } else { - http_conn.run(message_bus).await? - }; - - if let Some(raw) = raw { - raw.run(message_bus).await - } else { - Ok(()) - } - } -} - -/// Utilized by the [`Interceptor`] when it acts as an HTTP gateway. -/// See [`HttpConnection::run`] for usage. -struct HttpConnection { - /// Server address saved to allow for reconnecting in case a retry is required. - peer: SocketAddr, - /// Handle to the HTTP connection between the [`Interceptor`] the server. - sender: HttpSender, - /// Version of [`mirrord_protocol`] negotiated with the agent. - /// Determines which variant of [`LayerTcpSteal`](mirrord_protocol::tcp::LayerTcpSteal) - /// we use when sending HTTP responses. - agent_protocol_version: Option, -} - -impl HttpConnection { - /// Returns whether the agent supports - /// [`LayerTcpSteal::HttpResponseChunked`](mirrord_protocol::tcp::LayerTcpSteal::HttpResponseChunked). - pub fn agent_supports_streaming_response(&self) -> bool { - self.agent_protocol_version - .as_ref() - .map(|version| HTTP_CHUNKED_RESPONSE_VERSION.matches(version)) - .unwrap_or(false) - } - - /// Handles the result of sending an HTTP request. - /// Returns an [`HttpResponseFallback`] to be returned to the client or an [`InterceptorError`]. - /// - /// See [`HttpResponseFallback::response_from_request`] for notes on picking the correct - /// [`HttpResponseFallback`] variant. - #[tracing::instrument(level = Level::TRACE, skip(self, response), err(level = Level::WARN))] - async fn handle_response( - &self, - request: HttpRequestFallback, - response: InterceptorResult>, - ) -> InterceptorResult<(HttpResponseFallback, Option)> { - match response { - Err(InterceptorError::Hyper(e)) if e.is_closed() => { - tracing::warn!( - "Sending request to local application failed with: {e:?}. \ - Seems like the local application closed the connection too early, so \ - creating a new connection and trying again." - ); - tracing::trace!("The request to be retried: {request:?}."); - - Err(InterceptorError::ConnectionClosedTooSoon(request)) - } - Err(InterceptorError::Hyper(e)) if e.is_parse() => { - tracing::warn!( - "Could not parse HTTP response to filtered HTTP request, got error: {e:?}." - ); - let body_message = format!( - "mirrord: could not parse HTTP response from local application - {e:?}" - ); - Ok(( - HttpResponseFallback::response_from_request( - request, - StatusCode::BAD_GATEWAY, - &body_message, - self.agent_protocol_version.as_ref(), - ), - None, - )) - } - Err(InterceptorError::Hyper(e)) if e.is_incomplete_message() => { - tracing::warn!( - "Sending request to local application failed with: {e:?}. \ - Connection closed before the message could complete!" - ); - tracing::trace!( - ?request, - "Retrying the request, see \ - [https://github.com/hyperium/hyper/issues/2136] for more info." - ); - - Err(InterceptorError::IncompleteMessage(request)) - } - - Err(fail) => { - tracing::warn!(?fail, "Request to local application failed!"); - let body_message = format!( - "mirrord tried to forward the request to the local application and got {fail:?}" - ); - Ok(( - HttpResponseFallback::response_from_request( - request, - StatusCode::BAD_GATEWAY, - &body_message, - self.agent_protocol_version.as_ref(), - ), - None, - )) - } - - Ok(mut res) => { - let upgrade = if res.status() == StatusCode::SWITCHING_PROTOCOLS { - Some(hyper::upgrade::on(&mut res)) - } else { - None - }; - - let result = match &request { - HttpRequestFallback::Framed(..) => { - HttpResponse::::from_hyper_response( - res, - self.peer.port(), - request.connection_id(), - request.request_id(), - ) - .await - .map(HttpResponseFallback::Framed) - } - HttpRequestFallback::Fallback(..) => { - HttpResponse::>::from_hyper_response( - res, - self.peer.port(), - request.connection_id(), - request.request_id(), - ) - .await - .map(HttpResponseFallback::Fallback) - } - HttpRequestFallback::Streamed { .. } - if self.agent_supports_streaming_response() => - { - HttpResponse::::from_hyper_response( - res, - self.peer.port(), - request.connection_id(), - request.request_id(), - ) - .await - .map(|response| { - HttpResponseFallback::Streamed(response, Some(request.clone())) - }) - } - HttpRequestFallback::Streamed { .. } => { - HttpResponse::::from_hyper_response( - res, - self.peer.port(), - request.connection_id(), - request.request_id(), - ) - .await - .map(HttpResponseFallback::Framed) - } - }; - - Ok(result.map(|response| (response, upgrade))?) - } - } - } - - /// Sends the given [`HttpRequestFallback`] to the server. - /// - /// If we get a `RST_STREAM` error from the server, or the connection was closed too - /// soon starts a new connection and retries using a [`Backoff`] until we reach - /// [`RETRY_ON_RESET_ATTEMPTS`]. - /// - /// Returns [`HttpResponseFallback`] from the server. - #[tracing::instrument(level = Level::TRACE, skip(self), ret, err)] - async fn send( - &mut self, - request: HttpRequestFallback, - ) -> InterceptorResult<(HttpResponseFallback, Option)> { - let min = Duration::from_millis(10); - let max = Duration::from_millis(250); - - let mut backoffs = Backoff::new(RETRY_ON_RESET_ATTEMPTS, min, max) - .into_iter() - .flatten(); - - // Retry to handle this request a few times. - loop { - let response = self.sender.send(request.clone()).await; - - match self.handle_response(request.clone(), response).await { - Ok(response) => return Ok(response), - - Err(error @ InterceptorError::Reset) - | Err(error @ InterceptorError::ConnectionClosedTooSoon(_)) - | Err(error @ InterceptorError::IncompleteMessage(_)) => { - tracing::warn!( - ?request, - %error, - "Either the connection closed, the message is incomplete, \ - or we got a reset, retrying!" - ); - - let Some(backoff) = backoffs.next() else { - break; - }; - - sleep(backoff).await; - - // Create a new connection for the next attempt. - let socket = super::bind_similar(self.peer)?; - let stream = socket.connect(self.peer).await?; - let new_sender = super::http::handshake(request.version(), stream).await?; - self.sender = new_sender; - } - - Err(fail) => return Err(fail), - } - } - - Err(InterceptorError::MaxRetries) - } - - /// Proxies HTTP messages until an HTTP upgrade happens or the [`MessageBus`] closes. - /// Support retries (with reconnecting to the HTTP server). - /// - /// When an HTTP upgrade happens, the underlying [`TcpStream`] is reclaimed, wrapped - /// in a [`RawConnection`] and returned. When [`MessageBus`] closes, [`None`] is returned. - #[tracing::instrument(level = Level::TRACE, skip_all, ret, err)] - async fn run( - mut self, - message_bus: &mut MessageBus, - ) -> InterceptorResult> { - let upgrade = loop { - let Some(msg) = message_bus.recv().await else { - return Ok(None); - }; - - match msg { - MessageIn::Raw(..) => { - // We should not receive any raw data from the agent before sending a - //`101 SWITCHING PROTOCOLS` response. - return Err(InterceptorError::UnexpectedRawData); - } - - MessageIn::Http(req) => { - let (res, on_upgrade) = self.send(req).await.inspect_err(|fail| { - tracing::error!(?fail, "Failed getting a filtered http response!") - })?; - tracing::debug!("{} has upgrade: {}", res.request_id(), on_upgrade.is_some()); - message_bus.send(MessageOut::Http(res)).await; - - if let Some(on_upgrade) = on_upgrade { - break on_upgrade.await?; - } - } - } - }; - - let parts = upgrade - .downcast::>() - .expect("IO type is known"); - let stream = parts.io.into_inner(); - let read_buf = parts.read_buf; - - if !read_buf.is_empty() { - message_bus.send(MessageOut::Raw(read_buf.into())).await; - } - - Ok(Some(RawConnection { stream })) - } -} - -/// Utilized by the [`Interceptor`] when it acts as a TCP proxy. -/// See [`RawConnection::run`] for usage. -#[derive(Debug)] -struct RawConnection { - /// Connection between the [`Interceptor`] and the server. - stream: TcpStream, -} - -impl RawConnection { - /// Proxies raw TCP data until the [`MessageBus`] closes. - /// - /// # Notes - /// - /// 1. When the peer shuts down writing, a single 0-sized read is sent through the - /// [`MessageBus`]. This is to notify the agent about the shutdown condition. - /// - /// 2. A 0-sized read received from the [`MessageBus`] is treated as a shutdown on the agent - /// side. Connection with the peer is shut down as well. - /// - /// 3. This implementation exits only when an error is encountered or the [`MessageBus`] is - /// closed. - async fn run(mut self, message_bus: &mut MessageBus) -> InterceptorResult<()> { - let mut buf = BytesMut::with_capacity(64 * 1024); - let mut reading_closed = false; - let mut remote_closed = false; - - loop { - tokio::select! { - biased; - - res = self.stream.read_buf(&mut buf), if !reading_closed => match res { - Err(e) if e.kind() == ErrorKind::WouldBlock => {}, - Err(e) => break Err(e.into()), - Ok(..) => { - if buf.is_empty() { - tracing::trace!("incoming interceptor -> layer shutdown, sending a 0-sized read to inform the agent"); - reading_closed = true; - } - message_bus.send(MessageOut::Raw(buf.to_vec())).await; - buf.clear(); - } - }, - - msg = message_bus.recv(), if !remote_closed => match msg { - None => { - tracing::trace!("incoming interceptor -> message bus closed, waiting 1 second before exiting"); - remote_closed = true; - }, - Some(MessageIn::Raw(data)) => { - if data.is_empty() { - tracing::trace!("incoming interceptor -> agent shutdown, shutting down connection with layer"); - self.stream.shutdown().await?; - } else { - self.stream.write_all(&data).await?; - } - }, - Some(MessageIn::Http(..)) => break Err(InterceptorError::UnexpectedHttpRequest), - }, - - _ = time::sleep(Duration::from_secs(1)), if remote_closed => { - tracing::trace!("incoming interceptor -> layer silent for 1 second and message bus is closed, exiting"); - - break Ok(()); - }, - } - } - } -} - -#[cfg(test)] -mod test { - use std::{ - convert::Infallible, - sync::{Arc, Mutex}, - }; - - use bytes::Bytes; - use futures::future::FutureExt; - use http_body_util::{BodyExt, Empty, Full}; - use hyper::{ - body::Incoming, - header::{HeaderValue, CONNECTION, UPGRADE}, - server::conn::http1, - service::service_fn, - upgrade::Upgraded, - Method, Request, Response, - }; - use hyper_util::rt::{TokioExecutor, TokioIo}; - use mirrord_protocol::tcp::{HttpRequest, InternalHttpRequest, StreamingBody}; - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - sync::{watch, Notify}, - task, - }; - - use super::*; - use crate::background_tasks::{BackgroundTasks, TaskUpdate}; - - /// Binary protocol over TCP. - /// Server first sends bytes [`INITIAL_MESSAGE`], then echoes back all received data. - const TEST_PROTO: &str = "dummyecho"; - - const INITIAL_MESSAGE: &[u8] = &[0x4a, 0x50, 0x32, 0x47, 0x4d, 0x44]; - - /// Handles requests upgrading to the [`TEST_PROTO`] protocol. - async fn upgrade_req_handler( - mut req: Request, - ) -> hyper::Result>> { - async fn dummy_echo(upgraded: Upgraded) -> io::Result<()> { - let mut upgraded = TokioIo::new(upgraded); - let mut buf = [0_u8; 64]; - - upgraded.write_all(INITIAL_MESSAGE).await?; - - loop { - let bytes_read = upgraded.read(&mut buf[..]).await?; - if bytes_read == 0 { - break; - } - - let echo_back = buf.get(0..bytes_read).unwrap(); - upgraded.write_all(echo_back).await?; - } - - Ok(()) - } - - let mut res = Response::new(Empty::new()); - - let contains_expected_upgrade = req - .headers() - .get(UPGRADE) - .filter(|proto| *proto == TEST_PROTO) - .is_some(); - if !contains_expected_upgrade { - *res.status_mut() = StatusCode::BAD_REQUEST; - return Ok(res); - } - - task::spawn(async move { - match hyper::upgrade::on(&mut req).await { - Ok(upgraded) => { - if let Err(e) = dummy_echo(upgraded).await { - eprintln!("server foobar io error: {}", e) - }; - } - Err(e) => eprintln!("upgrade error: {}", e), - } - }); - - *res.status_mut() = StatusCode::SWITCHING_PROTOCOLS; - res.headers_mut() - .insert(UPGRADE, HeaderValue::from_static(TEST_PROTO)); - res.headers_mut() - .insert(CONNECTION, HeaderValue::from_static("upgrade")); - Ok(res) - } - - /// Runs a [`hyper`] server that accepts only requests upgrading to the [`TEST_PROTO`] protocol. - async fn dummy_echo_server(listener: TcpListener, mut shutdown: watch::Receiver) { - loop { - tokio::select! { - res = listener.accept() => { - let (stream, _) = res.expect("dummy echo server failed to accept connection"); - - let mut shutdown = shutdown.clone(); - - task::spawn(async move { - let conn = http1::Builder::new().serve_connection(TokioIo::new(stream), service_fn(upgrade_req_handler)); - let mut conn = conn.with_upgrades(); - let mut conn = Pin::new(&mut conn); - - tokio::select! { - res = &mut conn => { - res.expect("dummy echo server failed to serve connection"); - } - - _ = shutdown.changed() => { - conn.graceful_shutdown(); - } - } - }); - } - - _ = shutdown.changed() => break, - } - } - } - - #[tokio::test] - async fn upgrade_test() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let local_destination = listener.local_addr().unwrap(); - - let (shutdown_tx, shutdown_rx) = watch::channel(false); - let server_task = task::spawn(dummy_echo_server(listener, shutdown_rx)); - - let mut tasks: BackgroundTasks<(), MessageOut, InterceptorError> = Default::default(); - let interceptor = { - let socket = TcpSocket::new_v4().unwrap(); - socket.bind("127.0.0.1:0".parse().unwrap()).unwrap(); - tasks.register( - Interceptor::new( - socket, - local_destination, - Some(mirrord_protocol::VERSION.clone()), - ), - (), - 8, - ) - }; - - interceptor - .send(HttpRequestFallback::Fallback(HttpRequest { - connection_id: 0, - request_id: 0, - port: 80, - internal_request: InternalHttpRequest { - method: Method::GET, - uri: "dummyecho://www.mirrord.dev/".parse().unwrap(), - headers: [ - (CONNECTION, HeaderValue::from_static("upgrade")), - (UPGRADE, HeaderValue::from_static(TEST_PROTO)), - ] - .into_iter() - .collect(), - version: Version::HTTP_11, - body: Default::default(), - }, - })) - .await; - - let (_, update) = tasks.next().await.expect("no task result"); - match update { - TaskUpdate::Message(MessageOut::Http(res)) => { - let res = res - .into_hyper::() - .expect("failed to convert into hyper response"); - assert_eq!(res.status(), StatusCode::SWITCHING_PROTOCOLS); - println!("{:?}", res.headers()); - assert!(res - .headers() - .get(CONNECTION) - .filter(|v| *v == "upgrade") - .is_some()); - assert!(res - .headers() - .get(UPGRADE) - .filter(|v| *v == TEST_PROTO) - .is_some()); - } - _ => panic!("unexpected task update: {update:?}"), - } - - interceptor.send(b"test test test".to_vec()).await; - - let (_, update) = tasks.next().await.expect("no task result"); - match update { - TaskUpdate::Message(MessageOut::Raw(bytes)) => { - assert_eq!(bytes, INITIAL_MESSAGE); - } - _ => panic!("unexpected task update: {update:?}"), - } - - let (_, update) = tasks.next().await.expect("no task result"); - match update { - TaskUpdate::Message(MessageOut::Raw(bytes)) => { - assert_eq!(bytes, b"test test test"); - } - _ => panic!("unexpected task update: {update:?}"), - } - - let _ = shutdown_tx.send(true); - server_task.await.expect("dummy echo server panicked"); - } - - /// Ensure that [`HttpRequestFallback::Streamed`] are received frame by frame - #[tokio::test] - async fn receive_request_as_frames() { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let local_destination = listener.local_addr().unwrap(); - - let mut tasks: BackgroundTasks<(), MessageOut, InterceptorError> = Default::default(); - let socket = TcpSocket::new_v4().unwrap(); - socket.bind("127.0.0.1:0".parse().unwrap()).unwrap(); - let interceptor = Interceptor::new( - socket, - local_destination, - Some(mirrord_protocol::VERSION.clone()), - ); - let sender = tasks.register(interceptor, (), 8); - - let (tx, rx) = tokio::sync::mpsc::channel(12); - sender - .send(MessageIn::Http(HttpRequestFallback::Streamed { - request: HttpRequest { - internal_request: InternalHttpRequest { - method: Method::POST, - uri: "/".parse().unwrap(), - headers: Default::default(), - version: Version::HTTP_11, - body: StreamingBody::new(rx), - }, - connection_id: 1, - request_id: 2, - port: 3, - }, - retries: 0, - })) - .await; - let (connection, _peer_addr) = listener.accept().await.unwrap(); - - let tx = Mutex::new(Some(tx)); - let notifier = Arc::new(Notify::default()); - let finished = notifier.notified(); - - let service = service_fn(|mut req: Request| { - let tx = tx.lock().unwrap().take().unwrap(); - let notifier = notifier.clone(); - async move { - let x = req.body_mut().frame().now_or_never(); - assert!(x.is_none()); - tx.send(mirrord_protocol::tcp::InternalHttpBodyFrame::Data( - b"string".to_vec(), - )) - .await - .unwrap(); - let x = req - .body_mut() - .frame() - .await - .unwrap() - .unwrap() - .into_data() - .unwrap(); - assert_eq!(x, b"string".to_vec()); - let x = req.body_mut().frame().now_or_never(); - assert!(x.is_none()); - - tx.send(mirrord_protocol::tcp::InternalHttpBodyFrame::Data( - b"another_string".to_vec(), - )) - .await - .unwrap(); - let x = req - .body_mut() - .frame() - .await - .unwrap() - .unwrap() - .into_data() - .unwrap(); - assert_eq!(x, b"another_string".to_vec()); - - drop(tx); - let x = req.body_mut().frame().await; - assert!(x.is_none()); - - notifier.notify_waiters(); - Ok::<_, hyper::Error>(Response::new(Empty::::new())) - } - }); - let conn = http1::Builder::new().serve_connection(TokioIo::new(connection), service); - - tokio::select! { - result = conn => { - result.unwrap() - } - _ = finished => { - - } - } - } - - /// Checks that [`hyper`] and [`h2`] crate versions are in sync with each other. - /// - /// As we use `source.downcast_ref::` to drill down on [`h2`] errors from - /// [`hyper`], we need these two crates to stay in sync, otherwise we could always - /// fail some of our checks that rely on this `downcast` working. - /// - /// Even though we're using [`h2::Error::is_reset`] in intproxy, this test can be - /// for any error, and thus here we do it for [`h2::Error::is_go_away`] which is - /// easier to trigger. - #[tokio::test] - async fn hyper_and_h2_versions_in_sync() { - let notify = Arc::new(Notify::new()); - let wait_notify = notify.clone(); - - tokio::spawn(async move { - let listener = TcpListener::bind("127.0.0.1:6666").await.unwrap(); - - notify.notify_waiters(); - let (io, _) = listener.accept().await.unwrap(); - - if let Err(fail) = hyper::server::conn::http2::Builder::new(TokioExecutor::default()) - .serve_connection( - TokioIo::new(io), - service_fn(|_| async move { - Ok::<_, Infallible>(Response::new(Full::new(Bytes::from("Heresy!")))) - }), - ) - .await - { - assert!(fail - .source() - .and_then(|source| source.downcast_ref::()) - .is_some_and(h2::Error::is_go_away)); - } else { - panic!( - r"The request is supposed to fail with `GO_AWAY`! - Something is wrong if it didn't! - - >> If you're seeing this error, the cause is likely that `hyper` and `h2` - versions are out of sync, and we can't have that due to our use of - `downcast_ref` on some `h2` errors!" - ); - } - }); - - // Wait for the listener to be ready for our connection. - wait_notify.notified().await; - - assert!(reqwest::get("https://127.0.0.1:6666").await.is_err()); - } -} diff --git a/mirrord/intproxy/src/proxies/incoming/metadata_store.rs b/mirrord/intproxy/src/proxies/incoming/metadata_store.rs new file mode 100644 index 00000000000..fc15d7e9d4b --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/metadata_store.rs @@ -0,0 +1,48 @@ +use std::collections::HashMap; + +use mirrord_intproxy_protocol::{ConnMetadataRequest, ConnMetadataResponse}; +use mirrord_protocol::ConnectionId; + +/// Maps local socket address pairs to remote. +/// +/// Allows for extracting the original socket addresses of peers of a remote connection. +#[derive(Default)] +pub struct MetadataStore { + prepared_responses: HashMap, + expected_requests: HashMap, +} + +impl MetadataStore { + /// Retrieves remote addresses for the given pair of local addresses. + /// + /// If the mapping is not found, returns the local addresses unchanged. + pub fn get(&mut self, req: ConnMetadataRequest) -> ConnMetadataResponse { + self.prepared_responses + .remove(&req) + .unwrap_or_else(|| ConnMetadataResponse { + remote_source: req.peer_address, + local_address: req.listener_address.ip(), + }) + } + + /// Adds a new `req`->`res` mapping to this struct. + /// + /// Marks that the mapping is related to the remote connection with the given id. + pub fn expect( + &mut self, + req: ConnMetadataRequest, + connection: ConnectionId, + res: ConnMetadataResponse, + ) { + self.expected_requests.insert(connection, req.clone()); + self.prepared_responses.insert(req, res); + } + + /// Clears mapping related to the remote connection with the given id. + pub fn no_longer_expect(&mut self, connection: ConnectionId) { + let Some(req) = self.expected_requests.remove(&connection) else { + return; + }; + self.prepared_responses.remove(&req); + } +} diff --git a/mirrord/intproxy/src/proxies/incoming/port_subscription_ext.rs b/mirrord/intproxy/src/proxies/incoming/port_subscription_ext.rs index e928be69ace..58d92011076 100644 --- a/mirrord/intproxy/src/proxies/incoming/port_subscription_ext.rs +++ b/mirrord/intproxy/src/proxies/incoming/port_subscription_ext.rs @@ -3,7 +3,7 @@ use mirrord_intproxy_protocol::PortSubscription; use mirrord_protocol::{ tcp::{LayerTcp, LayerTcpSteal, StealType}, - ClientMessage, ConnectionId, Port, + ClientMessage, Port, }; /// Retrieves subscribed port from the given [`StealType`]. @@ -26,9 +26,6 @@ pub trait PortSubscriptionExt { /// Returns an unsubscribe request to be sent to the agent. fn wrap_agent_unsubscribe(&self) -> ClientMessage; - - /// Returns an unsubscribe connection request to be sent to the agent. - fn wrap_agent_unsubscribe_connection(&self, connection_id: ConnectionId) -> ClientMessage; } impl PortSubscriptionExt for PortSubscription { @@ -58,14 +55,4 @@ impl PortSubscriptionExt for PortSubscription { } } } - - /// [`LayerTcp::ConnectionUnsubscribe`] or [`LayerTcpSteal::ConnectionUnsubscribe`]. - fn wrap_agent_unsubscribe_connection(&self, connection_id: ConnectionId) -> ClientMessage { - match self { - Self::Mirror(..) => ClientMessage::Tcp(LayerTcp::ConnectionUnsubscribe(connection_id)), - Self::Steal(..) => { - ClientMessage::TcpSteal(LayerTcpSteal::ConnectionUnsubscribe(connection_id)) - } - } - } } diff --git a/mirrord/intproxy/src/proxies/incoming/tasks.rs b/mirrord/intproxy/src/proxies/incoming/tasks.rs new file mode 100644 index 00000000000..49a315636c5 --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/tasks.rs @@ -0,0 +1,113 @@ +use std::{convert::Infallible, fmt, io}; + +use hyper::{upgrade::OnUpgrade, Version}; +use mirrord_protocol::{ + tcp::{ChunkedResponse, HttpResponse, InternalHttpBody}, + ConnectionId, Port, RequestId, +}; +use thiserror::Error; + +/// Messages produced by the [`BackgroundTask`](crate::background_tasks::BackgroundTask)s used in +/// the [`IncomingProxy`](super::IncomingProxy). +pub enum InProxyTaskMessage { + /// Produced by the [`TcpProxyTask`](super::tcp_proxy::TcpProxyTask) in steal mode. + Tcp( + /// Data received from the local application. + Vec, + ), + /// Produced by the [`HttpGatewayTask`](super::http_gateway::HttpGatewayTask). + Http( + /// HTTP spefiic message. + HttpOut, + ), +} + +impl fmt::Debug for InProxyTaskMessage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Tcp(data) => f + .debug_tuple("Tcp") + .field(&format_args!("{} bytes", data.len())) + .finish(), + Self::Http(msg) => f.debug_tuple("Http").field(msg).finish(), + } + } +} + +/// Messages produced by the [`HttpGatewayTask`](super::http_gateway::HttpGatewayTask). +#[derive(Debug)] +pub enum HttpOut { + /// Response from the local application's HTTP server. + ResponseBasic(HttpResponse>), + /// Response from the local application's HTTP server. + ResponseFramed(HttpResponse), + /// Response from the local application's HTTP server. + ResponseChunked(ChunkedResponse), + /// Upgraded HTTP connection, to be handled as a remote connection stolen without any filter. + Upgraded(OnUpgrade), +} + +impl From> for InProxyTaskMessage { + fn from(value: Vec) -> Self { + Self::Tcp(value) + } +} + +impl From for InProxyTaskMessage { + fn from(value: HttpOut) -> Self { + Self::Http(value) + } +} + +/// Errors that can occur in the [`BackgroundTask`](crate::background_tasks::BackgroundTask)s used +/// in the [`IncomingProxy`](super::IncomingProxy). +/// +/// All of these can occur only in the [`TcpProxyTask`](super::tcp_proxy::TcpProxyTask) +/// and mean that the local connection is irreversibly broken. +/// The [`HttpGatewayTask`](super::http_gateway::HttpGatewayTask) produces no errors +/// and instead responds with an error HTTP response to the agent. +/// +/// However, due to [`BackgroundTasks`](crate::background_tasks::BackgroundTasks) +/// type constraints, we need a common error type. +/// Thus, this type implements [`From`]. +#[derive(Error, Debug)] +pub enum InProxyTaskError { + #[error("io failed: {0}")] + IoError(#[from] io::Error), + #[error("local HTTP upgrade failed: {0}")] + UpgradeError(#[source] hyper::Error), +} + +impl From for InProxyTaskError { + fn from(_: Infallible) -> Self { + unreachable!() + } +} + +/// Types of [`BackgroundTask`](crate::background_tasks::BackgroundTask)s used in the +/// [`IncomingProxy`](super::IncomingProxy). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum InProxyTask { + /// [`TcpProxyTask`](super::tcp_proxy::TcpProxyTask) handling a mirrored connection. + MirrorTcpProxy(ConnectionId), + /// [`TcpProxyTask`](super::tcp_proxy::TcpProxyTask) handling a stolen connection. + StealTcpProxy(ConnectionId), + /// [`HttpGatewayTask`](super::http_gateway::HttpGatewayTask) handling a stolen HTTP request. + HttpGateway(HttpGatewayId), +} + +/// Identifies a [`HttpGatewayTask`](super::http_gateway::HttpGatewayTask). +/// +/// ([`ConnectionId`], [`RequestId`]) would suffice, but storing extra data allows us to produce an +/// error response in case the task somehow panics. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct HttpGatewayId { + /// Id of the remote connection. + pub connection_id: ConnectionId, + /// Id of the stolen request. + pub request_id: RequestId, + /// Remote port from which the request was stolen. + pub port: Port, + /// HTTP version of the stolen request. + pub version: Version, +} diff --git a/mirrord/intproxy/src/proxies/incoming/tcp_proxy.rs b/mirrord/intproxy/src/proxies/incoming/tcp_proxy.rs new file mode 100644 index 00000000000..6929f65b2f0 --- /dev/null +++ b/mirrord/intproxy/src/proxies/incoming/tcp_proxy.rs @@ -0,0 +1,198 @@ +use std::{io::ErrorKind, net::SocketAddr, time::Duration}; + +use bytes::BytesMut; +use hyper::upgrade::OnUpgrade; +use hyper_util::rt::TokioIo; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, + time, +}; +use tracing::Level; + +use super::{ + bound_socket::BoundTcpSocket, + tasks::{InProxyTaskError, InProxyTaskMessage}, +}; +use crate::background_tasks::{BackgroundTask, MessageBus}; + +/// Local TCP connections between the [`TcpProxyTask`] and the user application. +#[derive(Debug)] +pub enum LocalTcpConnection { + /// Not yet established. Should be made by the [`TcpProxyTask`] from the given + /// [`BoundTcpSocket`]. + FromTheStart { + socket: BoundTcpSocket, + peer: SocketAddr, + }, + /// Upgraded HTTP connection from a previously stolen HTTP request. + AfterUpgrade(OnUpgrade), +} + +/// [`BackgroundTask`] of [`IncomingProxy`](super::IncomingProxy) that handles a remote +/// stolen/mirrored TCP connection. +/// +/// In steal mode, exits immediately when it's [`TaskSender`](crate::background_tasks::TaskSender) +/// is dropped. +/// +/// In mirror mode, when it's [`TaskSender`](crate::background_tasks::TaskSender) is dropped, +/// this proxy keeps reading data from the user application and exits after +/// [`Self::MIRROR_MODE_LINGER_TIMEOUT`] of silence. +#[derive(Debug)] +pub struct TcpProxyTask { + /// The local connection between this task and the user application. + connection: LocalTcpConnection, + /// Whether this task should silently discard data coming from the user application. + /// + /// The data is discarded only when the remote connection is mirrored. + discard_data: bool, +} + +impl TcpProxyTask { + /// Mirror mode only: how long do we wait before exiting after the [`MessageBus`] is closed + /// and user application doesn't send any data. + pub const MIRROR_MODE_LINGER_TIMEOUT: Duration = Duration::from_secs(1); + + /// Creates a new task. + /// + /// * This task will talk with the user application using the given [`LocalTcpConnection`]. + /// * If `discard_data` is set, this task will silently discard all data coming from the user + /// application. + pub fn new(connection: LocalTcpConnection, discard_data: bool) -> Self { + Self { + connection, + discard_data, + } + } +} + +impl BackgroundTask for TcpProxyTask { + type Error = InProxyTaskError; + type MessageIn = Vec; + type MessageOut = InProxyTaskMessage; + + #[tracing::instrument(level = Level::TRACE, name = "tcp_proxy_task_main_loop", skip(message_bus), err(level = Level::WARN))] + async fn run(self, message_bus: &mut MessageBus) -> Result<(), Self::Error> { + let mut stream = match self.connection { + LocalTcpConnection::FromTheStart { socket, peer } => { + let Some(stream) = message_bus + .closed() + .cancel_on_close(socket.connect(peer)) + .await + else { + return Ok(()); + }; + + stream? + } + + LocalTcpConnection::AfterUpgrade(on_upgrade) => { + let upgraded = on_upgrade.await.map_err(InProxyTaskError::UpgradeError)?; + let parts = upgraded + .downcast::>() + .expect("IO type is known"); + let stream = parts.io.into_inner(); + let read_buf = parts.read_buf; + + if !self.discard_data && !read_buf.is_empty() { + // We don't send empty data, + // because the agent recognizes it as a shutdown from the user application. + message_bus.send(Vec::from(read_buf)).await; + } + + stream + } + }; + + let peer_addr = stream.peer_addr()?; + let self_addr = stream.local_addr()?; + + let mut buf = BytesMut::with_capacity(64 * 1024); + let mut reading_closed = false; + let mut is_lingering = false; + + loop { + tokio::select! { + res = stream.read_buf(&mut buf), if !reading_closed => match res { + Err(e) if e.kind() == ErrorKind::WouldBlock => {}, + Err(e) => break Err(e.into()), + Ok(..) => { + if buf.is_empty() { + reading_closed = true; + + tracing::trace!( + peer_addr = %peer_addr, + self_addr = %self_addr, + "The user application shut down its side of the connection", + ) + } else { + tracing::trace!( + data_len = buf.len(), + peer_addr = %peer_addr, + self_addr = %self_addr, + "Received some data from the user application", + ); + } + + if !self.discard_data { + message_bus.send(buf.to_vec()).await; + } + + buf.clear(); + } + }, + + msg = message_bus.recv(), if !is_lingering => match msg { + None if self.discard_data => { + tracing::trace!( + peer_addr = %peer_addr, + self_addr = %self_addr, + "Message bus closed, waiting until the connection is silent", + ); + + is_lingering = true; + } + None => { + tracing::trace!( + peer_addr = %peer_addr, + self_addr = %self_addr, + "Message bus closed, exiting", + ); + + break Ok(()); + } + Some(data) => { + if data.is_empty() { + tracing::trace!( + peer_addr = %peer_addr, + self_addr = %self_addr, + "The agent shut down its side of the connection", + ); + + stream.shutdown().await?; + } else { + tracing::trace!( + data_len = data.len(), + peer_addr = %peer_addr, + self_addr = %self_addr, + "Received some data from the agent", + ); + + stream.write_all(&data).await?; + } + }, + }, + + _ = time::sleep(Self::MIRROR_MODE_LINGER_TIMEOUT), if is_lingering => { + tracing::trace!( + peer_addr = %peer_addr, + self_addr = %self_addr, + "Message bus is closed and the connection is silent, exiting", + ); + + break Ok(()); + } + } + } + } +} diff --git a/mirrord/layer/tests/common/mod.rs b/mirrord/layer/tests/common/mod.rs index 22945790e4b..d304642d28b 100644 --- a/mirrord/layer/tests/common/mod.rs +++ b/mirrord/layer/tests/common/mod.rs @@ -44,7 +44,7 @@ pub const RUST_OUTGOING_LOCAL: &str = "4.4.4.4:4444"; /// /// We take advantage of how Rust's thread naming scheme for tests to create the log files, /// and if we have no thread name, then we just write the logs to `stderr`. -pub fn init_tracing() -> Result> { +pub fn init_tracing() -> DefaultGuard { let subscriber = tracing_subscriber::fmt() .with_env_filter(EnvFilter::new("mirrord=trace")) .without_time() @@ -61,7 +61,7 @@ pub fn init_tracing() -> Result> { .map(|name| name.replace(':', "_")) { Some(test_name) => { - let mut logs_file = PathBuf::from_str("/tmp/intproxy_logs")?; + let mut logs_file = PathBuf::from("/tmp/intproxy_logs"); #[cfg(target_os = "macos")] logs_file.push("macos"); @@ -71,26 +71,28 @@ pub fn init_tracing() -> Result> { let _ = std::fs::create_dir_all(&logs_file).ok(); logs_file.push(&test_name); - match File::create(logs_file) { + match File::create(&logs_file) { + // Writes the logs to the file. Ok(file) => { + println!("Created intproxy log file: {}", logs_file.display()); let subscriber = subscriber.with_writer(Arc::new(file)).finish(); - - // Writes the logs to a file. - Ok(tracing::subscriber::set_default(subscriber)) + tracing::subscriber::set_default(subscriber) } - Err(_) => { + // File creation failure makes the output go to `stderr`. + Err(error) => { + println!("Failed to create intproxy log file at {}: {error}. Intproxy logs will be flushed to stderr", logs_file.display()); let subscriber = subscriber.with_writer(io::stderr).finish(); - - // File creation failure makes the output go to `stderr`. - Ok(tracing::subscriber::set_default(subscriber)) + tracing::subscriber::set_default(subscriber) } } } + // No thread name makes the output go to `stderr`. None => { + println!( + "Failed to obtain current thread name, intproxy logs will be flushed to stderr" + ); let subscriber = subscriber.with_writer(io::stderr).finish(); - - // No thread name makes the output go to `stderr`. - Ok(tracing::subscriber::set_default(subscriber)) + tracing::subscriber::set_default(subscriber) } } } @@ -782,9 +784,9 @@ pub enum Application { /// Mode to use when opening the file, accepted as `-m` param. mode: u32, }, - // For running applications with the executable and arguments determined at runtime. + /// For running applications with the executable and arguments determined at runtime. DynamicApp(String, Vec), - // Go app that only checks whether Linux pidfd syscalls are supported. + /// Go app that only checks whether Linux pidfd syscalls are supported. Go23Issue2988, } diff --git a/mirrord/layer/tests/fileops.rs b/mirrord/layer/tests/fileops.rs index 5a9ee96f726..5daacdadc50 100644 --- a/mirrord/layer/tests/fileops.rs +++ b/mirrord/layer/tests/fileops.rs @@ -44,7 +44,7 @@ async fn self_open( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer(dylib_path, vec![], None) @@ -65,7 +65,7 @@ async fn self_open( #[tokio::test] #[timeout(Duration::from_secs(20))] async fn read_from_mirrord_bin(dylib_path: &Path) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let contents = "please don't flake"; let temp_dir = env::temp_dir(); @@ -108,7 +108,7 @@ async fn read_from_mirrord_bin(dylib_path: &Path) { #[tokio::test] #[timeout(Duration::from_secs(60))] async fn pwrite(#[values(Application::RustFileOps)] application: Application, dylib_path: &Path) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); // add rw override for the specific path let (mut test_process, mut intproxy) = application @@ -228,7 +228,7 @@ async fn node_close( #[values(Application::NodeFileOps)] application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer( @@ -295,7 +295,7 @@ async fn go_stat( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); // add rw override for the specific path let (mut test_process, mut intproxy) = application @@ -358,7 +358,7 @@ async fn go_dir( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer( @@ -478,7 +478,7 @@ async fn go_dir_on_linux( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer( @@ -575,7 +575,7 @@ async fn go_dir_bypass( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let tmp_dir = temp_dir().join("go_dir_bypass_test"); std::fs::create_dir_all(tmp_dir.clone()).unwrap(); @@ -616,7 +616,7 @@ async fn read_go( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer(dylib_path, vec![("MIRRORD_FILE_MODE", "read")], None) @@ -658,7 +658,7 @@ async fn write_go( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut layer_connection) = application .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None) @@ -687,7 +687,7 @@ async fn lseek_go( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None) @@ -722,7 +722,7 @@ async fn faccessat_go( application: Application, dylib_path: &Path, ) { - let _tracing = init_tracing().unwrap(); + let _tracing = init_tracing(); let (mut test_process, mut intproxy) = application .start_process_with_layer(dylib_path, get_rw_test_file_env_vars(), None) diff --git a/mirrord/layer/tests/http_mirroring.rs b/mirrord/layer/tests/http_mirroring.rs index e37d433a0e3..96f2230ed18 100644 --- a/mirrord/layer/tests/http_mirroring.rs +++ b/mirrord/layer/tests/http_mirroring.rs @@ -30,6 +30,8 @@ async fn mirroring_with_http( dylib_path: &Path, config_dir: &Path, ) { + let _guard = init_tracing(); + let (mut test_process, mut intproxy) = application .start_process_with_layer_and_port( dylib_path, diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index e6c9980c15b..b7aaaca403a 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -21,6 +21,7 @@ actix-codec.workspace = true bincode.workspace = true bytes.workspace = true thiserror.workspace = true +futures.workspace = true hickory-resolver.workspace = true hickory-proto.workspace = true serde.workspace = true @@ -31,8 +32,6 @@ http-body-util = { workspace = true } fancy-regex = { workspace = true } socket2.workspace = true semver = { workspace = true, features = ["serde"] } -tokio-stream.workspace = true -tokio.workspace = true mirrord-macros = { path = "../macros" } diff --git a/mirrord/protocol/src/batched_body.rs b/mirrord/protocol/src/batched_body.rs new file mode 100644 index 00000000000..9f5780cf495 --- /dev/null +++ b/mirrord/protocol/src/batched_body.rs @@ -0,0 +1,86 @@ +use std::future::Future; + +use futures::FutureExt; +use http_body_util::BodyExt; +use hyper::body::{Body, Frame}; + +/// Utility extension trait for [`Body`]. +/// +/// Contains methods that allow for reading [`Frame`]s in batches. +pub trait BatchedBody: Body { + /// Reads all [`Frame`]s that are available without blocking. + fn ready_frames(&mut self) -> Result, Self::Error>; + + /// Waits for the next [`Frame`] then reads all [`Frame`]s that are available without blocking. + fn next_frames(&mut self) -> impl Future, Self::Error>>; +} + +impl BatchedBody for B +where + B: Body + Unpin, +{ + fn ready_frames(&mut self) -> Result, Self::Error> { + let mut frames = Frames { + frames: vec![], + is_last: false, + }; + extend_with_ready(self, &mut frames)?; + Ok(frames) + } + + async fn next_frames(&mut self) -> Result, Self::Error> { + let mut frames = Frames { + frames: vec![], + is_last: false, + }; + + match self.frame().await { + None => { + frames.is_last = true; + return Ok(frames); + } + Some(result) => { + frames.frames.push(result?); + } + } + + extend_with_ready(self, &mut frames)?; + + Ok(frames) + } +} + +/// Extends the given [`Frames`] instance with [`Frame`]s that are available without blocking. +fn extend_with_ready( + body: &mut B, + frames: &mut Frames, +) -> Result<(), B::Error> { + loop { + match body.frame().now_or_never() { + None => { + frames.is_last = false; + break; + } + Some(None) => { + frames.is_last = true; + break; + } + Some(Some(result)) => { + frames.frames.push(result?); + frames.is_last = false; + } + } + } + + Ok(()) +} + +/// A batch of body [`Frame`]s. +/// +/// `D` parameter determines [`Body::Data`] type. +pub struct Frames { + /// A batch of consecutive [`Frames`]. + pub frames: Vec>, + /// Whether the [`Body`] has finished and this is the last batch. + pub is_last: bool, +} diff --git a/mirrord/protocol/src/body_chunks.rs b/mirrord/protocol/src/body_chunks.rs deleted file mode 100644 index 19a42c78ae4..00000000000 --- a/mirrord/protocol/src/body_chunks.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::{ - future::Future, - pin::Pin, - task::{Context, Poll}, -}; - -use bytes::Bytes; -use hyper::body::{Body, Frame}; - -pub trait BodyExt { - fn next_frames(&mut self, no_wait: bool) -> FramesFut<'_, B>; -} - -impl BodyExt for B -where - B: Body, -{ - fn next_frames(&mut self, no_wait: bool) -> FramesFut<'_, B> { - FramesFut { - body: self, - no_wait, - } - } -} - -pub struct FramesFut<'a, B> { - body: &'a mut B, - no_wait: bool, -} - -impl Future for FramesFut<'_, B> -where - B: Body + Unpin, -{ - type Output = hyper::Result; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut frames = vec![]; - - loop { - let result = match Pin::new(&mut self.as_mut().body).poll_frame(cx) { - Poll::Ready(Some(Err(error))) => Poll::Ready(Err(error)), - Poll::Ready(Some(Ok(frame))) => { - frames.push(frame); - continue; - } - Poll::Ready(None) => Poll::Ready(Ok(Frames { - frames, - is_last: true, - })), - Poll::Pending => { - if frames.is_empty() && !self.no_wait { - Poll::Pending - } else { - Poll::Ready(Ok(Frames { - frames, - is_last: false, - })) - } - } - }; - - break result; - } - } -} - -pub struct Frames { - pub frames: Vec>, - pub is_last: bool, -} diff --git a/mirrord/protocol/src/lib.rs b/mirrord/protocol/src/lib.rs index 94a555ce0da..983fcd3536b 100644 --- a/mirrord/protocol/src/lib.rs +++ b/mirrord/protocol/src/lib.rs @@ -3,7 +3,7 @@ #![warn(clippy::indexing_slicing)] #![deny(unused_crate_dependencies)] -pub mod body_chunks; +pub mod batched_body; pub mod codec; pub mod dns; pub mod error; diff --git a/mirrord/protocol/src/tcp.rs b/mirrord/protocol/src/tcp.rs index 023369129ad..acf3d734121 100644 --- a/mirrord/protocol/src/tcp.rs +++ b/mirrord/protocol/src/tcp.rs @@ -5,27 +5,22 @@ use std::{ fmt, net::IpAddr, pin::Pin, - sync::{Arc, LazyLock, Mutex}, + sync::LazyLock, task::{Context, Poll}, }; use bincode::{Decode, Encode}; use bytes::Bytes; -use http_body_util::{combinators::BoxBody, BodyExt, Full, StreamBody}; +use http_body_util::BodyExt; use hyper::{ - body::{Body, Frame, Incoming}, - http, - http::response::Parts, + body::{Body, Frame}, HeaderMap, Method, Request, Response, StatusCode, Uri, Version, }; use mirrord_macros::protocol_break; use semver::VersionReq; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc::Receiver; -use tokio_stream::wrappers::ReceiverStream; -use tracing::{error, Level}; -use crate::{body_chunks::BodyExt as _, ConnectionId, Port, RemoteResult, RequestId}; +use crate::{ConnectionId, Port, RemoteResult, RequestId}; #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub struct NewTcpConnection { @@ -120,7 +115,7 @@ pub struct Filter(String); impl Filter { pub fn new(filter_str: String) -> Result> { let _ = fancy_regex::Regex::new(&filter_str).inspect_err(|fail| { - error!( + tracing::error!( r" Something went wrong while creating a regex for [{filter_str:#?}]! @@ -239,7 +234,7 @@ pub enum ChunkedResponse { /// (De-)Serializable HTTP request. #[derive(Serialize, Deserialize, PartialEq, Debug, Eq, Clone)] -pub struct InternalHttpRequest { +pub struct InternalHttpRequest { #[serde(with = "http_serde::method")] pub method: Method, @@ -252,60 +247,34 @@ pub struct InternalHttpRequest { #[serde(with = "http_serde::version")] pub version: Version, - pub body: Body, + pub body: B, } -impl From> for Request> -where - E: From, -{ - fn from(value: InternalHttpRequest) -> Self { +impl InternalHttpRequest { + pub fn map_body(self, cb: F) -> InternalHttpRequest + where + F: FnOnce(B) -> T, + { let InternalHttpRequest { + version, + headers, method, uri, - headers, - version, body, - } = value; - let mut request = Request::new(BoxBody::new(body.map_err(|e| e.into()))); - *request.method_mut() = method; - *request.uri_mut() = uri; - *request.version_mut() = version; - *request.headers_mut() = headers; - - request - } -} + } = self; -impl From>> for Request> -where - E: From, -{ - fn from(value: InternalHttpRequest>) -> Self { - let InternalHttpRequest { + InternalHttpRequest { + version, + headers, method, uri, - headers, - version, - body, - } = value; - let mut request = Request::new(BoxBody::new( - Full::new(Bytes::from(body)).map_err(|e| e.into()), - )); - *request.method_mut() = method; - *request.uri_mut() = uri; - *request.version_mut() = version; - *request.headers_mut() = headers; - - request + body: cb(body), + } } } -impl From> for Request> -where - E: From, -{ - fn from(value: InternalHttpRequest) -> Self { +impl From> for Request { + fn from(value: InternalHttpRequest) -> Self { let InternalHttpRequest { method, uri, @@ -313,7 +282,8 @@ where version, body, } = value; - let mut request = Request::new(BoxBody::new(body.map_err(|e| e.into()))); + + let mut request = Request::new(body); *request.method_mut() = method; *request.uri_mut() = uri; *request.version_mut() = version; @@ -323,118 +293,6 @@ where } } -#[derive(Clone, Debug)] -pub enum HttpRequestFallback { - Framed(HttpRequest), - Fallback(HttpRequest>), - Streamed { - request: HttpRequest, - retries: u32, - }, -} - -#[derive(Debug)] -pub struct StreamingBody { - /// Shared with instances acquired via [`Clone`]. - /// Allows the clones to receive a copy of the data. - origin: Arc, Vec)>>, - /// Index of the next frame to return from the buffer. - /// If outside of the buffer, we need to poll the stream to get the next frame. - /// Local state of this instance, zeroed when cloning. - idx: usize, -} - -impl StreamingBody { - pub fn new(rx: Receiver) -> Self { - Self { - origin: Arc::new(Mutex::new((rx, vec![]))), - idx: 0, - } - } -} - -impl Clone for StreamingBody { - fn clone(&self) -> Self { - Self { - origin: self.origin.clone(), - idx: 0, - } - } -} - -impl Body for StreamingBody { - type Data = Bytes; - - type Error = Infallible; - - fn poll_frame( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll, Self::Error>>> { - let this = self.get_mut(); - let mut guard = this.origin.lock().unwrap(); - - if let Some(frame) = guard.1.get(this.idx) { - this.idx += 1; - return Poll::Ready(Some(Ok(frame.clone().into()))); - } - - match std::task::ready!(guard.0.poll_recv(cx)) { - None => Poll::Ready(None), - Some(frame) => { - guard.1.push(frame.clone()); - this.idx += 1; - Poll::Ready(Some(Ok(frame.into()))) - } - } - } -} - -impl HttpRequestFallback { - pub fn connection_id(&self) -> ConnectionId { - match self { - HttpRequestFallback::Framed(req) => req.connection_id, - HttpRequestFallback::Fallback(req) => req.connection_id, - HttpRequestFallback::Streamed { request: req, .. } => req.connection_id, - } - } - - pub fn port(&self) -> Port { - match self { - HttpRequestFallback::Framed(req) => req.port, - HttpRequestFallback::Fallback(req) => req.port, - HttpRequestFallback::Streamed { request: req, .. } => req.port, - } - } - - pub fn request_id(&self) -> RequestId { - match self { - HttpRequestFallback::Framed(req) => req.request_id, - HttpRequestFallback::Fallback(req) => req.request_id, - HttpRequestFallback::Streamed { request: req, .. } => req.request_id, - } - } - - pub fn version(&self) -> Version { - match self { - HttpRequestFallback::Framed(req) => req.version(), - HttpRequestFallback::Fallback(req) => req.version(), - HttpRequestFallback::Streamed { request: req, .. } => req.version(), - } - } - - pub fn into_hyper(self) -> Request> - where - E: From, - { - match self { - HttpRequestFallback::Framed(req) => req.internal_request.into(), - HttpRequestFallback::Fallback(req) => req.internal_request.into(), - HttpRequestFallback::Streamed { request: req, .. } => req.internal_request.into(), - } - } -} - /// Minimal mirrord-protocol version that allows [`DaemonTcp::HttpRequestFramed`] and /// [`LayerTcpSteal::HttpResponseFramed`]. pub static HTTP_FRAMED_VERSION: LazyLock = @@ -478,6 +336,24 @@ impl HttpRequest { pub fn version(&self) -> Version { self.internal_request.version } + + pub fn map_body(self, map: F) -> HttpRequest + where + F: FnOnce(B) -> T, + { + HttpRequest { + connection_id: self.connection_id, + request_id: self.request_id, + port: self.port, + internal_request: InternalHttpRequest { + method: self.internal_request.method, + uri: self.internal_request.uri, + headers: self.internal_request.headers, + version: self.internal_request.version, + body: map(self.internal_request.body), + }, + } + } } /// (De-)Serializable HTTP response. @@ -516,16 +392,28 @@ impl InternalHttpResponse { } } -#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq, Clone)] -pub struct InternalHttpBody(VecDeque); +impl From> for Response { + fn from(value: InternalHttpResponse) -> Self { + let InternalHttpResponse { + status, + version, + headers, + body, + } = value; -impl InternalHttpBody { - pub fn from_bytes(bytes: &[u8]) -> Self { - InternalHttpBody(VecDeque::from([InternalHttpBodyFrame::Data( - bytes.to_vec(), - )])) + let mut response = Response::new(body); + *response.status_mut() = status; + *response.version_mut() = version; + *response.headers_mut() = headers; + + response } +} +#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Eq, Clone)] +pub struct InternalHttpBody(pub VecDeque); + +impl InternalHttpBody { pub async fn from_body(mut body: B) -> Result where B: Body + Unpin, @@ -565,15 +453,11 @@ pub enum InternalHttpBodyFrame { impl From> for InternalHttpBodyFrame { fn from(frame: Frame) -> Self { - if frame.is_data() { - InternalHttpBodyFrame::Data(frame.into_data().expect("Malfromed data frame").to_vec()) - } else if frame.is_trailers() { - InternalHttpBodyFrame::Trailers( - frame.into_trailers().expect("Malfromed trailers frame"), - ) - } else { - panic!("Malfromed frame type") - } + frame + .into_data() + .map(|bytes| Self::Data(bytes.into())) + .or_else(|frame| frame.into_trailers().map(Self::Trailers)) + .expect("malformed frame type") } } @@ -591,463 +475,28 @@ impl fmt::Debug for InternalHttpBodyFrame { } } -pub type ReceiverStreamBody = StreamBody>>>; - -#[derive(Debug)] -pub enum HttpResponseFallback { - Framed(HttpResponse), - Fallback(HttpResponse>), - - /// Holds the [`HttpResponse`] that we're supposed to send back to the agent. - /// - /// It also holds the original http request [`HttpRequestFallback`], so we can retry - /// if our hyper server sent us a - /// [`RST_STREAM`](https://docs.rs/h2/latest/h2/struct.Error.html#method.is_reset). - Streamed( - HttpResponse, - Option, - ), -} - -impl HttpResponseFallback { - pub fn connection_id(&self) -> ConnectionId { - match self { - HttpResponseFallback::Framed(req) => req.connection_id, - HttpResponseFallback::Fallback(req) => req.connection_id, - HttpResponseFallback::Streamed(req, _) => req.connection_id, - } - } - - pub fn request_id(&self) -> RequestId { - match self { - HttpResponseFallback::Framed(req) => req.request_id, - HttpResponseFallback::Fallback(req) => req.request_id, - HttpResponseFallback::Streamed(req, _) => req.request_id, - } - } - - #[tracing::instrument(level = Level::TRACE, err(level = Level::WARN))] - pub fn into_hyper(self) -> Result>, http::Error> - where - E: From, - { - match self { - HttpResponseFallback::Framed(req) => req.internal_response.try_into(), - HttpResponseFallback::Fallback(req) => req.internal_response.try_into(), - HttpResponseFallback::Streamed(req, _) => req.internal_response.try_into(), - } - } - - /// Produces an [`HttpResponseFallback`] to the given [`HttpRequestFallback`]. - /// - /// # Note on picking response variant - /// - /// Variant of returned [`HttpResponseFallback`] is picked based on the variant of given - /// [`HttpRequestFallback`] and agent protocol version. We need to consider both due - /// to: - /// 1. Old agent versions always responding with client's `mirrord_protocol` version to - /// [`ClientMessage::SwitchProtocolVersion`](super::ClientMessage::SwitchProtocolVersion), - /// 2. [`LayerTcpSteal::HttpResponseChunked`] being introduced after - /// [`DaemonTcp::HttpRequestChunked`]. - pub fn response_from_request( - request: HttpRequestFallback, - status: StatusCode, - message: &str, - agent_protocol_version: Option<&semver::Version>, - ) -> Self { - let agent_supports_streaming_response = agent_protocol_version - .map(|version| HTTP_CHUNKED_RESPONSE_VERSION.matches(version)) - .unwrap_or(false); - - match request.clone() { - // We received `DaemonTcp::HttpRequestFramed` from the agent, - // so we know it supports `LayerTcpSteal::HttpResponseFramed` (both were introduced in - // the same `mirrord_protocol` version). - HttpRequestFallback::Framed(request) => HttpResponseFallback::Framed( - HttpResponse::::response_from_request(request, status, message), - ), - - // We received `DaemonTcp::HttpRequest` from the agent, so we assume it only supports - // `LayerTcpSteal::HttpResponse`. - HttpRequestFallback::Fallback(request) => HttpResponseFallback::Fallback( - HttpResponse::>::response_from_request(request, status, message), - ), - - // We received `DaemonTcp::HttpRequestChunked` and the agent supports - // `LayerTcpSteal::HttpResponseChunked`. - HttpRequestFallback::Streamed { - request: streamed_request, - .. - } if agent_supports_streaming_response => HttpResponseFallback::Streamed( - HttpResponse::::response_from_request( - streamed_request, - status, - message, - ), - Some(request), - ), - - // We received `DaemonTcp::HttpRequestChunked` from the agent, - // but the agent does not support `LayerTcpSteal::HttpResponseChunked`. - // However, it must support the older `LayerTcpSteal::HttpResponseFramed` - // variant (was introduced before `DaemonTcp::HttpRequestChunked`). - HttpRequestFallback::Streamed { request, .. } => HttpResponseFallback::Framed( - HttpResponse::::response_from_request(request, status, message), - ), - } - } -} - #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] -#[bincode(bounds = "for<'de> Body: Serialize + Deserialize<'de>")] -pub struct HttpResponse { +#[bincode(bounds = "for<'de> B: Serialize + Deserialize<'de>")] +pub struct HttpResponse { /// This is used to make sure the response is sent in its turn, after responses to all earlier /// requests were already sent. pub port: Port, pub connection_id: ConnectionId, pub request_id: RequestId, #[bincode(with_serde)] - pub internal_response: InternalHttpResponse, -} - -impl HttpResponse { - /// We cannot implement this with the [`From`] trait as it doesn't support `async` conversions, - /// and we also need some extra parameters. - /// - /// So this is our alternative implementation to `From>`. - #[tracing::instrument(level = Level::TRACE, err(level = Level::WARN))] - pub async fn from_hyper_response( - response: Response, - port: Port, - connection_id: ConnectionId, - request_id: RequestId, - ) -> Result, hyper::Error> { - let ( - Parts { - status, - version, - headers, - .. - }, - body, - ) = response.into_parts(); - - let body = InternalHttpBody::from_body(body).await?; - - let internal_response = InternalHttpResponse { - status, - headers, - version, - body, - }; - - Ok(HttpResponse { - request_id, - port, - connection_id, - internal_response, - }) - } - - pub fn response_from_request( - request: HttpRequest, - status: StatusCode, - message: &str, - ) -> Self { - let HttpRequest { - internal_request: InternalHttpRequest { version, .. }, - connection_id, - request_id, - port, - } = request; - - let body = InternalHttpBody::from_bytes( - format!( - "{} {}\n{}\n", - status.as_str(), - status.canonical_reason().unwrap_or_default(), - message - ) - .as_bytes(), - ); - - Self { - port, - connection_id, - request_id, - internal_response: InternalHttpResponse { - status, - version, - headers: Default::default(), - body, - }, - } - } - - pub fn empty_response_from_request( - request: HttpRequest, - status: StatusCode, - ) -> Self { - let HttpRequest { - internal_request: InternalHttpRequest { version, .. }, - connection_id, - request_id, - port, - } = request; - - Self { - port, - connection_id, - request_id, - internal_response: InternalHttpResponse { - status, - version, - headers: Default::default(), - body: Default::default(), - }, - } - } -} - -impl HttpResponse> { - /// We cannot implement this with the [`From`] trait as it doesn't support `async` conversions, - /// and we also need some extra parameters. - /// - /// So this is our alternative implementation to `From>`. - #[tracing::instrument(level = Level::TRACE, err(level = Level::WARN))] - pub async fn from_hyper_response( - response: Response, - port: Port, - connection_id: ConnectionId, - request_id: RequestId, - ) -> Result>, hyper::Error> { - let ( - Parts { - status, - version, - headers, - .. - }, - body, - ) = response.into_parts(); - - let body = body.collect().await?.to_bytes().to_vec(); - - let internal_response = InternalHttpResponse { - status, - headers, - version, - body, - }; - - Ok(HttpResponse { - request_id, - port, - connection_id, - internal_response, - }) - } - - pub fn response_from_request( - request: HttpRequest>, - status: StatusCode, - message: &str, - ) -> Self { - let HttpRequest { - internal_request: InternalHttpRequest { version, .. }, - connection_id, - request_id, - port, - } = request; - - let body = format!( - "{} {}\n{}\n", - status.as_str(), - status.canonical_reason().unwrap_or_default(), - message - ) - .into_bytes(); - - Self { - port, - connection_id, - request_id, - internal_response: InternalHttpResponse { - status, - version, - headers: Default::default(), - body, - }, - } - } - - pub fn empty_response_from_request(request: HttpRequest>, status: StatusCode) -> Self { - let HttpRequest { - internal_request: InternalHttpRequest { version, .. }, - connection_id, - request_id, - port, - } = request; - - Self { - port, - connection_id, - request_id, - internal_response: InternalHttpResponse { - status, - version, - headers: Default::default(), - body: Default::default(), - }, - } - } + pub internal_response: InternalHttpResponse, } -impl HttpResponse { - #[tracing::instrument(level = Level::TRACE, err(level = Level::WARN))] - pub async fn from_hyper_response( - response: Response, - port: Port, - connection_id: ConnectionId, - request_id: RequestId, - ) -> Result, hyper::Error> { - let ( - Parts { - status, - version, - headers, - .. - }, - mut body, - ) = response.into_parts(); - - let frames = body.next_frames(true).await?; - let (tx, rx) = tokio::sync::mpsc::channel(frames.frames.len().max(12)); - for frame in frames.frames { - tx.try_send(Ok(frame)) - .expect("Channel is open, capacity sufficient") - } - if !frames.is_last { - tokio::spawn(async move { - while let Some(frame) = body.frame().await { - if tx.send(frame).await.is_err() { - return; - } - } - }); - }; - - let body = StreamBody::new(ReceiverStream::from(rx)); - - let internal_response = InternalHttpResponse { - status, - headers, - version, - body, - }; - - Ok(HttpResponse { - request_id, - port, - connection_id, - internal_response, - }) - } - - #[tracing::instrument(level = Level::TRACE, ret)] - pub fn response_from_request( - request: HttpRequest, - status: StatusCode, - message: &str, - ) -> Self { - let HttpRequest { - internal_request: InternalHttpRequest { version, .. }, - connection_id, - request_id, - port, - } = request; - - let (tx, rx) = tokio::sync::mpsc::channel(1); - let frame = Frame::data(Bytes::copy_from_slice(message.as_bytes())); - tx.try_send(Ok(frame)) - .expect("channel is open, capacity is sufficient"); - let body = StreamBody::new(ReceiverStream::new(rx)); - - Self { - port, - connection_id, - request_id, - internal_response: InternalHttpResponse { - status, - version, - headers: Default::default(), - body, - }, - } - } -} - -impl TryFrom> for Response> { - type Error = http::Error; - - fn try_from(value: InternalHttpResponse) -> Result { - let InternalHttpResponse { - status, - version, - headers, - body, - } = value; - - let mut builder = Response::builder().status(status).version(version); - if let Some(h) = builder.headers_mut() { - *h = headers; - } - - builder.body(BoxBody::new(body.map_err(|_| unreachable!()))) - } -} - -impl TryFrom>> for Response> { - type Error = http::Error; - - fn try_from(value: InternalHttpResponse>) -> Result { - let InternalHttpResponse { - status, - version, - headers, - body, - } = value; - - let mut builder = Response::builder().status(status).version(version); - if let Some(h) = builder.headers_mut() { - *h = headers; - } - - builder.body(BoxBody::new( - Full::new(Bytes::from(body)).map_err(|_| unreachable!()), - )) - } -} - -impl TryFrom> for Response> -where - E: From, -{ - type Error = http::Error; - - fn try_from(value: InternalHttpResponse) -> Result { - let InternalHttpResponse { - status, - version, - headers, - body, - } = value; - - let mut builder = Response::builder().status(status).version(version); - if let Some(h) = builder.headers_mut() { - *h = headers; +impl HttpResponse { + pub fn map_body(self, cb: F) -> HttpResponse + where + F: FnOnce(B) -> T, + { + HttpResponse { + connection_id: self.connection_id, + request_id: self.request_id, + port: self.port, + internal_response: self.internal_response.map_body(cb), } - - builder.body(BoxBody::new(body.map_err(|e| e.into()))) } } diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index 31b5f668a2b..bf417813cf0 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -1,21 +1,17 @@ -#![allow(dead_code, unused)] #[cfg(test)] mod steal_tests { - use std::{ - io::{BufRead, BufReader, Read, Write}, - net::{SocketAddr, TcpStream}, - path::Path, - time::Duration, - }; + use std::{net::SocketAddr, path::Path, time::Duration}; use futures_util::{SinkExt, StreamExt}; - use hyper::{body, client::conn, Request, StatusCode}; - use hyper_util::rt::TokioIo; use k8s_openapi::api::core::v1::Pod; use kube::{Api, Client}; use reqwest::{header::HeaderMap, Url}; use rstest::*; - use tokio::time::sleep; + use tokio::{ + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}, + net::TcpStream, + time::sleep, + }; use tokio_tungstenite::{ connect_async, tungstenite::{client::IntoClientRequest, Message}, @@ -280,7 +276,7 @@ mod steal_tests { .wait_for_line(Duration::from_secs(40), "daemon subscribed") .await; - let mut tcp_stream = TcpStream::connect((addr, port as u16)).unwrap(); + let mut tcp_stream = TcpStream::connect((addr, port as u16)).await.unwrap(); // Wait for the test app to close the socket and tell us about it. process @@ -289,10 +285,10 @@ mod steal_tests { const DATA: &[u8; 16] = b"upper me please\n"; - tcp_stream.write_all(DATA).unwrap(); + tcp_stream.write_all(DATA).await.unwrap(); let mut response = [0u8; DATA.len()]; - tcp_stream.read_exact(&mut response).unwrap(); + tcp_stream.read_exact(&mut response).await.unwrap(); process .write_to_stdin(b"Hey test app, please stop running and just exit successfuly.\n") @@ -625,11 +621,11 @@ mod steal_tests { .await; let addr = SocketAddr::new(host.trim().parse().unwrap(), port as u16); - let mut stream = TcpStream::connect(addr).unwrap(); - stream.write_all(tcp_data.as_bytes()).unwrap(); + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(tcp_data.as_bytes()).await.unwrap(); let mut reader = BufReader::new(stream); let mut buf = String::new(); - reader.read_line(&mut buf).unwrap(); + reader.read_line(&mut buf).await.unwrap(); println!("Got response: {buf}"); // replace "remote: " with empty string, since the response can be split into frames // and we just need assert the final response From 63bd25775bf7a90af39db4340ce3037ff2a63d3b Mon Sep 17 00:00:00 2001 From: Facundo Date: Tue, 21 Jan 2025 13:16:44 -0300 Subject: [PATCH 17/23] Add rmdir support (#2985) * Add rmdir support * Fix lint * Fix rmdir.c * Fix rmdir on go * SYS_rmdir doesn't exist on aarch64 * Add unlink/unlinkat support * Check protocol supported version on unlink/unlinkat requests * Revert message_bus.send * PR review * Fix && fmt * lint * PR comments --- Cargo.lock | 2 +- changelog.d/2221.added.md | 1 + mirrord/agent/src/file.rs | 66 ++++++++++++++++++- mirrord/intproxy/protocol/src/lib.rs | 21 ++++++ mirrord/intproxy/src/proxies/files.rs | 88 +++++++++++--------------- mirrord/layer/src/file/hooks.rs | 49 +++++++++++++- mirrord/layer/src/file/ops.rs | 71 +++++++++++++++++++-- mirrord/layer/src/go/linux_x64.rs | 5 ++ mirrord/layer/src/go/mod.rs | 5 ++ mirrord/layer/tests/apps/mkdir/mkdir.c | 1 - mirrord/layer/tests/apps/rmdir/rmdir.c | 20 ++++++ mirrord/layer/tests/common/mod.rs | 23 +++++++ mirrord/layer/tests/mkdir.rs | 4 +- mirrord/layer/tests/rmdir.rs | 31 +++++++++ mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/codec.rs | 5 ++ mirrord/protocol/src/file.rs | 23 +++++++ tests/go-e2e-dir/main.go | 6 ++ tests/python-e2e/ops.py | 11 +++- 19 files changed, 372 insertions(+), 62 deletions(-) create mode 100644 changelog.d/2221.added.md create mode 100644 mirrord/layer/tests/apps/rmdir/rmdir.c create mode 100644 mirrord/layer/tests/rmdir.rs diff --git a/Cargo.lock b/Cargo.lock index b6e72453953..3ce40063e35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.13.4" +version = "1.14.0" dependencies = [ "actix-codec", "bincode", diff --git a/changelog.d/2221.added.md b/changelog.d/2221.added.md new file mode 100644 index 00000000000..fe396a1ac03 --- /dev/null +++ b/changelog.d/2221.added.md @@ -0,0 +1 @@ +Add rmdir / unlink / unlinkat support diff --git a/mirrord/agent/src/file.rs b/mirrord/agent/src/file.rs index 54432a9779b..0bc30afb151 100644 --- a/mirrord/agent/src/file.rs +++ b/mirrord/agent/src/file.rs @@ -5,13 +5,17 @@ use std::{ io::{self, prelude::*, BufReader, SeekFrom}, iter::{Enumerate, Peekable}, ops::RangeInclusive, - os::unix::{fs::MetadataExt, prelude::FileExt}, + os::{ + fd::RawFd, + unix::{fs::MetadataExt, prelude::FileExt}, + }, path::{Path, PathBuf}, }; use faccess::{AccessMode, PathExt}; use libc::DT_DIR; use mirrord_protocol::{file::*, FileRequest, FileResponse, RemoteResult, ResponseError}; +use nix::unistd::UnlinkatFlags; use tracing::{error, trace, Level}; use crate::error::Result; @@ -258,6 +262,17 @@ impl FileManager { pathname, mode, }) => Some(FileResponse::MakeDir(self.mkdirat(dirfd, &pathname, mode))), + FileRequest::RemoveDir(RemoveDirRequest { pathname }) => { + Some(FileResponse::RemoveDir(self.rmdir(&pathname))) + } + FileRequest::Unlink(UnlinkRequest { pathname }) => { + Some(FileResponse::Unlink(self.unlink(&pathname))) + } + FileRequest::UnlinkAt(UnlinkAtRequest { + dirfd, + pathname, + flags, + }) => Some(FileResponse::Unlink(self.unlinkat(dirfd, &pathname, flags))), }) } @@ -520,6 +535,55 @@ impl FileManager { } } + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(crate) fn rmdir(&mut self, path: &Path) -> RemoteResult<()> { + let path = resolve_path(path, &self.root_path)?; + + std::fs::remove_dir(path.as_path()).map_err(ResponseError::from) + } + + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(crate) fn unlink(&mut self, path: &Path) -> RemoteResult<()> { + let path = resolve_path(path, &self.root_path)?; + + nix::unistd::unlink(path.as_path()) + .map_err(|error| ResponseError::from(std::io::Error::from_raw_os_error(error as i32))) + } + + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(crate) fn unlinkat( + &mut self, + dirfd: Option, + path: &Path, + flags: u32, + ) -> RemoteResult<()> { + let path = match dirfd { + Some(dirfd) => { + let relative_dir = self + .open_files + .get(&dirfd) + .ok_or(ResponseError::NotFound(dirfd))?; + + if let RemoteFile::Directory(relative_dir) = relative_dir { + relative_dir.join(path) + } else { + return Err(ResponseError::NotDirectory(dirfd)); + } + } + None => resolve_path(path, &self.root_path)?, + }; + + let flags = match flags { + 0 => UnlinkatFlags::RemoveDir, + _ => UnlinkatFlags::NoRemoveDir, + }; + + let fd: Option = dirfd.map(|fd| fd as RawFd); + + nix::unistd::unlinkat(fd, path.as_path(), flags) + .map_err(|error| ResponseError::from(std::io::Error::from_raw_os_error(error as i32))) + } + pub(crate) fn seek(&mut self, fd: u64, seek_from: SeekFrom) -> RemoteResult { trace!( "FileManager::seek -> fd {:#?} | seek_from {:#?}", diff --git a/mirrord/intproxy/protocol/src/lib.rs b/mirrord/intproxy/protocol/src/lib.rs index 623020718b6..69a551b50bc 100644 --- a/mirrord/intproxy/protocol/src/lib.rs +++ b/mirrord/intproxy/protocol/src/lib.rs @@ -324,6 +324,27 @@ impl_request!( res_path = ProxyToLayerMessage::File => FileResponse::MakeDir, ); +impl_request!( + req = RemoveDirRequest, + res = RemoteResult<()>, + req_path = LayerToProxyMessage::File => FileRequest::RemoveDir, + res_path = ProxyToLayerMessage::File => FileResponse::RemoveDir, +); + +impl_request!( + req = UnlinkRequest, + res = RemoteResult<()>, + req_path = LayerToProxyMessage::File => FileRequest::Unlink, + res_path = ProxyToLayerMessage::File => FileResponse::Unlink, +); + +impl_request!( + req = UnlinkAtRequest, + res = RemoteResult<()>, + req_path = LayerToProxyMessage::File => FileRequest::UnlinkAt, + res_path = ProxyToLayerMessage::File => FileResponse::Unlink, +); + impl_request!( req = SeekFileRequest, res = RemoteResult, diff --git a/mirrord/intproxy/src/proxies/files.rs b/mirrord/intproxy/src/proxies/files.rs index 79ac6695575..517c743ce12 100644 --- a/mirrord/intproxy/src/proxies/files.rs +++ b/mirrord/intproxy/src/proxies/files.rs @@ -6,7 +6,7 @@ use mirrord_protocol::{ file::{ CloseDirRequest, CloseFileRequest, DirEntryInternal, ReadDirBatchRequest, ReadDirResponse, ReadFileResponse, ReadLimitedFileRequest, SeekFromInternal, MKDIR_VERSION, - READDIR_BATCH_VERSION, READLINK_VERSION, + READDIR_BATCH_VERSION, READLINK_VERSION, RMDIR_VERSION, }, ClientMessage, DaemonMessage, ErrorKindInternal, FileRequest, FileResponse, RemoteIOError, ResponseError, @@ -253,6 +253,31 @@ impl FilesProxy { self.protocol_version.replace(version); } + /// Checks if the mirrord protocol version supports this [`FileRequest`]. + fn is_request_supported(&self, request: &FileRequest) -> Result<(), FileResponse> { + let protocol_version = self.protocol_version.as_ref(); + + match request { + FileRequest::ReadLink(..) + if protocol_version.is_some_and(|version| !READLINK_VERSION.matches(version)) => + { + Err(FileResponse::ReadLink(Err(ResponseError::NotImplemented))) + } + FileRequest::MakeDir(..) | FileRequest::MakeDirAt(..) + if protocol_version.is_some_and(|version| !MKDIR_VERSION.matches(version)) => + { + Err(FileResponse::MakeDir(Err(ResponseError::NotImplemented))) + } + FileRequest::RemoveDir(..) | FileRequest::Unlink(..) | FileRequest::UnlinkAt(..) + if protocol_version + .is_some_and(|version: &Version| !RMDIR_VERSION.matches(version)) => + { + Err(FileResponse::RemoveDir(Err(ResponseError::NotImplemented))) + } + _ => Ok(()), + } + } + // #[tracing::instrument(level = Level::TRACE, skip(message_bus))] async fn file_request( &mut self, @@ -261,6 +286,18 @@ impl FilesProxy { message_id: MessageId, message_bus: &mut MessageBus, ) { + // Not supported in old `mirrord-protocol` versions. + if let Err(response) = self.is_request_supported(&request) { + message_bus + .send(ToLayer { + message_id, + layer_id, + message: ProxyToLayerMessage::File(response), + }) + .await; + return; + } + match request { // Should trigger remote close only when the fd is closed in all layer instances. FileRequest::Close(close) => { @@ -454,31 +491,6 @@ impl FilesProxy { } }, - // Not supported in old `mirrord-protocol` versions. - req @ FileRequest::ReadLink(..) => { - let supported = self - .protocol_version - .as_ref() - .is_some_and(|version| READLINK_VERSION.matches(version)); - - if supported { - self.request_queue.push_back(message_id, layer_id); - message_bus - .send(ProxyMessage::ToAgent(ClientMessage::FileRequest(req))) - .await; - } else { - message_bus - .send(ToLayer { - message_id, - message: ProxyToLayerMessage::File(FileResponse::ReadLink(Err( - ResponseError::NotImplemented, - ))), - layer_id, - }) - .await; - } - } - // Should only be sent from intproxy, not from the layer. FileRequest::ReadDirBatch(..) => { unreachable!("ReadDirBatch request is never sent from the layer"); @@ -522,30 +534,6 @@ impl FilesProxy { .await; } - FileRequest::MakeDir(_) | FileRequest::MakeDirAt(_) => { - let supported = self - .protocol_version - .as_ref() - .is_some_and(|version| MKDIR_VERSION.matches(version)); - - if supported { - self.request_queue.push_back(message_id, layer_id); - message_bus - .send(ProxyMessage::ToAgent(ClientMessage::FileRequest(request))) - .await; - } else { - let file_response = FileResponse::MakeDir(Err(ResponseError::NotImplemented)); - - message_bus - .send(ToLayer { - message_id, - message: ProxyToLayerMessage::File(file_response), - layer_id, - }) - .await; - } - } - // Doesn't require any special logic. other => { self.request_queue.push_back(message_id, layer_id); diff --git a/mirrord/layer/src/file/hooks.rs b/mirrord/layer/src/file/hooks.rs index 4de1e73577f..7c46165b37a 100644 --- a/mirrord/layer/src/file/hooks.rs +++ b/mirrord/layer/src/file/hooks.rs @@ -1088,6 +1088,43 @@ pub(crate) unsafe extern "C" fn mkdirat_detour( }) } +/// Hook for `libc::rmdir`. +#[hook_guard_fn] +pub(crate) unsafe extern "C" fn rmdir_detour(pathname: *const c_char) -> c_int { + rmdir(pathname.checked_into()) + .map(|()| 0) + .unwrap_or_bypass_with(|bypass| { + let raw_path = update_ptr_from_bypass(pathname, &bypass); + FN_RMDIR(raw_path) + }) +} + +/// Hook for `libc::unlink`. +#[hook_guard_fn] +pub(crate) unsafe extern "C" fn unlink_detour(pathname: *const c_char) -> c_int { + unlink(pathname.checked_into()) + .map(|()| 0) + .unwrap_or_bypass_with(|bypass| { + let raw_path = update_ptr_from_bypass(pathname, &bypass); + FN_UNLINK(raw_path) + }) +} + +/// Hook for `libc::unlinkat`. +#[hook_guard_fn] +pub(crate) unsafe extern "C" fn unlinkat_detour( + dirfd: c_int, + pathname: *const c_char, + flags: u32, +) -> c_int { + unlinkat(dirfd, pathname.checked_into(), flags) + .map(|()| 0) + .unwrap_or_bypass_with(|bypass| { + let raw_path = update_ptr_from_bypass(pathname, &bypass); + FN_UNLINKAT(dirfd, raw_path, flags) + }) +} + /// Convenience function to setup file hooks (`x_detour`) with `frida_gum`. pub(crate) unsafe fn enable_file_hooks(hook_manager: &mut HookManager) { replace!(hook_manager, "open", open_detour, FnOpen, FN_OPEN); @@ -1163,7 +1200,6 @@ pub(crate) unsafe fn enable_file_hooks(hook_manager: &mut HookManager) { ); replace!(hook_manager, "mkdir", mkdir_detour, FnMkdir, FN_MKDIR); - replace!( hook_manager, "mkdirat", @@ -1172,6 +1208,17 @@ pub(crate) unsafe fn enable_file_hooks(hook_manager: &mut HookManager) { FN_MKDIRAT ); + replace!(hook_manager, "rmdir", rmdir_detour, FnRmdir, FN_RMDIR); + + replace!(hook_manager, "unlink", unlink_detour, FnUnlink, FN_UNLINK); + replace!( + hook_manager, + "unlinkat", + unlinkat_detour, + FnUnlinkat, + FN_UNLINKAT + ); + replace!(hook_manager, "lseek", lseek_detour, FnLseek, FN_LSEEK); replace!(hook_manager, "write", write_detour, FnWrite, FN_WRITE); diff --git a/mirrord/layer/src/file/ops.rs b/mirrord/layer/src/file/ops.rs index ca9dd10f951..bac20ad7cd9 100644 --- a/mirrord/layer/src/file/ops.rs +++ b/mirrord/layer/src/file/ops.rs @@ -4,12 +4,12 @@ use std::{env, ffi::CString, io::SeekFrom, os::unix::io::RawFd, path::PathBuf}; #[cfg(target_os = "linux")] use libc::{c_char, statx, statx_timestamp}; -use libc::{c_int, iovec, unlink, AT_FDCWD}; +use libc::{c_int, iovec, AT_FDCWD}; use mirrord_protocol::{ file::{ MakeDirAtRequest, MakeDirRequest, OpenFileRequest, OpenFileResponse, OpenOptionsInternal, - ReadFileResponse, ReadLinkFileRequest, ReadLinkFileResponse, SeekFileResponse, - WriteFileResponse, XstatFsResponse, XstatResponse, + ReadFileResponse, ReadLinkFileRequest, ReadLinkFileResponse, RemoveDirRequest, + SeekFileResponse, UnlinkAtRequest, WriteFileResponse, XstatFsResponse, XstatResponse, }, ResponseError, }; @@ -157,7 +157,7 @@ fn create_local_fake_file(remote_fd: u64) -> Detour { close_remote_file_on_failure(remote_fd)?; Detour::Error(HookError::LocalFileCreation(remote_fd, error.0)) } else { - unsafe { unlink(file_path_ptr) }; + unsafe { libc::unlink(file_path_ptr) }; Detour::Success(local_file_fd) } } @@ -392,6 +392,69 @@ pub(crate) fn mkdirat(dirfd: RawFd, pathname: Detour, mode: u32) -> Det } } +#[mirrord_layer_macro::instrument(level = Level::TRACE, ret)] +pub(crate) fn rmdir(pathname: Detour) -> Detour<()> { + let pathname = pathname?; + + check_relative_paths!(pathname); + + let path = remap_path!(pathname); + + ensure_not_ignored!(path, false); + + let rmdir = RemoveDirRequest { pathname: path }; + + // `NotImplemented` error here means that the protocol doesn't support it. + match common::make_proxy_request_with_response(rmdir)? { + Ok(response) => Detour::Success(response), + Err(ResponseError::NotImplemented) => Detour::Bypass(Bypass::NotImplemented), + Err(fail) => Detour::Error(fail.into()), + } +} + +#[mirrord_layer_macro::instrument(level = Level::TRACE, ret)] +pub(crate) fn unlink(pathname: Detour) -> Detour<()> { + let pathname = pathname?; + + check_relative_paths!(pathname); + + let path = remap_path!(pathname); + + ensure_not_ignored!(path, false); + + let unlink = RemoveDirRequest { pathname: path }; + + // `NotImplemented` error here means that the protocol doesn't support it. + match common::make_proxy_request_with_response(unlink)? { + Ok(response) => Detour::Success(response), + Err(ResponseError::NotImplemented) => Detour::Bypass(Bypass::NotImplemented), + Err(fail) => Detour::Error(fail.into()), + } +} + +#[mirrord_layer_macro::instrument(level = Level::TRACE, ret)] +pub(crate) fn unlinkat(dirfd: RawFd, pathname: Detour, flags: u32) -> Detour<()> { + let pathname = pathname?; + + let optional_dirfd = match pathname.is_absolute() { + true => None, + false => Some(get_remote_fd(dirfd)?), + }; + + let unlink = UnlinkAtRequest { + dirfd: optional_dirfd, + pathname: pathname.clone(), + flags, + }; + + // `NotImplemented` error here means that the protocol doesn't support it. + match common::make_proxy_request_with_response(unlink)? { + Ok(response) => Detour::Success(response), + Err(ResponseError::NotImplemented) => Detour::Bypass(Bypass::NotImplemented), + Err(fail) => Detour::Error(fail.into()), + } +} + pub(crate) fn pwrite(local_fd: RawFd, buffer: &[u8], offset: u64) -> Detour { let remote_fd = get_remote_fd(local_fd)?; trace!("pwrite: local_fd {local_fd}"); diff --git a/mirrord/layer/src/go/linux_x64.rs b/mirrord/layer/src/go/linux_x64.rs index 5acaceb13b2..18b36700cbe 100644 --- a/mirrord/layer/src/go/linux_x64.rs +++ b/mirrord/layer/src/go/linux_x64.rs @@ -344,6 +344,11 @@ unsafe extern "C" fn c_abi_syscall_handler( #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] libc::SYS_mkdir => mkdir_detour(param1 as _, param2 as _) as i64, libc::SYS_mkdirat => mkdirat_detour(param1 as _, param2 as _, param3 as _) as i64, + #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] + libc::SYS_rmdir => rmdir_detour(param1 as _) as i64, + #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] + libc::SYS_unlink => unlink_detour(param1 as _) as i64, + libc::SYS_unlinkat => unlinkat_detour(param1 as _, param2 as _, param3 as _) as i64, _ => { let (Ok(result) | Err(result)) = syscalls::syscall!( syscalls::Sysno::from(syscall as i32), diff --git a/mirrord/layer/src/go/mod.rs b/mirrord/layer/src/go/mod.rs index 003eed8692c..df810bbdcf9 100644 --- a/mirrord/layer/src/go/mod.rs +++ b/mirrord/layer/src/go/mod.rs @@ -113,6 +113,11 @@ unsafe extern "C" fn c_abi_syscall6_handler( #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] libc::SYS_mkdir => mkdir_detour(param1 as _, param2 as _) as i64, libc::SYS_mkdirat => mkdirat_detour(param1 as _, param2 as _, param3 as _) as i64, + #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] + libc::SYS_rmdir => rmdir_detour(param1 as _) as i64, + #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] + libc::SYS_unlink => unlink_detour(param1 as _) as i64, + libc::SYS_unlinkat => unlinkat_detour(param1 as _, param2 as _, param3 as _) as i64, _ => { let (Ok(result) | Err(result)) = syscalls::syscall!( syscalls::Sysno::from(syscall as i32), diff --git a/mirrord/layer/tests/apps/mkdir/mkdir.c b/mirrord/layer/tests/apps/mkdir/mkdir.c index 4253b6c089d..8631ae7a26a 100644 --- a/mirrord/layer/tests/apps/mkdir/mkdir.c +++ b/mirrord/layer/tests/apps/mkdir/mkdir.c @@ -1,6 +1,5 @@ #include #include -#include #include #include diff --git a/mirrord/layer/tests/apps/rmdir/rmdir.c b/mirrord/layer/tests/apps/rmdir/rmdir.c new file mode 100644 index 00000000000..f839e5256e7 --- /dev/null +++ b/mirrord/layer/tests/apps/rmdir/rmdir.c @@ -0,0 +1,20 @@ +#include +#include +#include +#include + +/// Test `rmdir`. +/// +/// Creates a folder and then removes it. +/// +int main() +{ + char *test_dir = "/test_dir"; + int mkdir_result = mkdir(test_dir, 0777); + assert(mkdir_result == 0); + + int rmdir_result = rmdir(test_dir); + assert(rmdir_result == 0); + + return 0; +} diff --git a/mirrord/layer/tests/common/mod.rs b/mirrord/layer/tests/common/mod.rs index d304642d28b..207f1c6c7a6 100644 --- a/mirrord/layer/tests/common/mod.rs +++ b/mirrord/layer/tests/common/mod.rs @@ -489,6 +489,25 @@ impl TestIntProxy { .unwrap(); } + /// Makes a [`FileRequest::RemoveDir`] and answers it. + pub async fn expect_remove_dir(&mut self, expected_dir_name: &str) { + // Expecting `rmdir` call with path. + assert_matches!( + self.recv().await, + ClientMessage::FileRequest(FileRequest::RemoveDir( + mirrord_protocol::file::RemoveDirRequest { pathname } + )) if pathname.to_str().unwrap() == expected_dir_name + ); + + // Answer `rmdir`. + self.codec + .send(DaemonMessage::File( + mirrord_protocol::FileResponse::RemoveDir(Ok(())), + )) + .await + .unwrap(); + } + /// Verify that the passed message (not the next message from self.codec!) is a file read. /// Return buffer size. pub async fn expect_message_file_read(message: ClientMessage, expected_fd: u64) -> u64 { @@ -765,6 +784,7 @@ pub enum Application { Fork, ReadLink, MakeDir, + RemoveDir, OpenFile, CIssue2055, CIssue2178, @@ -821,6 +841,7 @@ impl Application { Application::Fork => String::from("tests/apps/fork/out.c_test_app"), Application::ReadLink => String::from("tests/apps/readlink/out.c_test_app"), Application::MakeDir => String::from("tests/apps/mkdir/out.c_test_app"), + Application::RemoveDir => String::from("tests/apps/rmdir/out.c_test_app"), Application::Realpath => String::from("tests/apps/realpath/out.c_test_app"), Application::NodeHTTP | Application::NodeIssue2283 | Application::NodeIssue2807 => { String::from("node") @@ -1059,6 +1080,7 @@ impl Application { | Application::Fork | Application::ReadLink | Application::MakeDir + | Application::RemoveDir | Application::Realpath | Application::RustFileOps | Application::RustIssue1123 @@ -1137,6 +1159,7 @@ impl Application { | Application::Fork | Application::ReadLink | Application::MakeDir + | Application::RemoveDir | Application::Realpath | Application::Go21Issue834 | Application::Go22Issue834 diff --git a/mirrord/layer/tests/mkdir.rs b/mirrord/layer/tests/mkdir.rs index 5f0fe3301b3..76fa3e8936b 100644 --- a/mirrord/layer/tests/mkdir.rs +++ b/mirrord/layer/tests/mkdir.rs @@ -17,10 +17,10 @@ async fn mkdir(dylib_path: &Path) { .start_process_with_layer(dylib_path, Default::default(), None) .await; - println!("waiting for file request."); + println!("waiting for MakeDirRequest."); intproxy.expect_make_dir("/mkdir_test_path", 0o777).await; - println!("waiting for file request."); + println!("waiting for MakeDirRequest."); intproxy.expect_make_dir("/mkdirat_test_path", 0o777).await; assert_eq!(intproxy.try_recv().await, None); diff --git a/mirrord/layer/tests/rmdir.rs b/mirrord/layer/tests/rmdir.rs new file mode 100644 index 00000000000..910e3d18ff9 --- /dev/null +++ b/mirrord/layer/tests/rmdir.rs @@ -0,0 +1,31 @@ +#![feature(assert_matches)] +use std::{path::Path, time::Duration}; + +use rstest::rstest; + +mod common; +pub use common::*; + +/// Test for the [`libc::rmdir`] function. +#[rstest] +#[tokio::test] +#[timeout(Duration::from_secs(60))] +async fn rmdir(dylib_path: &Path) { + let application = Application::RemoveDir; + + let (mut test_process, mut intproxy) = application + .start_process_with_layer(dylib_path, Default::default(), None) + .await; + + println!("waiting for MakeDirRequest."); + intproxy.expect_make_dir("/test_dir", 0o777).await; + + println!("waiting for RemoveDirRequest."); + intproxy.expect_remove_dir("/test_dir").await; + + assert_eq!(intproxy.try_recv().await, None); + + test_process.wait_assert_success().await; + test_process.assert_no_error_in_stderr().await; + test_process.assert_no_error_in_stdout().await; +} diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index b7aaaca403a..276d5f164f4 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.13.4" +version = "1.14.0" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/codec.rs b/mirrord/protocol/src/codec.rs index 81e79bf5db7..1b0975d350d 100644 --- a/mirrord/protocol/src/codec.rs +++ b/mirrord/protocol/src/codec.rs @@ -91,6 +91,9 @@ pub enum FileRequest { ReadDirBatch(ReadDirBatchRequest), MakeDir(MakeDirRequest), MakeDirAt(MakeDirAtRequest), + RemoveDir(RemoveDirRequest), + Unlink(UnlinkRequest), + UnlinkAt(UnlinkAtRequest), } /// Minimal mirrord-protocol version that allows `ClientMessage::ReadyForLogs` message. @@ -136,6 +139,8 @@ pub enum FileResponse { ReadLink(RemoteResult), ReadDirBatch(RemoteResult), MakeDir(RemoteResult<()>), + RemoveDir(RemoteResult<()>), + Unlink(RemoteResult<()>), } /// `-agent` --> `-layer` messages. diff --git a/mirrord/protocol/src/file.rs b/mirrord/protocol/src/file.rs index c0b4cfe2f18..9a8622731fb 100644 --- a/mirrord/protocol/src/file.rs +++ b/mirrord/protocol/src/file.rs @@ -22,9 +22,15 @@ pub static READLINK_VERSION: LazyLock = pub static READDIR_BATCH_VERSION: LazyLock = LazyLock::new(|| ">=1.9.0".parse().expect("Bad Identifier")); +/// Minimal mirrord-protocol version that allows [`MakeDirRequest`] and [`MakeDirAtRequest`]. pub static MKDIR_VERSION: LazyLock = LazyLock::new(|| ">=1.13.0".parse().expect("Bad Identifier")); +/// Minimal mirrord-protocol version that allows [`RemoveDirRequest`], [`UnlinkRequest`] and +/// [`UnlinkAtRequest`].. +pub static RMDIR_VERSION: LazyLock = + LazyLock::new(|| ">=1.14.0".parse().expect("Bad Identifier")); + pub static OPEN_LOCAL_VERSION: LazyLock = LazyLock::new(|| ">=1.13.3".parse().expect("Bad Identifier")); @@ -285,6 +291,23 @@ pub struct MakeDirAtRequest { pub mode: u32, } +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub struct RemoveDirRequest { + pub pathname: PathBuf, +} + +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub struct UnlinkRequest { + pub pathname: PathBuf, +} + +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub struct UnlinkAtRequest { + pub dirfd: Option, + pub pathname: PathBuf, + pub flags: u32, +} + #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub struct ReadLimitedFileRequest { pub remote_fd: u64, diff --git a/tests/go-e2e-dir/main.go b/tests/go-e2e-dir/main.go index 80006ce58c5..4a505a4dcc8 100644 --- a/tests/go-e2e-dir/main.go +++ b/tests/go-e2e-dir/main.go @@ -37,6 +37,12 @@ func main() { os.Exit(-1) } + err = os.Remove("/app/test_mkdir") + if err != nil { + fmt.Printf("Rmdir error: %s\n", err) + os.Exit(-1) + } + // let close requests be sent for test time.Sleep(1 * time.Second) os.Exit(0) diff --git a/tests/python-e2e/ops.py b/tests/python-e2e/ops.py index 9f94425921e..c107ebb2375 100644 --- a/tests/python-e2e/ops.py +++ b/tests/python-e2e/ops.py @@ -87,7 +87,16 @@ def test_mkdir_errors(self): os.mkdir("test_mkdir_error_already_exists", dir_fd=dir) os.close(dir) - + + def test_rmdir(self): + """ + Creates a new directory in "/tmp" and removes it using rmdir. + """ + os.mkdir("/tmp/test_rmdir") + self.assertTrue(os.path.isdir("/tmp/test_rmdir")) + os.rmdir("/tmp/test_rmdir") + self.assertFalse(os.path.isdir("/tmp/test_rmdir")) + def _create_new_tmp_file(self): """ Creates a new file in /tmp and returns the path and name of the file. From b99ea5fbcae2946a8ecb3932fab4a4412d62f445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Tue, 21 Jan 2025 19:27:44 +0100 Subject: [PATCH 18/23] 3.130.0 (#3027) * 3.130.0 * Fixed mirrord ext * Fix on macos --- CHANGELOG.md | 34 +++++++++++ Cargo.lock | 56 +++++++++---------- Cargo.toml | 2 +- changelog.d/+added-log-leve.internal.md | 1 - .../+improve-env-config-docs.changed.md | 1 - changelog.d/2221.added.md | 1 - changelog.d/2936.fixed.md | 1 - changelog.d/3013.fixed.md | 2 - changelog.d/3024.fixed.md | 1 - mirrord/cli/src/execution.rs | 9 +-- mirrord/cli/src/extension.rs | 20 ++----- mirrord/config/src/lib.rs | 8 +-- 12 files changed, 78 insertions(+), 58 deletions(-) delete mode 100644 changelog.d/+added-log-leve.internal.md delete mode 100644 changelog.d/+improve-env-config-docs.changed.md delete mode 100644 changelog.d/2221.added.md delete mode 100644 changelog.d/2936.fixed.md delete mode 100644 changelog.d/3013.fixed.md delete mode 100644 changelog.d/3024.fixed.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 87bf68e74a9..0e23be72090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,40 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang +## [3.130.0](https://github.com/metalbear-co/mirrord/tree/3.130.0) - 2025-01-21 + + +### Added + +- Added support for `rmdir`, `unlink` and `unlinkat`. + [#2221](https://github.com/metalbear-co/mirrord/issues/2221) + + +### Changed + +- Updated `configuration.md` and improved `.feature.env.mapping` doc. + + +### Fixed + +- Stopped mirrord entering a crash loop when trying to load into some processes + like VSCode's `watchdog.js` when the user config contained a call to + `get_env()`, which occurred due to missing env - the config is now only + rendered once and set into an env var. + [#2936](https://github.com/metalbear-co/mirrord/issues/2936) +- Fixed an issue where HTTP requests stolen with a filter would hang with a + single-threaded local HTTP server. + Improved handling of incoming connections on the local machine (e.g + introduces reuse of local HTTP connections). + [#3013](https://github.com/metalbear-co/mirrord/issues/3013) +- Moved to an older builder base image for aarch64 to support centos-7 libc. + [#3024](https://github.com/metalbear-co/mirrord/issues/3024) + + +### Internal + +- Extended `mirrord-protocol` with info logs from the agent. + ## [3.129.0](https://github.com/metalbear-co/mirrord/tree/3.129.0) - 2025-01-14 diff --git a/Cargo.lock b/Cargo.lock index 3ce40063e35..fd42ec7bb37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2400,7 +2400,7 @@ dependencies = [ [[package]] name = "fileops" -version = "3.129.0" +version = "3.130.0" dependencies = [ "libc", ] @@ -3518,7 +3518,7 @@ checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "issue1317" -version = "3.129.0" +version = "3.130.0" dependencies = [ "actix-web", "env_logger 0.11.6", @@ -3528,7 +3528,7 @@ dependencies = [ [[package]] name = "issue1776" -version = "3.129.0" +version = "3.130.0" dependencies = [ "errno 0.3.10", "libc", @@ -3537,7 +3537,7 @@ dependencies = [ [[package]] name = "issue1776portnot53" -version = "3.129.0" +version = "3.130.0" dependencies = [ "libc", "socket2", @@ -3545,14 +3545,14 @@ dependencies = [ [[package]] name = "issue1899" -version = "3.129.0" +version = "3.130.0" dependencies = [ "libc", ] [[package]] name = "issue2001" -version = "3.129.0" +version = "3.130.0" dependencies = [ "libc", ] @@ -3873,7 +3873,7 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "listen_ports" -version = "3.129.0" +version = "3.130.0" [[package]] name = "litemap" @@ -4114,7 +4114,7 @@ dependencies = [ [[package]] name = "mirrord" -version = "3.129.0" +version = "3.130.0" dependencies = [ "actix-codec", "clap", @@ -4170,7 +4170,7 @@ dependencies = [ [[package]] name = "mirrord-agent" -version = "3.129.0" +version = "3.130.0" dependencies = [ "actix-codec", "async-trait", @@ -4225,7 +4225,7 @@ dependencies = [ [[package]] name = "mirrord-analytics" -version = "3.129.0" +version = "3.130.0" dependencies = [ "assert-json-diff", "base64 0.22.1", @@ -4239,7 +4239,7 @@ dependencies = [ [[package]] name = "mirrord-auth" -version = "3.129.0" +version = "3.130.0" dependencies = [ "bcder", "chrono", @@ -4260,7 +4260,7 @@ dependencies = [ [[package]] name = "mirrord-config" -version = "3.129.0" +version = "3.130.0" dependencies = [ "base64 0.22.1", "bimap", @@ -4284,7 +4284,7 @@ dependencies = [ [[package]] name = "mirrord-config-derive" -version = "3.129.0" +version = "3.130.0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -4294,7 +4294,7 @@ dependencies = [ [[package]] name = "mirrord-console" -version = "3.129.0" +version = "3.130.0" dependencies = [ "bincode", "drain", @@ -4310,7 +4310,7 @@ dependencies = [ [[package]] name = "mirrord-intproxy" -version = "3.129.0" +version = "3.130.0" dependencies = [ "bytes", "exponential-backoff", @@ -4338,7 +4338,7 @@ dependencies = [ [[package]] name = "mirrord-intproxy-protocol" -version = "3.129.0" +version = "3.130.0" dependencies = [ "bincode", "mirrord-protocol", @@ -4348,7 +4348,7 @@ dependencies = [ [[package]] name = "mirrord-kube" -version = "3.129.0" +version = "3.130.0" dependencies = [ "actix-codec", "async-stream", @@ -4372,7 +4372,7 @@ dependencies = [ [[package]] name = "mirrord-layer" -version = "3.129.0" +version = "3.130.0" dependencies = [ "actix-codec", "base64 0.22.1", @@ -4415,7 +4415,7 @@ dependencies = [ [[package]] name = "mirrord-layer-macro" -version = "3.129.0" +version = "3.130.0" dependencies = [ "proc-macro2", "quote", @@ -4424,7 +4424,7 @@ dependencies = [ [[package]] name = "mirrord-macros" -version = "3.129.0" +version = "3.130.0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", @@ -4434,7 +4434,7 @@ dependencies = [ [[package]] name = "mirrord-operator" -version = "3.129.0" +version = "3.130.0" dependencies = [ "base64 0.22.1", "bincode", @@ -4467,7 +4467,7 @@ dependencies = [ [[package]] name = "mirrord-progress" -version = "3.129.0" +version = "3.130.0" dependencies = [ "enum_dispatch", "indicatif", @@ -4501,7 +4501,7 @@ dependencies = [ [[package]] name = "mirrord-sip" -version = "3.129.0" +version = "3.130.0" dependencies = [ "apple-codesign", "object 0.36.7", @@ -4514,7 +4514,7 @@ dependencies = [ [[package]] name = "mirrord-vpn" -version = "3.129.0" +version = "3.130.0" dependencies = [ "futures", "ipnet", @@ -4862,7 +4862,7 @@ dependencies = [ [[package]] name = "outgoing" -version = "3.129.0" +version = "3.130.0" [[package]] name = "outref" @@ -5923,14 +5923,14 @@ dependencies = [ [[package]] name = "rust-bypassed-unix-socket" -version = "3.129.0" +version = "3.130.0" dependencies = [ "tokio", ] [[package]] name = "rust-e2e-fileops" -version = "3.129.0" +version = "3.130.0" dependencies = [ "libc", ] @@ -5946,7 +5946,7 @@ dependencies = [ [[package]] name = "rust-unix-socket-client" -version = "3.129.0" +version = "3.130.0" dependencies = [ "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 9a2b2a42cae..9f01963dc6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ resolver = "2" # latest commits on rustls suppress certificate verification [workspace.package] -version = "3.129.0" +version = "3.130.0" edition = "2021" license = "MIT" readme = "README.md" diff --git a/changelog.d/+added-log-leve.internal.md b/changelog.d/+added-log-leve.internal.md deleted file mode 100644 index 604957c5641..00000000000 --- a/changelog.d/+added-log-leve.internal.md +++ /dev/null @@ -1 +0,0 @@ -Extended `mirrord-protocol` with info logs from the agent. \ No newline at end of file diff --git a/changelog.d/+improve-env-config-docs.changed.md b/changelog.d/+improve-env-config-docs.changed.md deleted file mode 100644 index 79e19a7ced7..00000000000 --- a/changelog.d/+improve-env-config-docs.changed.md +++ /dev/null @@ -1 +0,0 @@ -Update configuration.md and improve config env.mapping list. diff --git a/changelog.d/2221.added.md b/changelog.d/2221.added.md deleted file mode 100644 index fe396a1ac03..00000000000 --- a/changelog.d/2221.added.md +++ /dev/null @@ -1 +0,0 @@ -Add rmdir / unlink / unlinkat support diff --git a/changelog.d/2936.fixed.md b/changelog.d/2936.fixed.md deleted file mode 100644 index e4dfd4792a6..00000000000 --- a/changelog.d/2936.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Stopped mirrord entering a crash loop when trying to load into some processes like VSCode's `watchdog.js` when the user config contained a call to `get_env()`, which occurred due to missing env - the config is now only rendered once and set into an env var. \ No newline at end of file diff --git a/changelog.d/3013.fixed.md b/changelog.d/3013.fixed.md deleted file mode 100644 index 811ab816b8c..00000000000 --- a/changelog.d/3013.fixed.md +++ /dev/null @@ -1,2 +0,0 @@ -Fixed an issue where HTTP requests stolen with a filter would hang with a single-threaded local HTTP server. -Improved handling of incoming connections on the local machine (e.g introduces reuse of local HTTP connections). diff --git a/changelog.d/3024.fixed.md b/changelog.d/3024.fixed.md deleted file mode 100644 index c584f5269ed..00000000000 --- a/changelog.d/3024.fixed.md +++ /dev/null @@ -1 +0,0 @@ -use older builder base image for aarch64 to support centos-7 libc diff --git a/mirrord/cli/src/execution.rs b/mirrord/cli/src/execution.rs index 5d3e9657e88..611c45172ce 100644 --- a/mirrord/cli/src/execution.rs +++ b/mirrord/cli/src/execution.rs @@ -7,7 +7,7 @@ use std::{ use mirrord_analytics::{AnalyticsError, AnalyticsReporter, Reporter}; use mirrord_config::{ config::ConfigError, feature::env::mapper::EnvVarsRemapper, - internal_proxy::MIRRORD_INTPROXY_CONNECT_TCP_ENV, LayerConfig, + internal_proxy::MIRRORD_INTPROXY_CONNECT_TCP_ENV, LayerConfig, MIRRORD_RESOLVED_CONFIG_ENV, }; use mirrord_intproxy::agent_conn::AgentConnectInfo; use mirrord_operator::client::OperatorSession; @@ -269,7 +269,7 @@ impl MirrordExecution { let stdout = proxy_process.stdout.take().expect("stdout was piped"); - let address: SocketAddr = BufReader::new(stdout) + let intproxy_address: SocketAddr = BufReader::new(stdout) .lines() .next_line() .await @@ -288,9 +288,10 @@ impl MirrordExecution { )) })?; - // Provide details for layer to connect to agent via internal proxy - config.connect_tcp = Some(format!("127.0.0.1:{}", address.port())); + config.connect_tcp.replace(intproxy_address.to_string()); config.update_env_var()?; + let config_as_env = config.to_env_var()?; + env_vars.insert(MIRRORD_RESOLVED_CONFIG_ENV.to_string(), config_as_env); // Fixes // by disabling the fork safety check in the Objective-C runtime. diff --git a/mirrord/cli/src/extension.rs b/mirrord/cli/src/extension.rs index 005491a9aed..f46e91e5092 100644 --- a/mirrord/cli/src/extension.rs +++ b/mirrord/cli/src/extension.rs @@ -4,7 +4,7 @@ use mirrord_analytics::{AnalyticsError, AnalyticsReporter, Reporter}; use mirrord_config::{LayerConfig, MIRRORD_CONFIG_FILE_ENV}; use mirrord_progress::{JsonProgress, Progress, ProgressTracker}; -use crate::{config::ExtensionExecArgs, error::CliError, execution::MirrordExecution, CliResult}; +use crate::{config::ExtensionExecArgs, execution::MirrordExecution, CliResult}; /// Actually facilitate execution after all preparations were complete async fn mirrord_exec

( @@ -40,23 +40,15 @@ where pub(crate) async fn extension_exec(args: ExtensionExecArgs, watch: drain::Watch) -> CliResult<()> { let progress = ProgressTracker::try_from_env("mirrord preparing to launch") .unwrap_or_else(|| JsonProgress::new("mirrord preparing to launch").into()); - let mut env: HashMap = HashMap::new(); + // Set environment required for `LayerConfig::from_env_with_warnings`. if let Some(config_file) = args.config_file.as_ref() { - // Set canoncialized path to config file, in case forks/children are in different - // working directories. - let full_path = std::fs::canonicalize(config_file) - .map_err(|e| CliError::CanonicalizeConfigPathFailed(config_file.into(), e))?; - std::env::set_var(MIRRORD_CONFIG_FILE_ENV, full_path.clone()); - env.insert( - MIRRORD_CONFIG_FILE_ENV.into(), - full_path.to_string_lossy().into(), - ); + std::env::set_var(MIRRORD_CONFIG_FILE_ENV, config_file); } if let Some(target) = args.target.as_ref() { std::env::set_var("MIRRORD_IMPERSONATED_TARGET", target.clone()); - env.insert("MIRRORD_IMPERSONATED_TARGET".into(), target.to_string()); } + let (config, mut context) = LayerConfig::from_env_with_warnings()?; let mut analytics = AnalyticsReporter::only_error(config.telemetry, Default::default(), watch); @@ -69,14 +61,14 @@ pub(crate) async fn extension_exec(args: ExtensionExecArgs, watch: drain::Watch) #[cfg(target_os = "macos")] let execution_result = mirrord_exec( args.executable.as_deref(), - env, + Default::default(), config, progress, &mut analytics, ) .await; #[cfg(not(target_os = "macos"))] - let execution_result = mirrord_exec(env, config, progress, &mut analytics).await; + let execution_result = mirrord_exec(Default::default(), config, progress, &mut analytics).await; if execution_result.is_err() && !analytics.has_error() { analytics.set_error(AnalyticsError::Unknown); diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index 8414bd742df..d3f8fae7bc6 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -345,15 +345,15 @@ impl LayerConfig { /// Given a [`LayerConfig`], serialise it and convert to base 64 so it can be /// set into [`MIRRORD_RESOLVED_CONFIG_ENV`]. - fn to_env_var(&self) -> Result { + pub fn to_env_var(&self) -> Result { let serialized = serde_json::to_string(self) .map_err(|error| ConfigError::EnvVarEncodeError(error.to_string()))?; Ok(BASE64_STANDARD.encode(serialized)) } - /// Given the encoded config as a string, set it into [`MIRRORD_RESOLVED_CONFIG_ENV`]. - /// Must be used when updating [`LayerConfig`] after creation in order for the config - /// in env to reflect the change. + /// Encode this config with [`Self::to_env_var`] and set it into + /// [`MIRRORD_RESOLVED_CONFIG_ENV`]. Must be used when updating [`LayerConfig`] after + /// creation in order for the config in env to reflect the change. pub fn update_env_var(&self) -> Result<(), ConfigError> { std::env::set_var(MIRRORD_RESOLVED_CONFIG_ENV, self.to_env_var()?); Ok(()) From 351220fc94a843429a212c2ea5f98db6ee38d831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Wed, 22 Jan 2025 11:04:12 +0100 Subject: [PATCH 19/23] Revert build base update (#3029) * Base images restored * Changelog updated --- CHANGELOG.md | 2 -- Cross.toml | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e23be72090..bda379d047d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,8 +34,6 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang Improved handling of incoming connections on the local machine (e.g introduces reuse of local HTTP connections). [#3013](https://github.com/metalbear-co/mirrord/issues/3013) -- Moved to an older builder base image for aarch64 to support centos-7 libc. - [#3024](https://github.com/metalbear-co/mirrord/issues/3024) ### Internal diff --git a/Cross.toml b/Cross.toml index dfca795a2e4..4c7e75cb182 100644 --- a/Cross.toml +++ b/Cross.toml @@ -5,7 +5,7 @@ passthrough = [ # Dockerfile used for building mirrord-layer for x64 with very old libc # this to support centos7 or Amazon Linux 2. [target.x86_64-unknown-linux-gnu] -image = "ghcr.io/metalbear-co/ci-layer-build:8ca4a4e9757a5749c384c57c3435e56c2b442458e14e4cb7ecbaec4d557f5d69" +image = "ghcr.io/metalbear-co/ci-layer-build:latest" [target.aarch64-unknown-linux-gnu] -image = "ghcr.io/metalbear-co/ci-layer-build-aarch64:4c99d799d297ddec935914907b94d899bd0a2349155cec934492ef19a69ddbf0" \ No newline at end of file +image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main" \ No newline at end of file From 15f9e34fcdda93f161b3b5e8c5dd4ba69a826745 Mon Sep 17 00:00:00 2001 From: t4lz Date: Wed, 22 Jan 2025 13:16:30 +0100 Subject: [PATCH 20/23] DNS resolution for IPv6 (#3023) * Fix ipv6 env var - not only incoming * outgoing ipv6 E2E test * getaddrinfo don't mix IPv4 and IPv6 * fix unused import warning * request V2 * make layer generate V2 requests * change agent to respect new dns request - not working? * logs * libc consts are different on mac and linux * test with kuberentes api * changelog * document tests in contributing guide * fix docs * fix integration test * fix another integration test * fix another integration test * change e2e test logs to debug for CI * fix? * Add other enum variants * CR: variant name Old->V1 * CR: protocol_version->set_protocol_version * CR: version support warning * bump version again as it was also bumped in main * lock --- CONTRIBUTING.md | 15 ++++ Cargo.lock | 2 +- changelog.d/2958.added.md | 1 + mirrord/agent/src/dns.rs | 68 ++++++++++++++--- mirrord/agent/src/entrypoint.rs | 14 +++- mirrord/config/src/feature/network.rs | 2 +- mirrord/intproxy/protocol/src/lib.rs | 8 +- mirrord/intproxy/src/background_tasks.rs | 2 +- mirrord/intproxy/src/lib.rs | 7 ++ mirrord/intproxy/src/proxies/simple.rs | 47 ++++++++++-- mirrord/layer/src/file/ops.rs | 4 +- mirrord/layer/src/socket.rs | 14 +++- mirrord/layer/src/socket/ops.rs | 64 ++++++++++++---- mirrord/layer/tests/dns_resolve.rs | 6 +- mirrord/layer/tests/issue2055.rs | 8 +- mirrord/layer/tests/issue2283.rs | 4 +- mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/codec.rs | 3 +- mirrord/protocol/src/dns.rs | 93 +++++++++++++++++++++++- tests/src/traffic.rs | 41 ++++++++++- tests/src/traffic/steal.rs | 2 +- tests/src/utils.rs | 12 ++- 22 files changed, 363 insertions(+), 56 deletions(-) create mode 100644 changelog.d/2958.added.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b99ddc4c65..0583782b4b1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,6 +131,21 @@ In order to test IPv6 on a local cluster on macOS, you can use Kind: 3. `kind create cluster --config kind-config.yaml` 4. When you run `kubectl get svc -o wide --all-namespaces` you should see IPv6 addresses. +In order to use an agent image from a local registry, you can load the image to kind's registry with: + +``` +kind load docker-image test:latest +``` + +In order to test on EKS, I used this blueprint: https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/patterns/ipv6-eks-cluster + +After creating the cluster, I had to give myself permissions to the K8s objects, I did that via the AWS console (in the browser). +Feel free to add instructions on how to make that "manual" step unnecessary. + +IPv6 tests (they currently don't run in the CI): +- steal_http_ipv6_traffic +- connect_to_kubernetes_api_service_over_ipv6 + ### Cleanup diff --git a/Cargo.lock b/Cargo.lock index fd42ec7bb37..1269d9ad275 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4477,7 +4477,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.14.0" +version = "1.15.0" dependencies = [ "actix-codec", "bincode", diff --git a/changelog.d/2958.added.md b/changelog.d/2958.added.md new file mode 100644 index 00000000000..af12472d466 --- /dev/null +++ b/changelog.d/2958.added.md @@ -0,0 +1 @@ +Support for in-cluster DNS resolution of IPv6 addresses. diff --git a/mirrord/agent/src/dns.rs b/mirrord/agent/src/dns.rs index 0ad44c76934..3240856275a 100644 --- a/mirrord/agent/src/dns.rs +++ b/mirrord/agent/src/dns.rs @@ -3,7 +3,7 @@ use std::{future, path::PathBuf, time::Duration}; use futures::{stream::FuturesOrdered, StreamExt}; use hickory_resolver::{system_conf::parse_resolv_conf, Hosts, Resolver}; use mirrord_protocol::{ - dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoResponse}, + dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoRequestV2, GetAddrInfoResponse}, DnsLookupError, RemoteResult, ResolveErrorKindInternal, ResponseError, }; use tokio::{ @@ -21,9 +21,24 @@ use crate::{ watched_task::TaskStatus, }; +#[derive(Debug)] +pub(crate) enum ClientGetAddrInfoRequest { + V1(GetAddrInfoRequest), + V2(GetAddrInfoRequestV2), +} + +impl ClientGetAddrInfoRequest { + pub(crate) fn into_v2(self) -> GetAddrInfoRequestV2 { + match self { + ClientGetAddrInfoRequest::V1(old_req) => old_req.into(), + ClientGetAddrInfoRequest::V2(v2_req) => v2_req, + } + } +} + #[derive(Debug)] pub(crate) struct DnsCommand { - request: GetAddrInfoRequest, + request: ClientGetAddrInfoRequest, response_tx: oneshot::Sender>, } @@ -34,6 +49,7 @@ pub(crate) struct DnsWorker { request_rx: Receiver, attempts: usize, timeout: Duration, + support_ipv6: bool, } impl DnsWorker { @@ -45,7 +61,11 @@ impl DnsWorker { /// # Note /// /// `pid` is used to find the correct path of `etc` directory. - pub(crate) fn new(pid: Option, request_rx: Receiver) -> Self { + pub(crate) fn new( + pid: Option, + request_rx: Receiver, + support_ipv6: bool, + ) -> Self { let etc_path = pid .map(|pid| { PathBuf::from("/proc") @@ -66,6 +86,7 @@ impl DnsWorker { .ok() .and_then(|attempts| attempts.parse().ok()) .unwrap_or(1), + support_ipv6, } } @@ -79,9 +100,10 @@ impl DnsWorker { #[tracing::instrument(level = Level::TRACE, ret, err(level = Level::TRACE))] async fn do_lookup( etc_path: PathBuf, - host: String, + request: GetAddrInfoRequestV2, attempts: usize, timeout: Duration, + support_ipv6: bool, ) -> RemoteResult { // Prepares the `Resolver` after reading some `/etc` DNS files. // @@ -94,13 +116,32 @@ impl DnsWorker { let hosts_conf = fs::read(hosts_path).await?; let (config, mut options) = parse_resolv_conf(resolv_conf)?; + tracing::debug!(?config, ?options, "parsed config options"); options.server_ordering_strategy = hickory_resolver::config::ServerOrderingStrategy::UserProvidedOrder; options.timeout = timeout; options.attempts = attempts; - options.ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4Only; + options.ip_strategy = if support_ipv6 { + tracing::debug!("IPv6 support enabled. Respecting client IP family."); + request + .family + .try_into() + .inspect_err(|e| { + tracing::error!(%e, + "Unknown address family in addrinfo request. Using IPv4 and IPv6.") + }) + // If the agent gets some new, unknown variant of family address, it's the + // client's fault, so the agent queries both IPv4 and IPv6 and if that's not + // good enough for the client, the client can error out. + .unwrap_or(hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6) + } else { + tracing::debug!("IPv6 support disabled. Resolving IPv4 only."); + hickory_resolver::config::LookupIpStrategy::Ipv4Only + }; + tracing::debug!(?config, ?options, "updated config options"); let mut resolver = Resolver::tokio(config, options); + tracing::debug!(?resolver, "tokio resolver"); let mut hosts = Hosts::default(); hosts.read_hosts_conf(hosts_conf.as_slice())?; @@ -111,9 +152,10 @@ impl DnsWorker { let lookup = resolver .inspect_err(|fail| tracing::error!(?fail, "Failed to build DNS resolver"))? - .lookup_ip(host) + .lookup_ip(request.node) .await - .inspect(|lookup| tracing::trace!(?lookup, "Lookup finished"))? + .inspect(|lookup| tracing::trace!(?lookup, "Lookup finished")) + .inspect_err(|e| tracing::trace!(%e, "lookup failed"))? .into(); Ok(lookup) @@ -125,8 +167,16 @@ impl DnsWorker { let etc_path = self.etc_path.clone(); let timeout = self.timeout; let attempts = self.attempts; + let support_ipv6 = self.support_ipv6; let lookup_future = async move { - let result = Self::do_lookup(etc_path, message.request.node, attempts, timeout).await; + let result = Self::do_lookup( + etc_path, + message.request.into_v2(), + attempts, + timeout, + support_ipv6, + ) + .await; if let Err(result) = message.response_tx.send(result) { tracing::error!(?result, "Failed to send query response"); @@ -174,7 +224,7 @@ impl DnsApi { /// Results of scheduled requests are available via [`Self::recv`] (order is preserved). pub(crate) async fn make_request( &mut self, - request: GetAddrInfoRequest, + request: ClientGetAddrInfoRequest, ) -> Result<(), AgentError> { let (response_tx, response_rx) = oneshot::channel(); diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index fff9ee7192f..407bf27c33f 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -10,7 +10,7 @@ use std::{ }; use client_connection::AgentTlsConnector; -use dns::{DnsCommand, DnsWorker}; +use dns::{ClientGetAddrInfoRequest, DnsCommand, DnsWorker}; use futures::TryFutureExt; use mirrord_protocol::{ClientMessage, DaemonMessage, GetEnvVarsRequest, LogMessage}; use sniffer::tcp_capture::RawSocketTcpCapture; @@ -433,7 +433,14 @@ impl ClientConnectionHandler { .await? } ClientMessage::GetAddrInfoRequest(request) => { - self.dns_api.make_request(request).await?; + self.dns_api + .make_request(ClientGetAddrInfoRequest::V1(request)) + .await?; + } + ClientMessage::GetAddrInfoRequestV2(request) => { + self.dns_api + .make_request(ClientGetAddrInfoRequest::V2(request)) + .await?; } ClientMessage::Ping => self.respond(DaemonMessage::Pong).await?, ClientMessage::Tcp(message) => { @@ -613,7 +620,8 @@ async fn start_agent(args: Args) -> Result<()> { let cancellation_token = cancellation_token.clone(); let watched_task = WatchedTask::new( DnsWorker::TASK_NAME, - DnsWorker::new(state.container_pid(), dns_command_rx).run(cancellation_token), + DnsWorker::new(state.container_pid(), dns_command_rx, args.ipv6) + .run(cancellation_token), ); let status = watched_task.status(); let task = run_thread_in_namespace( diff --git a/mirrord/config/src/feature/network.rs b/mirrord/config/src/feature/network.rs index 1d86071e32e..976adf2b814 100644 --- a/mirrord/config/src/feature/network.rs +++ b/mirrord/config/src/feature/network.rs @@ -10,7 +10,7 @@ use crate::{ util::MirrordToggleableConfig, }; -const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6"; +const IPV6_ENV_VAR: &str = "MIRRORD_ENABLE_IPV6"; pub mod dns; pub mod filter; diff --git a/mirrord/intproxy/protocol/src/lib.rs b/mirrord/intproxy/protocol/src/lib.rs index 69a551b50bc..e51fcdf773e 100644 --- a/mirrord/intproxy/protocol/src/lib.rs +++ b/mirrord/intproxy/protocol/src/lib.rs @@ -10,7 +10,7 @@ use std::{ use bincode::{Decode, Encode}; use mirrord_protocol::{ - dns::{GetAddrInfoRequest, GetAddrInfoResponse}, + dns::{GetAddrInfoRequestV2, GetAddrInfoResponse}, file::*, outgoing::SocketAddress, tcp::StealType, @@ -44,7 +44,7 @@ pub enum LayerToProxyMessage { /// A file operation request. File(FileRequest), /// A DNS request. - GetAddrInfo(GetAddrInfoRequest), + GetAddrInfo(GetAddrInfoRequestV2), /// A request to initiate a new outgoing connection. OutgoingConnect(OutgoingConnectRequest), /// Requests related to incoming connections. @@ -210,7 +210,7 @@ pub enum ProxyToLayerMessage { NewSession(LayerId), /// A response to layer's [`FileRequest`]. File(FileResponse), - /// A response to layer's [`GetAddrInfoRequest`]. + /// A response to layer's [`GetAddrInfoRequestV2`]. GetAddrInfo(GetAddrInfoResponse), /// A response to layer's [`OutgoingConnectRequest`]. OutgoingConnect(RemoteResult), @@ -428,7 +428,7 @@ impl_request!( ); impl_request!( - req = GetAddrInfoRequest, + req = GetAddrInfoRequestV2, res = GetAddrInfoResponse, req_path = LayerToProxyMessage::GetAddrInfo, res_path = ProxyToLayerMessage::GetAddrInfo, diff --git a/mirrord/intproxy/src/background_tasks.rs b/mirrord/intproxy/src/background_tasks.rs index 82e6865c67e..2c73f80bece 100644 --- a/mirrord/intproxy/src/background_tasks.rs +++ b/mirrord/intproxy/src/background_tasks.rs @@ -3,7 +3,7 @@ //! The proxy utilizes multiple background tasks to split the code into more self-contained parts. //! Structs in this module aim to ease managing their state. //! -//! Each background task implement the [`BackgroundTask`] trait, which specifies its properties and +//! Each background task implements the [`BackgroundTask`] trait, which specifies its properties and //! allows for managing groups of related tasks with one [`BackgroundTasks`] instance. use std::{collections::HashMap, fmt, future::Future, hash::Hash}; diff --git a/mirrord/intproxy/src/lib.rs b/mirrord/intproxy/src/lib.rs index 7dee93344bf..7b5944bc307 100644 --- a/mirrord/intproxy/src/lib.rs +++ b/mirrord/intproxy/src/lib.rs @@ -321,6 +321,13 @@ impl IntProxy { .send(FilesProxyMessage::ProtocolVersion(protocol_version.clone())) .await; + self.task_txs + .simple + .send(SimpleProxyMessage::ProtocolVersion( + protocol_version.clone(), + )) + .await; + self.task_txs .incoming .send(IncomingProxyMessage::AgentProtocolVersion(protocol_version)) diff --git a/mirrord/intproxy/src/proxies/simple.rs b/mirrord/intproxy/src/proxies/simple.rs index dae7881247e..49e643b259a 100644 --- a/mirrord/intproxy/src/proxies/simple.rs +++ b/mirrord/intproxy/src/proxies/simple.rs @@ -5,9 +5,10 @@ use std::collections::HashMap; use mirrord_intproxy_protocol::{LayerId, MessageId, ProxyToLayerMessage}; use mirrord_protocol::{ - dns::{GetAddrInfoRequest, GetAddrInfoResponse}, + dns::{AddressFamily, GetAddrInfoRequestV2, GetAddrInfoResponse, ADDRINFO_V2_VERSION}, ClientMessage, DaemonMessage, GetEnvVarsRequest, RemoteResult, }; +use semver::Version; use thiserror::Error; use crate::{ @@ -20,10 +21,12 @@ use crate::{ #[derive(Debug)] pub enum SimpleProxyMessage { - AddrInfoReq(MessageId, LayerId, GetAddrInfoRequest), + AddrInfoReq(MessageId, LayerId, GetAddrInfoRequestV2), AddrInfoRes(GetAddrInfoResponse), GetEnvReq(MessageId, LayerId, GetEnvVarsRequest), GetEnvRes(RemoteResult>), + /// Protocol version was negotiated with the agent. + ProtocolVersion(Version), } #[derive(Error, Debug)] @@ -34,10 +37,27 @@ pub struct SimpleProxyError(#[from] UnexpectedAgentMessage); /// Run as a [`BackgroundTask`]. #[derive(Default)] pub struct SimpleProxy { - /// For [`GetAddrInfoRequest`]s. + /// For [`GetAddrInfoRequestV2`]s. addr_info_reqs: RequestQueue, /// For [`GetEnvVarsRequest`]s. get_env_reqs: RequestQueue, + /// [`mirrord_protocol`] version negotiated with the agent. + /// Determines whether we can use `GetAddrInfoRequestV2`. + protocol_version: Option, +} + +impl SimpleProxy { + #[tracing::instrument(skip(self), level = tracing::Level::TRACE)] + fn set_protocol_version(&mut self, version: Version) { + self.protocol_version.replace(version); + } + + /// Returns whether [`mirrord_protocol`] version allows for a V2 addrinfo request. + fn addr_info_v2(&self) -> bool { + self.protocol_version + .as_ref() + .is_some_and(|version| ADDRINFO_V2_VERSION.matches(version)) + } } impl BackgroundTask for SimpleProxy { @@ -52,9 +72,23 @@ impl BackgroundTask for SimpleProxy { match msg { SimpleProxyMessage::AddrInfoReq(message_id, session_id, req) => { self.addr_info_reqs.push_back(message_id, session_id); - message_bus - .send(ClientMessage::GetAddrInfoRequest(req)) - .await; + if self.addr_info_v2() { + message_bus + .send(ClientMessage::GetAddrInfoRequestV2(req)) + .await; + } else { + if matches!(req.family, AddressFamily::Ipv6Only) { + tracing::warn!( + "The agent version you're using does not support DNS\ + queries for IPv6 addresses. This version will only fetch IPv4\ + address. Please update to a newer agent image for better IPv6\ + support." + ) + } + message_bus + .send(ClientMessage::GetAddrInfoRequest(req.into())) + .await; + } } SimpleProxyMessage::AddrInfoRes(res) => { let (message_id, layer_id) = @@ -88,6 +122,7 @@ impl BackgroundTask for SimpleProxy { }) .await } + SimpleProxyMessage::ProtocolVersion(version) => self.set_protocol_version(version), } } diff --git a/mirrord/layer/src/file/ops.rs b/mirrord/layer/src/file/ops.rs index bac20ad7cd9..fcc27507876 100644 --- a/mirrord/layer/src/file/ops.rs +++ b/mirrord/layer/src/file/ops.rs @@ -14,7 +14,9 @@ use mirrord_protocol::{ ResponseError, }; use rand::distributions::{Alphanumeric, DistString}; -use tracing::{error, trace, Level}; +#[cfg(debug_assertions)] +use tracing::Level; +use tracing::{error, trace}; use super::{hooks::FN_OPEN, open_dirs::OPEN_DIRS, *}; #[cfg(target_os = "linux")] diff --git a/mirrord/layer/src/socket.rs b/mirrord/layer/src/socket.rs index 79c197d408d..5c2d03f8ace 100644 --- a/mirrord/layer/src/socket.rs +++ b/mirrord/layer/src/socket.rs @@ -439,10 +439,22 @@ impl ProtocolAndAddressFilterExt for ProtocolAndAddressFilter { return Ok(false); } + let family = if address.is_ipv4() { + libc::AF_INET + } else { + libc::AF_INET6 + }; + + let addr_protocol = if matches!(protocol, NetProtocol::Stream) { + libc::SOCK_STREAM + } else { + libc::SOCK_DGRAM + }; + match &self.address { AddressFilter::Name(name, port) => { let resolved_ips = if crate::setup().remote_dns_enabled() && !force_local_dns { - match remote_getaddrinfo(name.to_string()) { + match remote_getaddrinfo(name.to_string(), *port, 0, family, 0, addr_protocol) { Ok(res) => res.into_iter().map(|(_, ip)| ip).collect(), Err(HookError::ResponseError(ResponseError::DnsLookup( DnsLookupError { diff --git a/mirrord/layer/src/socket/ops.rs b/mirrord/layer/src/socket/ops.rs index efc0095c8a4..a8a6e49e79a 100644 --- a/mirrord/layer/src/socket/ops.rs +++ b/mirrord/layer/src/socket/ops.rs @@ -22,7 +22,7 @@ use mirrord_intproxy_protocol::{ OutgoingConnectResponse, PortSubscribe, }; use mirrord_protocol::{ - dns::{GetAddrInfoRequest, LookupRecord}, + dns::{AddressFamily, GetAddrInfoRequestV2, LookupRecord, SockType}, file::{OpenFileResponse, OpenOptionsInternal, ReadFileResponse}, }; use nix::sys::socket::{sockopt, SockaddrIn, SockaddrIn6, SockaddrLike, SockaddrStorage}; @@ -209,7 +209,8 @@ fn is_ignored_tcp_port(addr: &SocketAddr, config: &IncomingConfig) -> bool { /// If the socket is not found in [`SOCKETS`], bypass. /// Otherwise, if it's not an ignored port, bind (possibly with a fallback to random port) and /// update socket state in [`SOCKETS`]. If it's an ignored port, remove the socket from [`SOCKETS`]. -#[mirrord_layer_macro::instrument(level = Level::TRACE, fields(pid = std::process::id()), ret, skip(raw_address))] +#[mirrord_layer_macro::instrument(level = Level::TRACE, fields(pid = std::process::id()), ret, skip(raw_address) +)] pub(super) fn bind( sockfd: c_int, raw_address: *const sockaddr, @@ -324,9 +325,9 @@ pub(super) fn bind( } }) } - .ok() - .and_then(|(_, address)| address.as_socket()) - .bypass(Bypass::AddressConversion)?; + .ok() + .and_then(|(_, address)| address.as_socket()) + .bypass(Bypass::AddressConversion)?; Arc::get_mut(&mut socket).unwrap().state = SocketState::Bound(Bound { requested_address, @@ -890,8 +891,33 @@ pub(super) fn dup(fd: c_int, dup_fd: i32) -> Result<(), /// /// This function updates the mapping in [`REMOTE_DNS_REVERSE_MAPPING`]. #[mirrord_layer_macro::instrument(level = Level::TRACE, ret, err)] -pub(super) fn remote_getaddrinfo(node: String) -> HookResult> { - let addr_info_list = common::make_proxy_request_with_response(GetAddrInfoRequest { node })?.0?; +pub(super) fn remote_getaddrinfo( + node: String, + service_port: u16, + flags: c_int, + family: c_int, + socktype: c_int, + protocol: c_int, +) -> HookResult> { + let family = match family { + libc::AF_INET => AddressFamily::Ipv4Only, + libc::AF_INET6 => AddressFamily::Ipv6Only, + _ => AddressFamily::Both, + }; + let socktype = match socktype { + libc::SOCK_STREAM => SockType::Stream, + libc::SOCK_DGRAM => SockType::Dgram, + _ => SockType::Any, + }; + let addr_info_list = common::make_proxy_request_with_response(GetAddrInfoRequestV2 { + node, + service_port, + flags, + family, + socktype, + protocol, + })? + .0?; let mut remote_dns_reverse_mapping = REMOTE_DNS_REVERSE_MAPPING.lock()?; addr_info_list.iter().for_each(|lookup| { @@ -946,29 +972,41 @@ pub(super) fn getaddrinfo( Bypass::CStrConversion })? + // TODO: according to the man page, service could also be a service name, it doesn't have to + // be a port number. .and_then(|service| service.parse::().ok()) .unwrap_or(0); - crate::setup().dns_selector().check_query(&node, service)?; + let setup = crate::setup(); + setup.dns_selector().check_query(&node, service)?; + let ipv6_enabled = setup.layer_config().feature.network.ipv6; let raw_hints = raw_hints .cloned() .unwrap_or_else(|| unsafe { mem::zeroed() }); - // TODO(alex): Use more fields from `raw_hints` to respect the user's `getaddrinfo` call. let libc::addrinfo { + ai_family, ai_socktype, ai_protocol, + ai_flags, .. } = raw_hints; // Some apps (gRPC on Python) use `::` to listen on all interfaces, and usually that just means - // resolve on unspecified. So we just return that in IpV4 because we don't support ipv6. - let resolved_addr = if node == "::" { + // resolve on unspecified. So we just return that in IPv4, if IPv6 support is disabled. + let resolved_addr = if ipv6_enabled.not() && (node == "::") { // name is "" because that's what happens in real flow. vec![("".to_string(), IpAddr::V4(Ipv4Addr::UNSPECIFIED))] } else { - remote_getaddrinfo(node.clone())? + remote_getaddrinfo( + node.clone(), + service, + ai_flags, + ai_family, + ai_socktype, + ai_protocol, + )? }; let mut managed_addr_info = MANAGED_ADDRINFO.lock()?; @@ -1067,7 +1105,7 @@ pub(super) fn gethostbyname(raw_name: Option<&CStr>) -> Detour<*mut hostent> { crate::setup().dns_selector().check_query(&name, 0)?; - let hosts_and_ips = remote_getaddrinfo(name.clone())?; + let hosts_and_ips = remote_getaddrinfo(name.clone(), 0, 0, 0, 0, 0)?; // We could `unwrap` here, as this would have failed on the previous conversion. let host_name = CString::new(name)?; diff --git a/mirrord/layer/tests/dns_resolve.rs b/mirrord/layer/tests/dns_resolve.rs index 508f546ec03..9085b708d0a 100644 --- a/mirrord/layer/tests/dns_resolve.rs +++ b/mirrord/layer/tests/dns_resolve.rs @@ -8,7 +8,7 @@ use rstest::rstest; mod common; pub use common::*; use mirrord_protocol::{ - dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoResponse, LookupRecord}, + dns::{DnsLookup, GetAddrInfoRequestV2, GetAddrInfoResponse, LookupRecord}, ClientMessage, DaemonMessage, DnsLookupError, ResolveErrorKindInternal::NoRecordsFound, }; @@ -25,7 +25,7 @@ async fn test_dns_resolve( .await; let msg = intproxy.recv().await; - let ClientMessage::GetAddrInfoRequest(GetAddrInfoRequest { node }) = msg else { + let ClientMessage::GetAddrInfoRequestV2(GetAddrInfoRequestV2 { node, .. }) = msg else { panic!("Invalid message received from layer: {msg:?}"); }; @@ -39,7 +39,7 @@ async fn test_dns_resolve( .await; let msg = intproxy.recv().await; - let ClientMessage::GetAddrInfoRequest(GetAddrInfoRequest { node: _ }) = msg else { + let ClientMessage::GetAddrInfoRequestV2(GetAddrInfoRequestV2 { .. }) = msg else { panic!("Invalid message received from layer: {msg:?}"); }; diff --git a/mirrord/layer/tests/issue2055.rs b/mirrord/layer/tests/issue2055.rs index c34d5e13f25..d24b71fadce 100644 --- a/mirrord/layer/tests/issue2055.rs +++ b/mirrord/layer/tests/issue2055.rs @@ -2,7 +2,7 @@ use std::{net::IpAddr, path::Path, time::Duration}; use mirrord_protocol::{ - dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoResponse, LookupRecord}, + dns::{DnsLookup, GetAddrInfoRequestV2, GetAddrInfoResponse, LookupRecord}, ClientMessage, DaemonMessage, DnsLookupError, ResolveErrorKindInternal::NoRecordsFound, ResponseError, @@ -23,10 +23,10 @@ async fn issue_2055(dylib_path: &Path) { .start_process_with_layer(dylib_path, vec![("MIRRORD_REMOTE_DNS", "true")], None) .await; - println!("Application started, waiting for `GetAddrInfoRequest`."); + println!("Application started, waiting for `GetAddrInfoRequestV2`."); let msg = intproxy.recv().await; - let ClientMessage::GetAddrInfoRequest(GetAddrInfoRequest { node }) = msg else { + let ClientMessage::GetAddrInfoRequestV2(GetAddrInfoRequestV2 { node, .. }) = msg else { panic!("Invalid message received from layer: {msg:?}"); }; @@ -40,7 +40,7 @@ async fn issue_2055(dylib_path: &Path) { .await; let msg = intproxy.recv().await; - let ClientMessage::GetAddrInfoRequest(GetAddrInfoRequest { node: _ }) = msg else { + let ClientMessage::GetAddrInfoRequestV2(GetAddrInfoRequestV2 { .. }) = msg else { panic!("Invalid message received from layer: {msg:?}"); }; diff --git a/mirrord/layer/tests/issue2283.rs b/mirrord/layer/tests/issue2283.rs index f3d7a2a9b2b..6987bbdffbf 100644 --- a/mirrord/layer/tests/issue2283.rs +++ b/mirrord/layer/tests/issue2283.rs @@ -3,7 +3,7 @@ use std::{assert_matches::assert_matches, net::SocketAddr, path::Path, time::Duration}; use mirrord_protocol::{ - dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoResponse, LookupRecord}, + dns::{DnsLookup, GetAddrInfoRequestV2, GetAddrInfoResponse, LookupRecord}, outgoing::{ tcp::{DaemonTcpOutgoing, LayerTcpOutgoing}, DaemonConnect, DaemonRead, LayerConnect, SocketAddress, @@ -48,7 +48,7 @@ async fn test_issue2283( } let message = intproxy.recv().await; - assert_matches!(message, ClientMessage::GetAddrInfoRequest(GetAddrInfoRequest { node }) if node == "test-server"); + assert_matches!(message, ClientMessage::GetAddrInfoRequestV2(GetAddrInfoRequestV2 { node, .. }) if node == "test-server"); let address = "1.2.3.4:80".parse::().unwrap(); diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 276d5f164f4..26623eb3c08 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.14.0" +version = "1.15.0" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/codec.rs b/mirrord/protocol/src/codec.rs index 1b0975d350d..39961b3bce2 100644 --- a/mirrord/protocol/src/codec.rs +++ b/mirrord/protocol/src/codec.rs @@ -12,7 +12,7 @@ use mirrord_macros::protocol_break; use semver::VersionReq; use crate::{ - dns::{GetAddrInfoRequest, GetAddrInfoResponse}, + dns::{GetAddrInfoRequest, GetAddrInfoRequestV2, GetAddrInfoResponse}, file::*, outgoing::{ tcp::{DaemonTcpOutgoing, LayerTcpOutgoing}, @@ -117,6 +117,7 @@ pub enum ClientMessage { SwitchProtocolVersion(#[bincode(with_serde)] semver::Version), ReadyForLogs, Vpn(ClientVpn), + GetAddrInfoRequestV2(GetAddrInfoRequestV2), } /// Type alias for `Result`s that should be returned from mirrord-agent to mirrord-layer. diff --git a/mirrord/protocol/src/dns.rs b/mirrord/protocol/src/dns.rs index 855f52b8af5..5958376cd94 100644 --- a/mirrord/protocol/src/dns.rs +++ b/mirrord/protocol/src/dns.rs @@ -1,12 +1,17 @@ extern crate alloc; use core::ops::Deref; -use std::net::IpAddr; +use std::{net::IpAddr, sync::LazyLock}; use bincode::{Decode, Encode}; use hickory_resolver::{lookup_ip::LookupIp, proto::rr::resource::RecordParts}; +use semver::VersionReq; use crate::RemoteResult; +/// Minimal mirrord-protocol version that allows [`GetAddrInfoRequestV2`]. +pub static ADDRINFO_V2_VERSION: LazyLock = + LazyLock::new(|| ">=1.15.0".parse().expect("Bad Identifier")); + #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub struct LookupRecord { pub name: String, @@ -73,3 +78,89 @@ impl Deref for GetAddrInfoResponse { pub struct GetAddrInfoRequest { pub node: String, } + +/// For when the new request is not supported, and we have to fall back to the old version. +impl From for GetAddrInfoRequest { + fn from(value: GetAddrInfoRequestV2) -> Self { + Self { node: value.node } + } +} + +#[derive( + serde::Serialize, serde::Deserialize, Encode, Decode, Debug, PartialEq, Eq, Copy, Clone, +)] +pub enum AddressFamily { + Ipv4Only, + Ipv6Only, + Both, + Any, + /// If we add a variant and a new client sends an old agent the new variant, the agent will see + /// this variant. + #[serde(other, skip_serializing)] + UnknownAddressFamilyFromNewerClient, +} + +#[derive(thiserror::Error, Debug)] +pub enum AddressFamilyError { + #[error( + "The agent received a GetAddrInfoRequestV2 with an address family that is not yet known \ + to this version of the agent." + )] + UnsupportedFamily, +} + +impl TryFrom for hickory_resolver::config::LookupIpStrategy { + type Error = AddressFamilyError; + + fn try_from(value: AddressFamily) -> Result { + match value { + AddressFamily::Ipv4Only => Ok(Self::Ipv4Only), + AddressFamily::Ipv6Only => Ok(Self::Ipv6Only), + AddressFamily::Both => Ok(Self::Ipv4AndIpv6), + AddressFamily::Any => Ok(Self::Ipv4thenIpv6), + AddressFamily::UnknownAddressFamilyFromNewerClient => { + Err(AddressFamilyError::UnsupportedFamily) + } + } + } +} + +#[derive(serde::Serialize, serde::Deserialize, Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub enum SockType { + Stream, + Dgram, + Any, + /// If we add a variant and a new client sends an old agent the new variant, the agent will see + /// this variant. + #[serde(other, skip_serializing)] + UnknownSockTypeFromNewerClient, +} + +/// Newer, advanced version of [`GetAddrInfoRequest`] +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub struct GetAddrInfoRequestV2 { + pub node: String, + /// Currently not respected by the agent, there for future use. + pub service_port: u16, + pub family: AddressFamily, + pub socktype: SockType, + /// Including these fields so we can use them in the future without introducing a new request + /// type. But note that the constants are different on macOS and Linux so they should be + /// converted to the linux values first (on the client, because the agent does not know the + /// client is macOS). + pub flags: i32, + pub protocol: i32, +} + +impl From for GetAddrInfoRequestV2 { + fn from(value: GetAddrInfoRequest) -> Self { + Self { + node: value.node, + service_port: 0, + flags: 0, + family: AddressFamily::Ipv4Only, + socktype: SockType::Any, + protocol: 0, + } + } +} diff --git a/tests/src/traffic.rs b/tests/src/traffic.rs index abdaec6404c..c511eb12e10 100644 --- a/tests/src/traffic.rs +++ b/tests/src/traffic.rs @@ -16,8 +16,8 @@ mod traffic_tests { use tokio::{fs::File, io::AsyncWriteExt}; use crate::utils::{ - config_dir, hostname_service, kube_client, run_exec_with_target, service, - udp_logger_service, KubeService, CONTAINER_NAME, + config_dir, hostname_service, ipv6::ipv6_service, kube_client, run_exec_with_target, + service, udp_logger_service, Application, KubeService, CONTAINER_NAME, }; #[cfg_attr(not(feature = "job"), ignore)] @@ -114,6 +114,43 @@ mod traffic_tests { assert!(res.success()); } + #[rstest] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[ignore] + pub async fn outgoing_traffic_single_request_ipv6_enabled(#[future] ipv6_service: KubeService) { + let service = ipv6_service.await; + let node_command = vec![ + "node", + "node-e2e/outgoing/test_outgoing_traffic_single_request_ipv6.mjs", + ]; + let mut process = run_exec_with_target( + node_command, + &service.pod_container_target(), + None, + None, + Some(vec![("MIRRORD_ENABLE_IPV6", "true")]), + ) + .await; + + let res = process.wait().await; + assert!(res.success()); + } + + #[rstest] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[timeout(Duration::from_secs(30))] + #[ignore] + pub async fn connect_to_kubernetes_api_service_over_ipv6() { + let app = Application::CurlToKubeApi; + let mut process = app + .run_targetless(None, None, Some(vec![("MIRRORD_ENABLE_IPV6", "true")])) + .await; + let res = process.wait().await; + assert!(res.success()); + let stdout = process.get_stdout().await; + assert!(stdout.contains(r#""apiVersion": "v1""#)) + } + #[cfg_attr(not(feature = "job"), ignore)] #[rstest] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/tests/src/traffic/steal.rs b/tests/src/traffic/steal.rs index bf417813cf0..4b9fcbb2ddd 100644 --- a/tests/src/traffic/steal.rs +++ b/tests/src/traffic/steal.rs @@ -90,7 +90,7 @@ mod steal_tests { &service.pod_container_target(), Some(&service.namespace), Some(flags), - Some(vec![("MIRRORD_INCOMING_ENABLE_IPV6", "true")]), + Some(vec![("MIRRORD_ENABLE_IPV6", "true")]), ) .await; diff --git a/tests/src/utils.rs b/tests/src/utils.rs index bf8e0d1a79b..7fa4b7c68e7 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -99,6 +99,7 @@ pub enum Application { Go22HTTP, Go23HTTP, CurlToKubeApi, + CurlToKubeApiOverIpv6, PythonCloseSocket, PythonCloseSocketKeepConnection, RustWebsockets, @@ -477,6 +478,9 @@ impl Application { Application::CurlToKubeApi => { vec!["curl", "https://kubernetes/api", "--insecure"] } + Application::CurlToKubeApiOverIpv6 => { + vec!["curl", "-6", "https://kubernetes/api", "--insecure"] + } Application::RustWebsockets => vec!["../target/debug/rust-websockets"], Application::RustSqs => vec!["../target/debug/rust-sqs-printer"], } @@ -633,11 +637,17 @@ pub async fn run_exec( .into_iter() .chain(process_cmd.into_iter()) .collect(); + let agent_image_env = "MIRRORD_AGENT_IMAGE"; + let agent_image_from_devs_env = std::env::var(agent_image_env); // used by the CI, to load the image locally: // docker build -t test . -f mirrord/agent/Dockerfile // minikube load image test:latest let mut base_env = HashMap::new(); - base_env.insert("MIRRORD_AGENT_IMAGE", "test"); + base_env.insert( + agent_image_env, + // Let devs running the test specify an agent image per env var. + agent_image_from_devs_env.as_deref().unwrap_or("test"), + ); base_env.insert("MIRRORD_CHECK_VERSION", "false"); base_env.insert("MIRRORD_AGENT_RUST_LOG", "warn,mirrord=debug"); base_env.insert("MIRRORD_AGENT_COMMUNICATION_TIMEOUT", "180"); From 57bdab5ce2c67905a2de1cbfc9a90cdad8889761 Mon Sep 17 00:00:00 2001 From: meowjesty <43983236+meowjesty@users.noreply.github.com> Date: Wed, 22 Jan 2025 15:31:13 -0300 Subject: [PATCH 21/23] Adds prometheus export point to the agent pod. (#2983) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add metrics system (prometheus) to the agent. * Always have metrics, just doesnt do anything. * Add metrics for sniffer. * Add metrics for stealer. * How to install prometheus // Add more metrics to the reply * Do not register the gauge multiple times. * tcpoutgoing metrics * udpoutgoing metrics * docs * move metrics to modules * bump protocol * fix axum-server version * info -> trace * just have metrics atomics everywhere * drop actors * gate metrics behind config * use socketaddr * spawn inside spawn * error impl send * move where we inc subs for sniffer * no clone * dec things closer to remove * better help Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * fix help Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * no async file manager * split filtered unfiltered steal inc * remove unfiltered inc from wrong place * cancellation token * unit test for metrics * remove unused * rustfmt * schema * schema 2 * md * appease clippy * crypto provider for test * encode_to_string // cancellation on error * better error Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * no more metricserror * outdated docs * connection metrics to run tasks * unused * remove todo * line Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * inc not dec Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * inc stream fd Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * only dec once Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> * drop filemanager updates metrics * sniffer drop and update_packet_filter metrics * remove dec from extra sub case * move sub to PortSub add * drop for PortSubs, zero subs counter * new and drop for unfiltered task * remove inc from run * drop for filtered * docs * remove metrics from some places * cancel * near insert and remove * connection sub inc * connected clients * dns request * http in progress * protocol * http request in progress * dns * 2 new decs in tcp outgoing * dec fd twice * filtered port * some docs * improve error with display impl * Make UdpOutgoing look more like TcpOutgoing * AgentError everywhere * allow type complexity * Remove a few extra AgentResult * bump protocol * fix link doc * fix wrong doc * Put global gauges in state so tests dont explode. * Fix repeated value. * changelog * Dont use default registry. * no ignoring alreadyreg * lil docs * Remove example comments from readme. --------- Co-authored-by: Michał Smolarek <34063647+Razz4780@users.noreply.github.com> --- Cargo.lock | 584 ++++++++++-------- changelog.d/2975.added.md | 1 + mirrord-schema.json | 12 +- mirrord/agent/Cargo.toml | 3 + mirrord/agent/README.md | 195 ++++++ mirrord/agent/src/cli.rs | 9 +- mirrord/agent/src/client_connection.rs | 18 +- mirrord/agent/src/container_handle.rs | 4 +- mirrord/agent/src/dns.rs | 18 +- mirrord/agent/src/entrypoint.rs | 62 +- mirrord/agent/src/env.rs | 4 +- mirrord/agent/src/error.rs | 2 +- mirrord/agent/src/file.rs | 88 ++- mirrord/agent/src/main.rs | 4 +- mirrord/agent/src/metrics.rs | 366 +++++++++++ mirrord/agent/src/outgoing.rs | 38 +- mirrord/agent/src/outgoing/udp.rs | 416 ++++++++----- mirrord/agent/src/sniffer.rs | 25 +- mirrord/agent/src/sniffer/api.rs | 22 +- mirrord/agent/src/sniffer/tcp_capture.rs | 4 +- mirrord/agent/src/steal/api.rs | 60 +- mirrord/agent/src/steal/connection.rs | 90 ++- mirrord/agent/src/steal/connections.rs | 18 +- .../agent/src/steal/connections/filtered.rs | 40 +- .../agent/src/steal/connections/unfiltered.rs | 31 +- mirrord/agent/src/steal/ip_tables.rs | 43 +- mirrord/agent/src/steal/ip_tables/chain.rs | 10 +- .../src/steal/ip_tables/flush_connections.rs | 14 +- mirrord/agent/src/steal/ip_tables/mesh.rs | 20 +- .../agent/src/steal/ip_tables/mesh/istio.rs | 14 +- mirrord/agent/src/steal/ip_tables/output.rs | 16 +- .../agent/src/steal/ip_tables/prerouting.rs | 14 +- mirrord/agent/src/steal/ip_tables/redirect.rs | 10 +- mirrord/agent/src/steal/ip_tables/standard.rs | 14 +- mirrord/agent/src/steal/subscriptions.rs | 40 +- mirrord/agent/src/util.rs | 4 +- mirrord/agent/src/vpn.rs | 16 +- mirrord/agent/src/watched_task.rs | 4 +- mirrord/config/configuration.md | 22 +- mirrord/config/src/agent.rs | 22 +- mirrord/config/src/lib.rs | 3 +- mirrord/kube/src/api/container/util.rs | 10 +- mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/codec.rs | 18 + mirrord/protocol/src/error.rs | 2 +- mirrord/protocol/src/lib.rs | 2 + mirrord/protocol/src/outgoing/tcp.rs | 29 + mirrord/protocol/src/outgoing/udp.rs | 36 ++ mirrord/protocol/src/tcp.rs | 45 ++ 49 files changed, 1856 insertions(+), 668 deletions(-) create mode 100644 changelog.d/2975.added.md create mode 100644 mirrord/agent/src/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 1269d9ad275..73c192236bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "bytes", "futures-core", "futures-sink", @@ -31,7 +31,7 @@ dependencies = [ "actix-utils", "ahash", "base64 0.22.1", - "bitflags 2.6.0", + "bitflags 2.8.0", "brotli", "bytes", "bytestring", @@ -65,7 +65,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -182,7 +182,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -299,11 +299,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -321,7 +322,7 @@ dependencies = [ "anyhow", "base64 0.21.7", "bcder", - "bitflags 2.6.0", + "bitflags 2.8.0", "bytes", "chrono", "clap", @@ -358,7 +359,7 @@ dependencies = [ "reqwest 0.11.27", "ring", "scroll", - "semver 1.0.24", + "semver 1.0.25", "serde", "serde_json", "serde_yaml", @@ -440,7 +441,7 @@ checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "synstructure 0.13.1", ] @@ -463,7 +464,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -508,7 +509,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -540,7 +541,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -557,7 +558,7 @@ checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -583,9 +584,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.13" +version = "1.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a50b30228d3af8865ce83376b4e99e1ffa34728220fe2860e4df0bb5278d6" +checksum = "9f40e82e858e02445402906e454a73e244c7f501fcae198977585946c48e8697" dependencies = [ "aws-credential-types", "aws-runtime", @@ -625,9 +626,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.0" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" +checksum = "4c2b7ddaa2c56a367ad27a094ad8ef4faacf8a617c2575acb2ba88949df999ca" dependencies = [ "aws-lc-sys", "paste", @@ -636,9 +637,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" +checksum = "71b2ddd3ada61a305e1d8bb6c005d1eaa7d14d903681edfc400406d523a9b491" dependencies = [ "bindgen", "cc", @@ -650,9 +651,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16d1aa50accc11a4b4d5c50f7fb81cc0cf60328259c587d0e6b0f11385bde46" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -675,9 +676,9 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.53.0" +version = "1.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6493ce2b27a2687b0d8a2453bf6ad2499012e9720c3367cb1206496ede475443" +checksum = "2db64ffe78706b344b7c9b620f96c3c0655745e006b87bad20f424562656a0dd" dependencies = [ "aws-credential-types", "aws-runtime", @@ -697,9 +698,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.53.0" +version = "1.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1605dc0bf9f0a4b05b451441a17fcb0bda229db384f23bf5cead3adbab0664ac" +checksum = "33993c0b054f4251ff2946941b56c26b582677303eeca34087594eb901ece022" dependencies = [ "aws-credential-types", "aws-runtime", @@ -719,9 +720,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.54.0" +version = "1.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f3f73466ff24f6ad109095e0f3f2c830bfb4cd6c8b12f744c8e61ebf4d3ba1" +checksum = "3bd3ceba74a584337a8f3839c818f14f1a2288bfd24235120ff22d7e17a0dd54" dependencies = [ "aws-credential-types", "aws-runtime", @@ -741,9 +742,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.54.0" +version = "1.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249b2acaa8e02fd4718705a9494e3eb633637139aa4bb09d70965b0448e865db" +checksum = "07835598e52dd354368429cb2abf447ce523ea446d0a533a63cb42cd0d2d9280" dependencies = [ "aws-credential-types", "aws-runtime", @@ -764,9 +765,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -787,9 +788,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -798,9 +799,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -818,9 +819,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" dependencies = [ "aws-smithy-types", ] @@ -837,9 +838,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.6" +version = "1.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a05dd41a70fc74051758ee75b5c4db2c0ca070ed9229c3df50e9475cda1cb985" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -881,9 +882,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" dependencies = [ "base64-simd", "bytes", @@ -916,9 +917,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -936,6 +937,7 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", + "axum-macros", "base64 0.22.1", "bytes", "futures-util", @@ -986,6 +988,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "backoff" version = "0.4.0" @@ -1057,9 +1070,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bcder" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627747a6774aab38beb35990d88309481378558875a41da1a4b2e373c906ef0" +checksum = "89ffdaa8c6398acd07176317eb6c1f9082869dd1cc3fee7c72c6354866b928cc" dependencies = [ "bytes", "smallvec", @@ -1096,7 +1109,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1109,7 +1122,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.95", + "syn 2.0.96", "which 4.4.2", ] @@ -1136,9 +1149,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "bitvec" @@ -1218,7 +1231,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-util", "tower-service", @@ -1250,9 +1263,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1380,9 +1393,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.7" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -1470,9 +1483,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.24" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9560b07a799281c7e0958b9296854d6fafd4c5f31444a7e5bb1ad6dde5ccf1bd" +checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" dependencies = [ "clap_builder", "clap_derive", @@ -1480,9 +1493,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.24" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874e0dd3eb68bf99058751ac9712f622e61e6f393a94f7128fa26e3f02f5c7cd" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -1492,9 +1505,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.41" +version = "4.5.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942dc5991a34d8cf58937ec33201856feba9cbceeeab5adf04116ec7c763bff1" +checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0" dependencies = [ "clap", ] @@ -1508,7 +1521,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1582,9 +1595,9 @@ dependencies = [ [[package]] name = "const_panic" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53857514f72ee4a2b583de67401e3ff63a5472ca4acf289d09a9ea7636dfec17" +checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" [[package]] name = "containerd-client" @@ -1708,9 +1721,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-bigint" @@ -1789,7 +1802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1821,7 +1834,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1845,7 +1858,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1856,7 +1869,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1875,9 +1888,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "der" @@ -1932,7 +1945,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1942,7 +1955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1955,7 +1968,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2054,7 +2067,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2116,7 +2129,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2168,7 +2181,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2188,7 +2201,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2200,7 +2213,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2302,9 +2315,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -2569,9 +2582,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1" +checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532" dependencies = [ "futures-core", "pin-project-lite", @@ -2585,7 +2598,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2663,14 +2676,14 @@ dependencies = [ [[package]] name = "getset" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f636605b743120a8d32ed92fc27b6cde1a769f8f936c065151eb66f88ded513c" +checksum = "eded738faa0e88d3abc9d1a13cb11adc2073c400969eeb8793cf7132589959fc" dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2704,7 +2717,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "ignore", "walkdir", ] @@ -2743,7 +2756,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -2762,7 +2775,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.2.0", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -2862,7 +2875,7 @@ dependencies = [ "once_cell", "rand", "serde", - "thiserror 2.0.9", + "thiserror 2.0.11", "tinyvec", "tokio", "tracing", @@ -2886,7 +2899,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -3135,7 +3148,7 @@ dependencies = [ "hyper 1.5.2", "hyper-util", "log", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -3344,7 +3357,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3409,9 +3422,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -3470,9 +3483,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "ipnetwork" @@ -3495,13 +3508,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3608,9 +3621,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -3743,7 +3756,7 @@ dependencies = [ "kube-core", "pem", "rand", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pemfile 2.2.0", "secrecy", "serde", @@ -3784,7 +3797,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3860,16 +3873,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", "redox_syscall", ] [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "listen_ports" @@ -3922,9 +3935,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "loom" @@ -3988,8 +4001,8 @@ dependencies = [ "clap", "glob", "rand", - "syn 2.0.95", - "thiserror 2.0.9", + "syn 2.0.96", + "thiserror 2.0.11", "tracing", "tracing-subscriber", ] @@ -4020,9 +4033,9 @@ dependencies = [ [[package]] name = "mid" -version = "3.0.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82da46e09851e48c020d3460f6f9bf4349153f4bc500ed357fa7e8a2193a16bb" +checksum = "231c7a3a36cd2b353dad91ce88e85f3a2c5079f18b6221495d44de4a8477181e" dependencies = [ "hex", "hmac-sha256", @@ -4056,7 +4069,7 @@ checksum = "23c9b935fbe1d6cbd1dac857b54a688145e2d93f48db36010514d0f612d0ad67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4093,9 +4106,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -4151,14 +4164,14 @@ dependencies = [ "regex", "reqwest 0.12.12", "rstest", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pemfile 2.2.0", - "semver 1.0.24", + "semver 1.0.25", "serde", "serde_json", "socket2", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-rustls 0.26.1", "tokio-stream", @@ -4174,6 +4187,7 @@ version = "3.130.0" dependencies = [ "actix-codec", "async-trait", + "axum", "bollard", "bytes", "clap", @@ -4198,19 +4212,21 @@ dependencies = [ "nix 0.29.0", "oci-spec", "pnet", - "procfs", + "procfs 0.17.0", + "prometheus", "rand", "rawsocket", "rcgen", + "reqwest 0.12.12", "rstest", - "rustls 0.23.20", - "semver 1.0.24", + "rustls 0.23.21", + "semver 1.0.25", "serde", "serde_json", "socket2", "streammap-ext", "test_bin", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-rustls 0.26.1", "tokio-stream", @@ -4251,7 +4267,7 @@ dependencies = [ "reqwest 0.12.12", "serde", "serde_yaml", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", "whoami", @@ -4264,7 +4280,7 @@ version = "3.130.0" dependencies = [ "base64 0.22.1", "bimap", - "bitflags 2.6.0", + "bitflags 2.8.0", "fancy-regex", "ipnet", "k8s-openapi", @@ -4277,7 +4293,7 @@ dependencies = [ "serde_json", "serde_yaml", "tera", - "thiserror 2.0.9", + "thiserror 2.0.11", "toml 0.8.19", "tracing", ] @@ -4289,7 +4305,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4301,7 +4317,7 @@ dependencies = [ "log", "miette", "mirrord-intproxy-protocol", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-util", "tracing", @@ -4325,11 +4341,11 @@ dependencies = [ "mirrord-protocol", "rand", "rstest", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pemfile 2.2.0", - "semver 1.0.24", + "semver 1.0.25", "serde", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-rustls 0.26.1", "tokio-stream", @@ -4342,7 +4358,7 @@ version = "3.130.0" dependencies = [ "bincode", "mirrord-protocol", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", ] @@ -4364,7 +4380,7 @@ dependencies = [ "serde", "serde_json", "shellexpand", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-retry", "tracing", @@ -4407,7 +4423,7 @@ dependencies = [ "tempfile", "test-cdylib", "tests", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", "tracing-subscriber", @@ -4419,7 +4435,7 @@ version = "3.130.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4428,8 +4444,8 @@ version = "3.130.0" dependencies = [ "proc-macro2", "proc-macro2-diagnostics", - "semver 1.0.24", - "syn 2.0.95", + "semver 1.0.25", + "syn 2.0.96", ] [[package]] @@ -4455,11 +4471,11 @@ dependencies = [ "rand", "rstest", "schemars", - "semver 1.0.24", + "semver 1.0.25", "serde", "serde_json", "serde_yaml", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tokio-tungstenite", "tracing", @@ -4477,7 +4493,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.15.0" +version = "1.15.1" dependencies = [ "actix-codec", "bincode", @@ -4492,10 +4508,10 @@ dependencies = [ "libc", "mirrord-macros", "nix 0.29.0", - "semver 1.0.24", + "semver 1.0.25", "serde", "socket2", - "thiserror 2.0.9", + "thiserror 2.0.11", "tracing", ] @@ -4507,7 +4523,7 @@ dependencies = [ "object 0.36.7", "once_cell", "tempfile", - "thiserror 2.0.9", + "thiserror 2.0.11", "tracing", "which 7.0.1", ] @@ -4522,9 +4538,9 @@ dependencies = [ "kube", "mirrord-protocol", "pnet_packet", - "semver 1.0.24", + "semver 1.0.25", "serde_yaml", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", "tun2", @@ -4553,7 +4569,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4583,9 +4599,9 @@ checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] name = "neli" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43" +checksum = "93062a0dce6da2517ea35f301dfc88184ce18d3601ec786a727a87bf535deca9" dependencies = [ "byteorder", "libc", @@ -4595,9 +4611,9 @@ dependencies = [ [[package]] name = "neli-proc-macros" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4" +checksum = "0c8034b7fbb6f9455b2a96c19e6edf8dc9fc34c70449938d8ee3b4df363f61fe" dependencies = [ "either", "proc-macro2", @@ -4625,7 +4641,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "libc", ] @@ -4636,7 +4652,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -4726,7 +4742,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4783,7 +4799,7 @@ dependencies = [ "crc32fast", "flate2", "hashbrown 0.14.5", - "indexmap 2.7.0", + "indexmap 2.7.1", "memchr", "ruzstd 0.5.0", ] @@ -4812,7 +4828,7 @@ dependencies = [ "serde_json", "strum", "strum_macros", - "thiserror 2.0.9", + "thiserror 2.0.11", ] [[package]] @@ -4866,9 +4882,9 @@ version = "3.130.0" [[package]] name = "outref" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "overload" @@ -4974,7 +4990,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5009,7 +5025,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.9", + "thiserror 2.0.11", "ucd-trie", ] @@ -5033,7 +5049,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5054,7 +5070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.7.0", + "indexmap 2.7.1", ] [[package]] @@ -5112,7 +5128,7 @@ checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5178,7 +5194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42cf17e9a1800f5f396bc67d193dc9411b59012a5876445ef450d449881e1016" dependencies = [ "base64 0.22.1", - "indexmap 2.7.0", + "indexmap 2.7.1", "quick-xml", "serde", "time", @@ -5229,7 +5245,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5324,12 +5340,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5383,14 +5399,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -5403,36 +5419,76 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "version_check", "yansi", ] +[[package]] +name = "procfs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4" +dependencies = [ + "bitflags 2.8.0", + "hex", + "lazy_static", + "procfs-core 0.16.0", + "rustix", +] + [[package]] name = "procfs" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "chrono", "flate2", "hex", - "procfs-core", + "procfs-core 0.17.0", "rustix", ] +[[package]] +name = "procfs-core" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29" +dependencies = [ + "bitflags 2.8.0", + "hex", +] + [[package]] name = "procfs-core" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "chrono", "hex", ] +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs 0.16.0", + "protobuf", + "thiserror 1.0.69", +] + [[package]] name = "prost" version = "0.13.4" @@ -5459,7 +5515,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.95", + "syn 2.0.96", "tempfile", ] @@ -5473,7 +5529,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -5485,6 +5541,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "quick-error" version = "1.2.3" @@ -5511,9 +5573,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.21", "socket2", - "thiserror 2.0.9", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -5529,10 +5591,10 @@ dependencies = [ "rand", "ring", "rustc-hash 2.1.0", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-pki-types", "slab", - "thiserror 2.0.9", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -5707,7 +5769,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -5845,7 +5907,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.21", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -5917,7 +5979,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.95", + "syn 2.0.96", "unicode-ident", ] @@ -5983,7 +6045,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.24", + "semver 1.0.25", ] [[package]] @@ -5997,11 +6059,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno 0.3.10", "libc", "linux-raw-sys", @@ -6036,9 +6098,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "aws-lc-rs", "log", @@ -6215,7 +6277,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6247,7 +6309,7 @@ checksum = "7f81c2fde025af7e69b1d1420531c8a8811ca898919db177141a85313b1cb932" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6289,7 +6351,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -6302,7 +6364,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -6331,9 +6393,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -6371,7 +6433,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6382,14 +6444,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "930cfb6e6abf99298aaad7d29abbef7a9999a9a8806a40088f55f0dcec03146b" dependencies = [ "itoa", "memchr", @@ -6415,7 +6477,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6449,7 +6511,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_derive", "serde_json", @@ -6462,7 +6524,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -6690,7 +6752,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6733,9 +6795,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.95" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -6777,7 +6839,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -6950,7 +7012,7 @@ dependencies = [ "regex", "reqwest 0.12.12", "rstest", - "rustls 0.23.20", + "rustls 0.23.21", "serde", "serde_json", "tempfile", @@ -6980,11 +7042,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.9", + "thiserror-impl 2.0.11", ] [[package]] @@ -6995,18 +7057,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.9" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -7086,9 +7148,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -7104,13 +7166,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -7140,7 +7202,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.21", "tokio", ] @@ -7230,7 +7292,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -7278,7 +7340,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -7325,7 +7387,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "base64 0.22.1", - "bitflags 2.6.0", + "bitflags 2.8.0", "bytes", "http 1.2.0", "http-body 1.0.1", @@ -7368,7 +7430,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -7693,18 +7755,18 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ "getrandom", ] [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "version_check" @@ -7763,34 +7825,35 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -7801,9 +7864,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7811,28 +7874,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -7981,7 +8047,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -7992,7 +8058,7 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -8175,9 +8241,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.22" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] @@ -8200,16 +8266,16 @@ checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] name = "wintun-bindings" -version = "0.7.27" +version = "0.7.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e35d3911efde5ee25586385204127ff6a3f251477dcdd3b222775aaa4d95977" +checksum = "67a02981bed4592bcd271f9bfe154228ddbd2fd69e37a7d358da5d3a1251d696" dependencies = [ "blocking", "c2rust-bitfields", "futures", "libloading", "log", - "thiserror 2.0.9", + "thiserror 2.0.11", "windows-sys 0.59.0", ] @@ -8318,9 +8384,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea8b391c9a790b496184c29f7f93b9ed5b16abb306c05415b68bcc16e4d06432" +checksum = "c5b940ebc25896e71dd073bad2dbaa2abfe97b0a391415e22ad1326d9c54e3c4" [[package]] name = "xmlparser" @@ -8381,7 +8447,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "synstructure 0.13.1", ] @@ -8403,7 +8469,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -8423,7 +8489,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "synstructure 0.13.1", ] @@ -8444,7 +8510,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -8466,7 +8532,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] diff --git a/changelog.d/2975.added.md b/changelog.d/2975.added.md new file mode 100644 index 00000000000..17c59da1f7b --- /dev/null +++ b/changelog.d/2975.added.md @@ -0,0 +1 @@ +Add prometheus metrics to the mirrord-agent. diff --git a/mirrord-schema.json b/mirrord-schema.json index 5beadff3c58..19577b952af 100644 --- a/mirrord-schema.json +++ b/mirrord-schema.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "LayerFileConfig", - "description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo use a configuration file in the CLI, use the `-f ` flag. Or if using VSCode Extension or JetBrains plugin, simply create a `.mirrord/mirrord.json` file or use the UI.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"json_log\": false, \"labels\": { \"user\": \"meow\" }, \"annotations\": { \"cats.io/inject\": \"enabled\" }, \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" }, \"mapping\": { \".+_TIMEOUT\": \"1000\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\\\.json\" , \"read_only\": [ \".+\\\\.yaml\", \".+important-file\\\\.txt\" ], \"local\": [ \".+\\\\.js\", \".+\\\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_filter\": { \"header_filter\": \"host: api\\\\..+\" }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": { \"enabled\": true, \"filter\": { \"local\": [\"1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\"] } } }, \"copy_target\": { \"scale_down\": false } }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}", + "description": "mirrord allows for a high degree of customization when it comes to which features you want to enable, and how they should function.\n\nAll of the configuration fields have a default value, so a minimal configuration would be no configuration at all.\n\nThe configuration supports templating using the [Tera](https://keats.github.io/tera/docs/) template engine. Currently we don't provide additional values to the context, if you have anything you want us to provide please let us know.\n\nTo use a configuration file in the CLI, use the `-f ` flag. Or if using VSCode Extension or JetBrains plugin, simply create a `.mirrord/mirrord.json` file or use the UI.\n\nTo help you get started, here are examples of a basic configuration file, and a complete configuration file containing all fields.\n\n### Basic `config.json` {#root-basic}\n\n```json { \"target\": \"pod/bear-pod\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Basic `config.json` with templating {#root-basic-templating}\n\n```json { \"target\": \"{{ get_env(name=\"TARGET\", default=\"pod/fallback\") }}\", \"feature\": { \"env\": true, \"fs\": \"read\", \"network\": true } } ```\n\n### Complete `config.json` {#root-complete}\n\nDon't use this example as a starting point, it's just here to show you all the available options. ```json { \"accept_invalid_certificates\": false, \"skip_processes\": \"ide-debugger\", \"target\": { \"path\": \"pod/bear-pod\", \"namespace\": \"default\" }, \"connect_tcp\": null, \"agent\": { \"log_level\": \"info\", \"json_log\": false, \"labels\": { \"user\": \"meow\" }, \"annotations\": { \"cats.io/inject\": \"enabled\" }, \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": true, \"metrics\": \"0.0.0.0:9000\", }, \"feature\": { \"env\": { \"include\": \"DATABASE_USER;PUBLIC_ENV\", \"exclude\": \"DATABASE_PASSWORD;SECRET_ENV\", \"override\": { \"DATABASE_CONNECTION\": \"db://localhost:7777/my-db\", \"LOCAL_BEAR\": \"panda\" }, \"mapping\": { \".+_TIMEOUT\": \"1000\" } }, \"fs\": { \"mode\": \"write\", \"read_write\": \".+\\\\.json\" , \"read_only\": [ \".+\\\\.yaml\", \".+important-file\\\\.txt\" ], \"local\": [ \".+\\\\.js\", \".+\\\\.mjs\" ] }, \"network\": { \"incoming\": { \"mode\": \"steal\", \"http_filter\": { \"header_filter\": \"host: api\\\\..+\" }, \"port_mapping\": [[ 7777, 8888 ]], \"ignore_localhost\": false, \"ignore_ports\": [9999, 10000] }, \"outgoing\": { \"tcp\": true, \"udp\": true, \"filter\": { \"local\": [\"tcp://1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\", \":53\"] }, \"ignore_localhost\": false, \"unix_streams\": \"bear.+\" }, \"dns\": { \"enabled\": true, \"filter\": { \"local\": [\"1.1.1.0/24:1337\", \"1.1.5.0/24\", \"google.com\"] } } }, \"copy_target\": { \"scale_down\": false } }, \"operator\": true, \"kubeconfig\": \"~/.kube/config\", \"sip_binaries\": \"bash\", \"telemetry\": true, \"kube_context\": \"my-cluster\" } ```\n\n# Options {#root-options}", "type": "object", "properties": { "accept_invalid_certificates": { @@ -255,7 +255,7 @@ "properties": { "annotations": { "title": "agent.annotations {#agent-annotations}", - "description": "Allows setting up custom annotations for the agent Job and Pod.\n\n```json { \"annotations\": { \"cats.io/inject\": \"enabled\" } } ```", + "description": "Allows setting up custom annotations for the agent Job and Pod.\n\n```json { \"annotations\": { \"cats.io/inject\": \"enabled\" \"prometheus.io/scrape\": \"true\", \"prometheus.io/port\": \"9000\" } } ```", "type": [ "object", "null" @@ -378,6 +378,14 @@ "null" ] }, + "metrics": { + "title": "agent.metrics {#agent-metrics}", + "description": "Enables prometheus metrics for the agent pod.\n\nYou might need to add annotations to the agent pod depending on how prometheus is configured to scrape for metrics.\n\n```json { \"metrics\": \"0.0.0.0:9000\" } ```", + "type": [ + "string", + "null" + ] + }, "namespace": { "title": "agent.namespace {#agent-namespace}", "description": "Namespace where the agent shall live. Note: Doesn't work with ephemeral containers. Defaults to the current kubernetes namespace.", diff --git a/mirrord/agent/Cargo.toml b/mirrord/agent/Cargo.toml index cdba788acf3..07757b7900e 100644 --- a/mirrord/agent/Cargo.toml +++ b/mirrord/agent/Cargo.toml @@ -69,6 +69,8 @@ x509-parser = "0.16" rustls.workspace = true envy = "0.4" socket2.workspace = true +prometheus = { version = "0.13", features = ["process"] } +axum = { version = "0.7", features = ["macros"] } iptables = { git = "https://github.com/metalbear-co/rust-iptables.git", rev = "e66c7332e361df3c61a194f08eefe3f40763d624" } rawsocket = { git = "https://github.com/metalbear-co/rawsocket.git" } procfs = "0.17.0" @@ -78,3 +80,4 @@ rstest.workspace = true mockall = "0.13" test_bin = "0.4" rcgen.workspace = true +reqwest.workspace = true diff --git a/mirrord/agent/README.md b/mirrord/agent/README.md index bf077b5fdcf..d7456ead64c 100644 --- a/mirrord/agent/README.md +++ b/mirrord/agent/README.md @@ -6,3 +6,198 @@ Agent part of [mirrord](https://github.com/metalbear-co/mirrord) responsible for mirrord-agent is written in Rust for safety, low memory consumption and performance. mirrord-agent is distributed as a container image (currently only x86) that is published on [GitHub Packages publicly](https://github.com/metalbear-co/mirrord-agent/pkgs/container/mirrord-agent). + +## Enabling prometheus metrics + +To start the metrics server, you'll need to add this config to your `mirrord.json`: + +```json +{ + "agent": { + "metrics": "0.0.0.0:9000", + "annotations": { + "prometheus.io/scrape": "true", + "prometheus.io/port": "9000" + } +} +``` + +Remember to change the `port` in both `metrics` and `annotations`, they have to match, +otherwise prometheus will try to scrape on `port: 80` or other commonly used ports. + +### Installing prometheus + +Run `kubectl apply -f {file-name}.yaml` on these sequences of `yaml` files and you should +get prometheus running in your cluster. You can access the dashboard from your browser at +`http://{cluster-ip}:30909`, if you're using minikube it might be +`http://192.168.49.2:30909`. + +You'll get prometheus running under the `monitoring` namespace, but it'll be able to look +into resources from all namespaces. The config in `configmap.yaml` sets prometheus to look +at pods only, if you want to use it to scrape other stuff, check +[this example](https://github.com/prometheus/prometheus/blob/main/documentation/examples/prometheus-kubernetes.yml). + +1. `create-namespace.yaml` + +```yaml +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +``` + +2. `cluster-role.yaml` + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +``` + +3. `service-account.yaml` + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus + namespace: monitoring +``` + +4. `cluster-role-binding.yaml` + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: monitoring +``` + +5. `configmap.yaml` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: monitoring +data: + prometheus.yml: | + global: + keep_dropped_targets: 100 + + scrape_configs: + - job_name: "kubernetes-pods" + + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod +``` + +- If you make any changes to the 5-configmap.yaml file, remember to `kubectl apply` it + **before** restarting the `prometheus` deployment. + +6. `deployment.yaml` + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring + labels: + app: prometheus +spec: + replicas: 1 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: prometheus + containers: + - name: prometheus + image: prom/prometheus + args: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - name: web + containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus + restartPolicy: Always + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-config +``` + +7. `service.yaml` + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-service + namespace: monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' +spec: + selector: + app: prometheus + type: NodePort + ports: + - port: 8080 + targetPort: 9090 + nodePort: 30909 +``` diff --git a/mirrord/agent/src/cli.rs b/mirrord/agent/src/cli.rs index 07c8aa97503..bbcf23f1816 100644 --- a/mirrord/agent/src/cli.rs +++ b/mirrord/agent/src/cli.rs @@ -1,8 +1,11 @@ #![deny(missing_docs)] +use std::net::SocketAddr; + use clap::{Parser, Subcommand}; use mirrord_protocol::{ - MeshVendor, AGENT_IPV6_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV, + MeshVendor, AGENT_IPV6_ENV, AGENT_METRICS_ENV, AGENT_NETWORK_INTERFACE_ENV, + AGENT_OPERATOR_CERT_ENV, }; const DEFAULT_RUNTIME: &str = "containerd"; @@ -28,6 +31,10 @@ pub struct Args { #[arg(short = 'i', long, env = AGENT_NETWORK_INTERFACE_ENV)] pub network_interface: Option, + /// Controls whether metrics are enabled, and the address to set up the metrics server. + #[arg(long, env = AGENT_METRICS_ENV)] + pub metrics: Option, + /// Return an error after accepting the first client connection, in order to test agent error /// cleanup. /// diff --git a/mirrord/agent/src/client_connection.rs b/mirrord/agent/src/client_connection.rs index 8181e4baabd..7b484cc25da 100644 --- a/mirrord/agent/src/client_connection.rs +++ b/mirrord/agent/src/client_connection.rs @@ -208,7 +208,7 @@ enum ConnectionFramed { #[cfg(test)] mod test { - use std::sync::Arc; + use std::sync::{Arc, Once}; use futures::StreamExt; use mirrord_protocol::ClientCodec; @@ -220,10 +220,19 @@ mod test { use super::*; + static CRYPTO_PROVIDER: Once = Once::new(); + /// Verifies that [`AgentTlsConnector`] correctly accepts a /// connection from a server using the provided certificate. #[tokio::test] async fn agent_tls_connector_valid_cert() { + CRYPTO_PROVIDER.call_once(|| { + rustls::crypto::CryptoProvider::install_default( + rustls::crypto::aws_lc_rs::default_provider(), + ) + .expect("Failed to install crypto provider") + }); + let cert = rcgen::generate_simple_self_signed(vec!["operator".to_string()]).unwrap(); let cert_bytes = cert.cert.der(); let key_bytes = cert.key_pair.serialize_der(); @@ -269,6 +278,13 @@ mod test { /// connection from a server using some other certificate. #[tokio::test] async fn agent_tls_connector_invalid_cert() { + CRYPTO_PROVIDER.call_once(|| { + rustls::crypto::CryptoProvider::install_default( + rustls::crypto::aws_lc_rs::default_provider(), + ) + .expect("Failed to install crypto provider") + }); + let server_cert = rcgen::generate_simple_self_signed(vec!["operator".to_string()]).unwrap(); let cert_bytes = server_cert.cert.der(); let key_bytes = server_cert.key_pair.serialize_der(); diff --git a/mirrord/agent/src/container_handle.rs b/mirrord/agent/src/container_handle.rs index 6e8ba78173d..dd6755e766d 100644 --- a/mirrord/agent/src/container_handle.rs +++ b/mirrord/agent/src/container_handle.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use crate::{ - error::Result, + error::AgentResult, runtime::{Container, ContainerInfo, ContainerRuntime}, }; @@ -22,7 +22,7 @@ pub(crate) struct ContainerHandle(Arc); impl ContainerHandle { /// Retrieve info about the container and initialize this struct. #[tracing::instrument(level = "trace")] - pub(crate) async fn new(container: Container) -> Result { + pub(crate) async fn new(container: Container) -> AgentResult { let ContainerInfo { pid, env: raw_env } = container.get_info().await?; let inner = Inner { pid, raw_env }; diff --git a/mirrord/agent/src/dns.rs b/mirrord/agent/src/dns.rs index 3240856275a..b92487594e0 100644 --- a/mirrord/agent/src/dns.rs +++ b/mirrord/agent/src/dns.rs @@ -16,10 +16,7 @@ use tokio::{ use tokio_util::sync::CancellationToken; use tracing::Level; -use crate::{ - error::{AgentError, Result}, - watched_task::TaskStatus, -}; +use crate::{error::AgentResult, metrics::DNS_REQUEST_COUNT, watched_task::TaskStatus}; #[derive(Debug)] pub(crate) enum ClientGetAddrInfoRequest { @@ -167,6 +164,9 @@ impl DnsWorker { let etc_path = self.etc_path.clone(); let timeout = self.timeout; let attempts = self.attempts; + + DNS_REQUEST_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let support_ipv6 = self.support_ipv6; let lookup_future = async move { let result = Self::do_lookup( @@ -181,15 +181,13 @@ impl DnsWorker { if let Err(result) = message.response_tx.send(result) { tracing::error!(?result, "Failed to send query response"); } + DNS_REQUEST_COUNT.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); }; tokio::spawn(lookup_future); } - pub(crate) async fn run( - mut self, - cancellation_token: CancellationToken, - ) -> Result<(), AgentError> { + pub(crate) async fn run(mut self, cancellation_token: CancellationToken) -> AgentResult<()> { loop { tokio::select! { _ = cancellation_token.cancelled() => break Ok(()), @@ -225,7 +223,7 @@ impl DnsApi { pub(crate) async fn make_request( &mut self, request: ClientGetAddrInfoRequest, - ) -> Result<(), AgentError> { + ) -> AgentResult<()> { let (response_tx, response_rx) = oneshot::channel(); let command = DnsCommand { @@ -244,7 +242,7 @@ impl DnsApi { /// Returns the result of the oldest outstanding DNS request issued with this struct (see /// [`Self::make_request`]). #[tracing::instrument(level = Level::TRACE, skip(self), ret, err)] - pub(crate) async fn recv(&mut self) -> Result { + pub(crate) async fn recv(&mut self) -> AgentResult { let Some(response) = self.responses.next().await else { return future::pending().await; }; diff --git a/mirrord/agent/src/entrypoint.rs b/mirrord/agent/src/entrypoint.rs index 407bf27c33f..ac9157897a0 100644 --- a/mirrord/agent/src/entrypoint.rs +++ b/mirrord/agent/src/entrypoint.rs @@ -12,6 +12,7 @@ use std::{ use client_connection::AgentTlsConnector; use dns::{ClientGetAddrInfoRequest, DnsCommand, DnsWorker}; use futures::TryFutureExt; +use metrics::{start_metrics, CLIENT_COUNT}; use mirrord_protocol::{ClientMessage, DaemonMessage, GetEnvVarsRequest, LogMessage}; use sniffer::tcp_capture::RawSocketTcpCapture; use tokio::{ @@ -32,7 +33,7 @@ use crate::{ client_connection::ClientConnection, container_handle::ContainerHandle, dns::DnsApi, - error::{AgentError, Result}, + error::{AgentError, AgentResult}, file::FileManager, outgoing::{TcpOutgoingApi, UdpOutgoingApi}, runtime::get_container, @@ -72,7 +73,7 @@ struct State { impl State { /// Return [`Err`] if container runtime operations failed. - pub async fn new(args: &Args) -> Result { + pub async fn new(args: &Args) -> AgentResult { let tls_connector = args .operator_tls_cert_pem .clone() @@ -205,6 +206,12 @@ struct ClientConnectionHandler { ready_for_logs: bool, } +impl Drop for ClientConnectionHandler { + fn drop(&mut self) { + CLIENT_COUNT.fetch_sub(1, Ordering::Relaxed); + } +} + impl ClientConnectionHandler { /// Initializes [`ClientConnectionHandler`]. pub async fn new( @@ -212,7 +219,7 @@ impl ClientConnectionHandler { mut connection: ClientConnection, bg_tasks: BackgroundTasks, state: State, - ) -> Result { + ) -> AgentResult { let pid = state.container_pid(); let file_manager = FileManager::new(pid.or_else(|| state.ephemeral.then_some(1))); @@ -238,6 +245,8 @@ impl ClientConnectionHandler { ready_for_logs: false, }; + CLIENT_COUNT.fetch_add(1, Ordering::Relaxed); + Ok(client_handler) } @@ -273,7 +282,7 @@ impl ClientConnectionHandler { id: ClientId, task: BackgroundTask, connection: &mut ClientConnection, - ) -> Result> { + ) -> AgentResult> { if let BackgroundTask::Running(stealer_status, stealer_sender) = task { match TcpStealerApi::new( id, @@ -313,7 +322,7 @@ impl ClientConnectionHandler { /// /// Breaks upon receiver/sender drop. #[tracing::instrument(level = "trace", skip(self))] - async fn start(mut self, cancellation_token: CancellationToken) -> Result<()> { + async fn start(mut self, cancellation_token: CancellationToken) -> AgentResult<()> { let error = loop { select! { message = self.connection.receive() => { @@ -364,7 +373,7 @@ impl ClientConnectionHandler { Ok(message) => self.respond(DaemonMessage::TcpOutgoing(message)).await?, Err(e) => break e, }, - message = self.udp_outgoing_api.daemon_message() => match message { + message = self.udp_outgoing_api.recv_from_task() => match message { Ok(message) => self.respond(DaemonMessage::UdpOutgoing(message)).await?, Err(e) => break e, }, @@ -389,7 +398,7 @@ impl ClientConnectionHandler { /// Sends a [`DaemonMessage`] response to the connected client (`mirrord-layer`). #[tracing::instrument(level = "trace", skip(self))] - async fn respond(&mut self, response: DaemonMessage) -> Result<()> { + async fn respond(&mut self, response: DaemonMessage) -> AgentResult<()> { self.connection.send(response).await.map_err(Into::into) } @@ -397,7 +406,7 @@ impl ClientConnectionHandler { /// /// Returns `false` if the client disconnected. #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] - async fn handle_client_message(&mut self, message: ClientMessage) -> Result { + async fn handle_client_message(&mut self, message: ClientMessage) -> AgentResult { match message { ClientMessage::FileRequest(req) => { if let Some(response) = self.file_manager.handle_message(req)? { @@ -415,7 +424,7 @@ impl ClientConnectionHandler { self.tcp_outgoing_api.send_to_task(layer_message).await? } ClientMessage::UdpOutgoing(layer_message) => { - self.udp_outgoing_api.layer_message(layer_message).await? + self.udp_outgoing_api.send_to_task(layer_message).await? } ClientMessage::GetEnvVarsRequest(GetEnvVarsRequest { env_vars_filter, @@ -495,8 +504,8 @@ impl ClientConnectionHandler { } /// Initializes the agent's [`State`], channels, threads, and runs [`ClientConnectionHandler`]s. -#[tracing::instrument(level = "trace", ret)] -async fn start_agent(args: Args) -> Result<()> { +#[tracing::instrument(level = Level::TRACE, ret, err)] +async fn start_agent(args: Args) -> AgentResult<()> { trace!("start_agent -> Starting agent with args: {args:?}"); // listen for client connections @@ -534,6 +543,18 @@ async fn start_agent(args: Args) -> Result<()> { // To make sure that background tasks are cancelled when we exit early from this function. let cancel_guard = cancellation_token.clone().drop_guard(); + if let Some(metrics_address) = args.metrics { + let cancellation_token = cancellation_token.clone(); + tokio::spawn(async move { + start_metrics(metrics_address, cancellation_token.clone()) + .await + .inspect_err(|fail| { + tracing::error!(?fail, "Failed starting metrics server!"); + cancellation_token.cancel(); + }) + }); + } + let (sniffer_command_tx, sniffer_command_rx) = mpsc::channel::(1000); let (stealer_command_tx, stealer_command_rx) = mpsc::channel::(1000); let (dns_command_tx, dns_command_rx) = mpsc::channel::(1000); @@ -755,7 +776,7 @@ async fn start_agent(args: Args) -> Result<()> { Ok(()) } -async fn clear_iptable_chain() -> Result<()> { +async fn clear_iptable_chain() -> AgentResult<()> { let ipt = new_iptables(); SafeIpTables::load(IPTablesWrapper::from(ipt), false) @@ -766,7 +787,7 @@ async fn clear_iptable_chain() -> Result<()> { Ok(()) } -async fn run_child_agent() -> Result<()> { +async fn run_child_agent() -> AgentResult<()> { let command_args = std::env::args().collect::>(); let (command, args) = command_args .split_first() @@ -790,7 +811,7 @@ async fn run_child_agent() -> Result<()> { /// /// Captures SIGTERM signals sent by Kubernetes when the pod is gracefully deleted. /// When a signal is captured, the child process is killed and the iptables are cleaned. -async fn start_iptable_guard(args: Args) -> Result<()> { +async fn start_iptable_guard(args: Args) -> AgentResult<()> { debug!("start_iptable_guard -> Initializing iptable-guard."); let state = State::new(&args).await?; @@ -827,7 +848,18 @@ async fn start_iptable_guard(args: Args) -> Result<()> { result } -pub async fn main() -> Result<()> { +/// The agent is somewhat started twice, first with [`start_iptable_guard`], and then the +/// proper agent with [`start_agent`]. +/// +/// ## Things to keep in mind due to the double initialization +/// +/// Since the _second_ agent gets spawned as a child of the _first_, they share resources, +/// like the `namespace`, which means: +/// +/// 1. If you try to `bind` a socket to some address before [`start_agent`], it'll actually be bound +/// **twice**, which incurs an error (address already in use). You could get around this by +/// `bind`ing on `0.0.0.0:0`, but this is most likely **not** what you want. +pub async fn main() -> AgentResult<()> { rustls::crypto::CryptoProvider::install_default(rustls::crypto::aws_lc_rs::default_provider()) .expect("Failed to install crypto provider"); diff --git a/mirrord/agent/src/env.rs b/mirrord/agent/src/env.rs index 26fa4681431..5a349709f2d 100644 --- a/mirrord/agent/src/env.rs +++ b/mirrord/agent/src/env.rs @@ -7,7 +7,7 @@ use mirrord_protocol::RemoteResult; use tokio::io::AsyncReadExt; use wildmatch::WildMatch; -use crate::error::Result; +use crate::error::AgentResult; struct EnvFilter { include: Vec, @@ -97,7 +97,7 @@ pub(crate) fn parse_raw_env<'a, S: AsRef + 'a + ?Sized, T: IntoIterator>() } -pub(crate) async fn get_proc_environ(path: PathBuf) -> Result> { +pub(crate) async fn get_proc_environ(path: PathBuf) -> AgentResult> { let mut environ_file = tokio::fs::File::open(path).await?; let mut raw_env_vars = String::with_capacity(8192); diff --git a/mirrord/agent/src/error.rs b/mirrord/agent/src/error.rs index d9ae7cb8b9d..88b811e590b 100644 --- a/mirrord/agent/src/error.rs +++ b/mirrord/agent/src/error.rs @@ -96,4 +96,4 @@ impl From> for AgentError { } } -pub(crate) type Result = std::result::Result; +pub(crate) type AgentResult = std::result::Result; diff --git a/mirrord/agent/src/file.rs b/mirrord/agent/src/file.rs index 0bc30afb151..0fa945fbd85 100644 --- a/mirrord/agent/src/file.rs +++ b/mirrord/agent/src/file.rs @@ -18,7 +18,7 @@ use mirrord_protocol::{file::*, FileRequest, FileResponse, RemoteResult, Respons use nix::unistd::UnlinkatFlags; use tracing::{error, trace, Level}; -use crate::error::Result; +use crate::{error::AgentResult, metrics::OPEN_FD_COUNT}; #[derive(Debug)] pub enum RemoteFile { @@ -76,15 +76,11 @@ pub(crate) struct FileManager { fds_iter: RangeInclusive, } -impl Default for FileManager { - fn default() -> Self { - Self { - root_path: Default::default(), - open_files: Default::default(), - dir_streams: Default::default(), - getdents_streams: Default::default(), - fds_iter: (0..=u64::MAX), - } +impl Drop for FileManager { + fn drop(&mut self) { + let descriptors = + self.open_files.len() + self.dir_streams.len() + self.getdents_streams.len(); + OPEN_FD_COUNT.fetch_sub(descriptors as i64, std::sync::atomic::Ordering::Relaxed); } } @@ -152,7 +148,10 @@ pub fn resolve_path + std::fmt::Debug, R: AsRef + std::fmt: impl FileManager { /// Executes the request and returns the response. #[tracing::instrument(level = Level::TRACE, skip(self), ret, err(level = Level::DEBUG))] - pub fn handle_message(&mut self, request: FileRequest) -> Result> { + pub(crate) fn handle_message( + &mut self, + request: FileRequest, + ) -> AgentResult> { Ok(match request { FileRequest::Open(OpenFileRequest { path, open_options }) => { // TODO: maybe not agent error on this? @@ -206,10 +205,7 @@ impl FileManager { let write_result = self.write_limited(remote_fd, start_from, write_bytes); Some(FileResponse::WriteLimited(write_result)) } - FileRequest::Close(CloseFileRequest { fd }) => { - self.close(fd); - None - } + FileRequest::Close(CloseFileRequest { fd }) => self.close(fd), FileRequest::Access(AccessFileRequest { pathname, mode }) => { let pathname = pathname .strip_prefix("/") @@ -244,10 +240,7 @@ impl FileManager { let read_dir_result = self.read_dir_batch(remote_fd, amount); Some(FileResponse::ReadDirBatch(read_dir_result)) } - FileRequest::CloseDir(CloseDirRequest { remote_fd }) => { - self.close_dir(remote_fd); - None - } + FileRequest::CloseDir(CloseDirRequest { remote_fd }) => self.close_dir(remote_fd), FileRequest::GetDEnts64(GetDEnts64Request { remote_fd, buffer_size, @@ -280,10 +273,13 @@ impl FileManager { pub fn new(pid: Option) -> Self { let root_path = get_root_path_from_optional_pid(pid); trace!("Agent root path >> {root_path:?}"); + Self { - open_files: HashMap::new(), root_path, - ..Default::default() + open_files: Default::default(), + dir_streams: Default::default(), + getdents_streams: Default::default(), + fds_iter: (0..=u64::MAX), } } @@ -309,7 +305,9 @@ impl FileManager { RemoteFile::File(file) }; - self.open_files.insert(fd, remote_file); + if self.open_files.insert(fd, remote_file).is_none() { + OPEN_FD_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } Ok(OpenFileResponse { fd }) } @@ -343,7 +341,9 @@ impl FileManager { RemoteFile::File(file) }; - self.open_files.insert(fd, remote_file); + if self.open_files.insert(fd, remote_file).is_none() { + OPEN_FD_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } Ok(OpenFileResponse { fd }) } else { @@ -636,20 +636,36 @@ impl FileManager { }) } - pub(crate) fn close(&mut self, fd: u64) { - trace!("FileManager::close -> fd {:#?}", fd,); - + /// Always returns `None`, since we don't return any [`FileResponse`] back to mirrord + /// on `close` of an fd. + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(crate) fn close(&mut self, fd: u64) -> Option { if self.open_files.remove(&fd).is_none() { - error!("FileManager::close -> fd {:#?} not found", fd); + error!(fd, "fd not found!"); + } else { + OPEN_FD_COUNT.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); } - } - pub(crate) fn close_dir(&mut self, fd: u64) { - trace!("FileManager::close_dir -> fd {:#?}", fd,); + None + } - if self.dir_streams.remove(&fd).is_none() && self.getdents_streams.remove(&fd).is_none() { + /// Always returns `None`, since we don't return any [`FileResponse`] back to mirrord + /// on `close_dir` of an fd. + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(crate) fn close_dir(&mut self, fd: u64) -> Option { + let closed_dir_stream = self.dir_streams.remove(&fd); + let closed_getdents_stream = self.getdents_streams.remove(&fd); + + if closed_dir_stream.is_some() && closed_getdents_stream.is_some() { + // Closed `dirstream` and `dentsstream` + OPEN_FD_COUNT.fetch_sub(2, std::sync::atomic::Ordering::Relaxed); + } else if closed_dir_stream.is_some() || closed_getdents_stream.is_some() { + OPEN_FD_COUNT.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } else { error!("FileManager::close_dir -> fd {:#?} not found", fd); } + + None } pub(crate) fn access( @@ -753,7 +769,7 @@ impl FileManager { }) } - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self), err(level = Level::DEBUG))] pub(crate) fn fdopen_dir(&mut self, fd: u64) -> RemoteResult { let path = match self .open_files @@ -770,7 +786,10 @@ impl FileManager { .ok_or_else(|| ResponseError::IdsExhausted("fdopen_dir".to_string()))?; let dir_stream = path.read_dir()?.enumerate(); - self.dir_streams.insert(fd, dir_stream); + + if self.dir_streams.insert(fd, dir_stream).is_none() { + OPEN_FD_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } Ok(OpenDirResponse { fd }) } @@ -819,7 +838,7 @@ impl FileManager { /// The possible remote errors are: /// [`ResponseError::NotFound`] if there is not such fd here. /// [`ResponseError::NotDirectory`] if the fd points to a file with a non-directory file type. - #[tracing::instrument(level = "trace", skip(self))] + #[tracing::instrument(level = Level::TRACE, skip(self))] pub(crate) fn get_or_create_getdents64_stream( &mut self, fd: u64, @@ -832,6 +851,7 @@ impl FileManager { let current_and_parent = Self::get_current_and_parent_entries(dir); let stream = GetDEnts64Stream::new(dir.read_dir()?, current_and_parent).peekable(); + OPEN_FD_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(e.insert(stream)) } }, diff --git a/mirrord/agent/src/main.rs b/mirrord/agent/src/main.rs index 305ec50e0ed..e9f3e107907 100644 --- a/mirrord/agent/src/main.rs +++ b/mirrord/agent/src/main.rs @@ -22,6 +22,7 @@ mod env; mod error; mod file; mod http; +mod metrics; mod namespace; mod outgoing; mod runtime; @@ -31,7 +32,8 @@ mod util; mod vpn; mod watched_task; +#[cfg(target_os = "linux")] #[tokio::main(flavor = "current_thread")] -async fn main() -> crate::error::Result<()> { +async fn main() -> crate::error::AgentResult<()> { crate::entrypoint::main().await } diff --git a/mirrord/agent/src/metrics.rs b/mirrord/agent/src/metrics.rs new file mode 100644 index 00000000000..59fefbc2305 --- /dev/null +++ b/mirrord/agent/src/metrics.rs @@ -0,0 +1,366 @@ +use std::{ + net::SocketAddr, + sync::{atomic::AtomicI64, Arc}, +}; + +use axum::{extract::State, routing::get, Router}; +use http::StatusCode; +use prometheus::{proto::MetricFamily, IntGauge, Registry}; +use tokio::net::TcpListener; +use tokio_util::sync::CancellationToken; +use tracing::Level; + +use crate::error::AgentError; + +/// Incremented whenever we get a new client in `ClientConnectionHandler`, and decremented +/// when this client is dropped. +pub(crate) static CLIENT_COUNT: AtomicI64 = AtomicI64::new(0); + +/// Incremented whenever we handle a new `DnsCommand`, and decremented after the result of +/// `do_lookup` has been sent back through the response channel. +pub(crate) static DNS_REQUEST_COUNT: AtomicI64 = AtomicI64::new(0); + +/// Incremented and decremented in _open-ish_/_close-ish_ file operations in `FileManager`, +/// Also gets decremented when `FileManager` is dropped. +pub(crate) static OPEN_FD_COUNT: AtomicI64 = AtomicI64::new(0); + +/// Follows the amount of subscribed ports in `update_packet_filter`. We don't really +/// increment/decrement this one, and mostly `set` it to the latest amount of ports, zeroing it when +/// the `TcpConnectionSniffer` gets dropped. +pub(crate) static MIRROR_PORT_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static MIRROR_CONNECTION_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static STEAL_FILTERED_PORT_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static STEAL_UNFILTERED_PORT_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static STEAL_FILTERED_CONNECTION_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static HTTP_REQUEST_IN_PROGRESS_COUNT: AtomicI64 = AtomicI64::new(0); + +pub(crate) static TCP_OUTGOING_CONNECTION: AtomicI64 = AtomicI64::new(0); + +pub(crate) static UDP_OUTGOING_CONNECTION: AtomicI64 = AtomicI64::new(0); + +/// The state with all the metrics [`IntGauge`]s and the prometheus [`Registry`] where we keep them. +/// +/// **Do not** modify the gauges directly! +/// +/// Instead rely on [`Metrics::gather_metrics`], as we actually use a bunch of [`AtomicI64`]s to +/// keep track of the values, they are the ones being (de|in)cremented. These gauges are just set +/// when it's time to send them via [`get_metrics`]. +#[derive(Debug)] +struct Metrics { + registry: Registry, + client_count: IntGauge, + dns_request_count: IntGauge, + open_fd_count: IntGauge, + mirror_port_subscription: IntGauge, + mirror_connection_subscription: IntGauge, + steal_filtered_port_subscription: IntGauge, + steal_unfiltered_port_subscription: IntGauge, + steal_filtered_connection_subscription: IntGauge, + steal_unfiltered_connection_subscription: IntGauge, + http_request_in_progress_count: IntGauge, + tcp_outgoing_connection: IntGauge, + udp_outgoing_connection: IntGauge, +} + +impl Metrics { + /// Creates a [`Registry`] to ... register our [`IntGauge`]s. + fn new() -> Self { + use prometheus::Opts; + + let registry = Registry::new(); + + let client_count = { + let opts = Opts::new( + "mirrord_agent_client_count", + "amount of connected clients to this mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let dns_request_count = { + let opts = Opts::new( + "mirrord_agent_dns_request_count", + "amount of in-progress dns requests in the mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let open_fd_count = { + let opts = Opts::new( + "mirrord_agent_open_fd_count", + "amount of open file descriptors in mirrord-agent file manager", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let mirror_port_subscription = { + let opts = Opts::new( + "mirrord_agent_mirror_port_subscription_count", + "amount of mirror port subscriptions in mirror-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let mirror_connection_subscription = { + let opts = Opts::new( + "mirrord_agent_mirror_connection_subscription_count", + "amount of connections in mirror mode in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let steal_filtered_port_subscription = { + let opts = Opts::new( + "mirrord_agent_steal_filtered_port_subscription_count", + "amount of filtered steal port subscriptions in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let steal_unfiltered_port_subscription = { + let opts = Opts::new( + "mirrord_agent_steal_unfiltered_port_subscription_count", + "amount of unfiltered steal port subscriptions in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let steal_filtered_connection_subscription = { + let opts = Opts::new( + "mirrord_agent_steal_connection_subscription_count", + "amount of filtered connections in steal mode in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let steal_unfiltered_connection_subscription = { + let opts = Opts::new( + "mirrord_agent_steal_unfiltered_connection_subscription_count", + "amount of unfiltered connections in steal mode in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let http_request_in_progress_count = { + let opts = Opts::new( + "mirrord_agent_http_request_in_progress_count", + "amount of in-progress http requests in the mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let tcp_outgoing_connection = { + let opts = Opts::new( + "mirrord_agent_tcp_outgoing_connection_count", + "amount of tcp outgoing connections in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + let udp_outgoing_connection = { + let opts = Opts::new( + "mirrord_agent_udp_outgoing_connection_count", + "amount of udp outgoing connections in mirrord-agent", + ); + IntGauge::with_opts(opts).expect("Valid at initialization!") + }; + + registry + .register(Box::new(client_count.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(dns_request_count.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(open_fd_count.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(mirror_port_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(mirror_connection_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(steal_filtered_port_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(steal_unfiltered_port_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(steal_filtered_connection_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(steal_unfiltered_connection_subscription.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(http_request_in_progress_count.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(tcp_outgoing_connection.clone())) + .expect("Register must be valid at initialization!"); + registry + .register(Box::new(udp_outgoing_connection.clone())) + .expect("Register must be valid at initialization!"); + + Self { + registry, + client_count, + dns_request_count, + open_fd_count, + mirror_port_subscription, + mirror_connection_subscription, + steal_filtered_port_subscription, + steal_unfiltered_port_subscription, + steal_filtered_connection_subscription, + steal_unfiltered_connection_subscription, + http_request_in_progress_count, + tcp_outgoing_connection, + udp_outgoing_connection, + } + } + + /// Calls [`IntGauge::set`] on every [`IntGauge`] of `Self`, setting it to the value of + /// the corresponding [`AtomicI64`] global (the uppercase named version of the gauge). + /// + /// Returns the list of [`MetricFamily`] registered in our [`Metrics::registry`], ready to be + /// encoded and sent to prometheus. + fn gather_metrics(&self) -> Vec { + use std::sync::atomic::Ordering; + + let Self { + registry, + client_count, + dns_request_count, + open_fd_count, + mirror_port_subscription, + mirror_connection_subscription, + steal_filtered_port_subscription, + steal_unfiltered_port_subscription, + steal_filtered_connection_subscription, + steal_unfiltered_connection_subscription, + http_request_in_progress_count, + tcp_outgoing_connection, + udp_outgoing_connection, + } = self; + + client_count.set(CLIENT_COUNT.load(Ordering::Relaxed)); + dns_request_count.set(DNS_REQUEST_COUNT.load(Ordering::Relaxed)); + open_fd_count.set(OPEN_FD_COUNT.load(Ordering::Relaxed)); + mirror_port_subscription.set(MIRROR_PORT_SUBSCRIPTION.load(Ordering::Relaxed)); + mirror_connection_subscription.set(MIRROR_CONNECTION_SUBSCRIPTION.load(Ordering::Relaxed)); + steal_filtered_port_subscription + .set(STEAL_FILTERED_PORT_SUBSCRIPTION.load(Ordering::Relaxed)); + steal_unfiltered_port_subscription + .set(STEAL_UNFILTERED_PORT_SUBSCRIPTION.load(Ordering::Relaxed)); + steal_filtered_connection_subscription + .set(STEAL_FILTERED_CONNECTION_SUBSCRIPTION.load(Ordering::Relaxed)); + steal_unfiltered_connection_subscription + .set(STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.load(Ordering::Relaxed)); + http_request_in_progress_count.set(HTTP_REQUEST_IN_PROGRESS_COUNT.load(Ordering::Relaxed)); + tcp_outgoing_connection.set(TCP_OUTGOING_CONNECTION.load(Ordering::Relaxed)); + udp_outgoing_connection.set(UDP_OUTGOING_CONNECTION.load(Ordering::Relaxed)); + + registry.gather() + } +} + +/// `GET /metrics` +/// +/// Prepares all the metrics with [`Metrics::gather_metrics`], and responds to the prometheus +/// request. +#[tracing::instrument(level = Level::TRACE, ret)] +async fn get_metrics(State(state): State>) -> (StatusCode, String) { + use prometheus::TextEncoder; + + let metric_families = state.gather_metrics(); + match TextEncoder.encode_to_string(&metric_families) { + Ok(response) => (StatusCode::OK, response), + Err(fail) => { + tracing::error!(?fail, "Failed GET /metrics"); + (StatusCode::INTERNAL_SERVER_ERROR, fail.to_string()) + } + } +} + +/// Starts the mirrord-agent prometheus metrics service. +/// +/// You can get the metrics from `GET address/metrics`. +/// +/// - `address`: comes from a mirrord-agent config. +#[tracing::instrument(level = Level::TRACE, skip_all, ret ,err)] +pub(crate) async fn start_metrics( + address: SocketAddr, + cancellation_token: CancellationToken, +) -> Result<(), axum::BoxError> { + let metrics_state = Arc::new(Metrics::new()); + + let app = Router::new() + .route("/metrics", get(get_metrics)) + .with_state(metrics_state); + + let listener = TcpListener::bind(address) + .await + .map_err(AgentError::from) + .inspect_err(|fail| { + tracing::error!(?fail, "Failed to bind TCP socket for metrics server") + })?; + + let cancel_on_error = cancellation_token.clone(); + axum::serve(listener, app) + .with_graceful_shutdown(async move { cancellation_token.cancelled().await }) + .await + .inspect_err(|fail| { + tracing::error!(%fail, "Could not start agent metrics server!"); + cancel_on_error.cancel(); + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{sync::atomic::Ordering, time::Duration}; + + use tokio_util::sync::CancellationToken; + + use super::OPEN_FD_COUNT; + use crate::metrics::start_metrics; + + #[tokio::test] + async fn test_metrics() { + let metrics_address = "127.0.0.1:9000".parse().unwrap(); + let cancellation_token = CancellationToken::new(); + + let metrics_cancellation = cancellation_token.child_token(); + tokio::spawn(async move { + start_metrics(metrics_address, metrics_cancellation) + .await + .unwrap() + }); + + OPEN_FD_COUNT.fetch_add(1, Ordering::Relaxed); + + // Give the server some time to start. + tokio::time::sleep(Duration::from_secs(1)).await; + + let get_all_metrics = reqwest::get("http://127.0.0.1:9000/metrics") + .await + .unwrap() + .error_for_status() + .unwrap() + .text() + .await + .unwrap(); + + assert!(get_all_metrics.contains("mirrord_agent_open_fd_count 1")); + + cancellation_token.drop_guard(); + } +} diff --git a/mirrord/agent/src/outgoing.rs b/mirrord/agent/src/outgoing.rs index 13e3a9e1e06..96a063d7a05 100644 --- a/mirrord/agent/src/outgoing.rs +++ b/mirrord/agent/src/outgoing.rs @@ -18,7 +18,8 @@ use tokio_util::io::ReaderStream; use tracing::Level; use crate::{ - error::Result, + error::AgentResult, + metrics::TCP_OUTGOING_CONNECTION, util::run_thread_in_namespace, watched_task::{TaskStatus, WatchedTask}, }; @@ -81,7 +82,7 @@ impl TcpOutgoingApi { /// Sends the [`LayerTcpOutgoing`] message to the background task. #[tracing::instrument(level = Level::TRACE, skip(self), err)] - pub(crate) async fn send_to_task(&mut self, message: LayerTcpOutgoing) -> Result<()> { + pub(crate) async fn send_to_task(&mut self, message: LayerTcpOutgoing) -> AgentResult<()> { if self.layer_tx.send(message).await.is_ok() { Ok(()) } else { @@ -91,7 +92,7 @@ impl TcpOutgoingApi { /// Receives a [`DaemonTcpOutgoing`] message from the background task. #[tracing::instrument(level = Level::TRACE, skip(self), err)] - pub(crate) async fn recv_from_task(&mut self) -> Result { + pub(crate) async fn recv_from_task(&mut self) -> AgentResult { match self.daemon_rx.recv().await { Some(msg) => Ok(msg), None => Err(self.task_status.unwrap_err().await), @@ -112,6 +113,13 @@ struct TcpOutgoingTask { daemon_tx: Sender, } +impl Drop for TcpOutgoingTask { + fn drop(&mut self) { + let connections = self.readers.keys().chain(self.writers.keys()).count(); + TCP_OUTGOING_CONNECTION.fetch_sub(connections as i64, std::sync::atomic::Ordering::Relaxed); + } +} + impl fmt::Debug for TcpOutgoingTask { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TcpOutgoingTask") @@ -152,7 +160,7 @@ impl TcpOutgoingTask { /// Runs this task as long as the channels connecting it with [`TcpOutgoingApi`] are open. /// This routine never fails and returns [`Result`] only due to [`WatchedTask`] constraints. #[tracing::instrument(level = Level::TRACE, skip(self))] - async fn run(mut self) -> Result<()> { + async fn run(mut self) -> AgentResult<()> { loop { let channel_closed = select! { biased; @@ -216,6 +224,7 @@ impl TcpOutgoingTask { self.readers.remove(&connection_id); self.writers.remove(&connection_id); + TCP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); let daemon_message = DaemonTcpOutgoing::Close(connection_id); self.daemon_tx.send(daemon_message).await?; @@ -246,6 +255,8 @@ impl TcpOutgoingTask { "Layer connection is shut down as well, sending close message.", ); + TCP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + self.daemon_tx .send(DaemonTcpOutgoing::Close(connection_id)) .await?; @@ -287,6 +298,7 @@ impl TcpOutgoingTask { connection_id, ReaderStream::with_capacity(read_half, Self::READ_BUFFER_SIZE), ); + TCP_OUTGOING_CONNECTION.fetch_add(1, std::sync::atomic::Ordering::Relaxed); Ok(DaemonConnect { connection_id, @@ -299,9 +311,12 @@ impl TcpOutgoingTask { result = ?daemon_connect, "Connection attempt finished.", ); + self.daemon_tx .send(DaemonTcpOutgoing::Connect(daemon_connect)) - .await + .await?; + + Ok(()) } // This message handles two cases: @@ -341,9 +356,14 @@ impl TcpOutgoingTask { connection_id, "Peer connection is shut down as well, sending close message to the client.", ); + TCP_OUTGOING_CONNECTION + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + self.daemon_tx .send(DaemonTcpOutgoing::Close(connection_id)) - .await + .await?; + + Ok(()) } } @@ -352,6 +372,7 @@ impl TcpOutgoingTask { Err(error) => { self.writers.remove(&connection_id); self.readers.remove(&connection_id); + TCP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); tracing::trace!( connection_id, @@ -360,7 +381,9 @@ impl TcpOutgoingTask { ); self.daemon_tx .send(DaemonTcpOutgoing::Close(connection_id)) - .await + .await?; + + Ok(()) } } } @@ -370,6 +393,7 @@ impl TcpOutgoingTask { LayerTcpOutgoing::Close(LayerClose { connection_id }) => { self.writers.remove(&connection_id); self.readers.remove(&connection_id); + TCP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); Ok(()) } diff --git a/mirrord/agent/src/outgoing/udp.rs b/mirrord/agent/src/outgoing/udp.rs index b6baa5e537e..0dab137a92b 100644 --- a/mirrord/agent/src/outgoing/udp.rs +++ b/mirrord/agent/src/outgoing/udp.rs @@ -1,10 +1,11 @@ +use core::fmt; use std::{ collections::HashMap, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, thread, }; -use bytes::BytesMut; +use bytes::{Bytes, BytesMut}; use futures::{ prelude::*, stream::{SplitSink, SplitStream}, @@ -15,21 +16,262 @@ use mirrord_protocol::{ }; use streammap_ext::StreamMap; use tokio::{ + io, net::UdpSocket, select, - sync::mpsc::{self, Receiver, Sender}, + sync::mpsc::{self, error::SendError, Receiver, Sender}, }; use tokio_util::{codec::BytesCodec, udp::UdpFramed}; -use tracing::{debug, trace, warn}; +use tracing::Level; use crate::{ - error::Result, + error::AgentResult, + metrics::UDP_OUTGOING_CONNECTION, util::run_thread_in_namespace, watched_task::{TaskStatus, WatchedTask}, }; -type Layer = LayerUdpOutgoing; -type Daemon = DaemonUdpOutgoing; +/// Task that handles [`LayerUdpOutgoing`] and [`DaemonUdpOutgoing`] messages. +/// +/// We start these tasks from the [`UdpOutgoingApi`] as a [`WatchedTask`]. +struct UdpOutgoingTask { + next_connection_id: ConnectionId, + /// Writing halves of peer connections made on layer's requests. + #[allow(clippy::type_complexity)] + writers: HashMap< + ConnectionId, + ( + SplitSink, (BytesMut, SocketAddr)>, + SocketAddr, + ), + >, + /// Reading halves of peer connections made on layer's requests. + readers: StreamMap>>, + /// Optional pid of agent's target. Used in `SocketStream::connect`. + pid: Option, + layer_rx: Receiver, + daemon_tx: Sender, +} + +impl Drop for UdpOutgoingTask { + fn drop(&mut self) { + let connections = self.readers.keys().chain(self.writers.keys()).count(); + UDP_OUTGOING_CONNECTION.fetch_sub(connections as i64, std::sync::atomic::Ordering::Relaxed); + } +} + +impl fmt::Debug for UdpOutgoingTask { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("UdpOutgoingTask") + .field("next_connection_id", &self.next_connection_id) + .field("writers", &self.writers.len()) + .field("readers", &self.readers.len()) + .field("pid", &self.pid) + .finish() + } +} + +impl UdpOutgoingTask { + fn new( + pid: Option, + layer_rx: Receiver, + daemon_tx: Sender, + ) -> Self { + Self { + next_connection_id: 0, + writers: Default::default(), + readers: Default::default(), + pid, + layer_rx, + daemon_tx, + } + } + + /// Runs this task as long as the channels connecting it with [`UdpOutgoingApi`] are open. + /// This routine never fails and returns [`AgentResult`] only due to [`WatchedTask`] + /// constraints. + #[tracing::instrument(level = Level::TRACE, skip(self))] + pub(super) async fn run(mut self) -> AgentResult<()> { + loop { + let channel_closed = select! { + biased; + + message = self.layer_rx.recv() => match message { + // We have a message from the layer to be handled. + Some(message) => { + self.handle_layer_msg(message).await.is_err() + }, + // Our channel with the layer is closed, this task is no longer needed. + None => true, + }, + + // We have data coming from one of our peers. + Some((connection_id, remote_read)) = self.readers.next() => { + self.handle_connection_read(connection_id, remote_read.transpose().map(|remote| remote.map(|(read, _)| read.into()))).await.is_err() + }, + }; + + if channel_closed { + tracing::trace!("Client channel closed, exiting"); + break Ok(()); + } + } + } + + /// Returns [`Err`] only when the client has disconnected. + #[tracing::instrument( + level = Level::TRACE, + skip(read), + fields(read = ?read.as_ref().map(|data| data.as_ref().map(Bytes::len).unwrap_or_default())) + err(level = Level::TRACE) + )] + async fn handle_connection_read( + &mut self, + connection_id: ConnectionId, + read: io::Result>, + ) -> Result<(), SendError> { + match read { + Ok(Some(read)) => { + let message = DaemonUdpOutgoing::Read(Ok(DaemonRead { + connection_id, + bytes: read.to_vec(), + })); + + self.daemon_tx.send(message).await? + } + // An error occurred when reading from a peer connection. + // We remove both io halves and inform the layer that the connection is closed. + // We remove the reader, because otherwise the `StreamMap` will produce an extra `None` + // item from the related stream. + Err(error) => { + tracing::trace!( + ?error, + connection_id, + "Reading from peer connection failed, sending close message.", + ); + + self.readers.remove(&connection_id); + self.writers.remove(&connection_id); + UDP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + let daemon_message = DaemonUdpOutgoing::Close(connection_id); + self.daemon_tx.send(daemon_message).await?; + } + Ok(None) => { + self.writers.remove(&connection_id); + self.readers.remove(&connection_id); + UDP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + let daemon_message = DaemonUdpOutgoing::Close(connection_id); + self.daemon_tx.send(daemon_message).await?; + } + } + + Ok(()) + } + + /// Returns [`Err`] only when the client has disconnected. + #[allow(clippy::type_complexity)] + #[tracing::instrument(level = Level::TRACE, ret)] + async fn handle_layer_msg( + &mut self, + message: LayerUdpOutgoing, + ) -> Result<(), SendError> { + match message { + // [user] -> [layer] -> [agent] -> [layer] + // `user` is asking us to connect to some remote host. + LayerUdpOutgoing::Connect(LayerConnect { remote_address }) => { + let daemon_connect = + connect(remote_address.clone()) + .await + .and_then(|mirror_socket| { + let connection_id = self.next_connection_id; + self.next_connection_id += 1; + + let peer_address = mirror_socket.peer_addr()?; + let local_address = mirror_socket.local_addr()?; + let local_address = SocketAddress::Ip(local_address); + + let framed = UdpFramed::new(mirror_socket, BytesCodec::new()); + + let (sink, stream): ( + SplitSink, (BytesMut, SocketAddr)>, + SplitStream>, + ) = framed.split(); + + self.writers.insert(connection_id, (sink, peer_address)); + self.readers.insert(connection_id, stream); + UDP_OUTGOING_CONNECTION + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + Ok(DaemonConnect { + connection_id, + remote_address, + local_address, + }) + }); + + tracing::trace!( + result = ?daemon_connect, + "Connection attempt finished.", + ); + + self.daemon_tx + .send(DaemonUdpOutgoing::Connect(daemon_connect)) + .await?; + + Ok(()) + } + // [user] -> [layer] -> [agent] -> [remote] + // `user` wrote some message to the remote host. + LayerUdpOutgoing::Write(LayerWrite { + connection_id, + bytes, + }) => { + let write_result = match self + .writers + .get_mut(&connection_id) + .ok_or(ResponseError::NotFound(connection_id)) + { + Ok((mirror, remote_address)) => mirror + .send((BytesMut::from(bytes.as_slice()), *remote_address)) + .await + .map_err(ResponseError::from), + Err(fail) => Err(fail), + }; + + match write_result { + Ok(()) => Ok(()), + Err(error) => { + self.writers.remove(&connection_id); + self.readers.remove(&connection_id); + UDP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + tracing::trace!( + connection_id, + ?error, + "Failed to handle layer write, sending close message to the client.", + ); + + let daemon_message = DaemonUdpOutgoing::Close(connection_id); + self.daemon_tx.send(daemon_message).await?; + + Ok(()) + } + } + } + // [layer] -> [agent] + // `layer` closed their interceptor stream. + LayerUdpOutgoing::Close(LayerClose { ref connection_id }) => { + self.writers.remove(connection_id); + self.readers.remove(connection_id); + UDP_OUTGOING_CONNECTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + + Ok(()) + } + } + } +} /// Handles (briefly) the `UdpOutgoingRequest` and `UdpOutgoingResponse` messages, mostly the /// passing of these messages to the `interceptor_task` thread. @@ -41,10 +283,10 @@ pub(crate) struct UdpOutgoingApi { task_status: TaskStatus, /// Sends the `Layer` message to the `interceptor_task`. - layer_tx: Sender, + layer_tx: Sender, /// Reads the `Daemon` message from the `interceptor_task`. - daemon_rx: Receiver, + daemon_rx: Receiver, } /// Performs an [`UdpSocket::connect`] that handles 3 situations: @@ -55,8 +297,9 @@ pub(crate) struct UdpOutgoingApi { /// read access to `/etc/resolv.conf`, otherwise they'll be getting a mismatched connection; /// 3. User is trying to use `sendto` and `recvfrom`, we use the same hack as in DNS to fake a /// connection. -#[tracing::instrument(level = "trace", ret)] -async fn connect(remote_address: SocketAddr) -> Result { +#[tracing::instrument(level = Level::TRACE, ret, err(level = Level::DEBUG))] +async fn connect(remote_address: SocketAddress) -> Result { + let remote_address = remote_address.try_into()?; let mirror_address = match remote_address { std::net::SocketAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), std::net::SocketAddr::V6(_) => SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0), @@ -75,8 +318,10 @@ impl UdpOutgoingApi { let (layer_tx, layer_rx) = mpsc::channel(1000); let (daemon_tx, daemon_rx) = mpsc::channel(1000); - let watched_task = - WatchedTask::new(Self::TASK_NAME, Self::interceptor_task(layer_rx, daemon_tx)); + let watched_task = WatchedTask::new( + Self::TASK_NAME, + UdpOutgoingTask::new(pid, layer_rx, daemon_tx).run(), + ); let task_status = watched_task.status(); let task = run_thread_in_namespace( @@ -94,150 +339,9 @@ impl UdpOutgoingApi { } } - /// The [`UdpOutgoingApi`] task. - /// - /// Receives [`LayerUdpOutgoing`] messages and replies with [`DaemonUdpOutgoing`]. - #[allow(clippy::type_complexity)] - async fn interceptor_task( - mut layer_rx: Receiver, - daemon_tx: Sender, - ) -> Result<()> { - let mut connection_ids = 0..=ConnectionId::MAX; - - // TODO: Right now we're manually keeping these 2 maps in sync (aviram suggested using - // `Weak` for `writers`). - let mut writers: HashMap< - ConnectionId, - ( - SplitSink, (BytesMut, SocketAddr)>, - SocketAddr, - ), - > = HashMap::default(); - - let mut readers: StreamMap>> = - StreamMap::default(); - - loop { - select! { - biased; - - // [layer] -> [agent] - Some(layer_message) = layer_rx.recv() => { - trace!("udp: interceptor_task -> layer_message {:?}", layer_message); - match layer_message { - // [user] -> [layer] -> [agent] -> [layer] - // `user` is asking us to connect to some remote host. - LayerUdpOutgoing::Connect(LayerConnect { remote_address }) => { - let daemon_connect = connect(remote_address.clone().try_into()?) - .await - .and_then(|mirror_socket| { - let connection_id = connection_ids - .next() - .ok_or_else(|| ResponseError::IdsExhausted("connect".into()))?; - - debug!("interceptor_task -> mirror_socket {:#?}", mirror_socket); - let peer_address = mirror_socket.peer_addr()?; - let local_address = mirror_socket.local_addr()?; - let local_address = SocketAddress::Ip(local_address); - let framed = UdpFramed::new(mirror_socket, BytesCodec::new()); - debug!("interceptor_task -> framed {:#?}", framed); - let (sink, stream): ( - SplitSink, (BytesMut, SocketAddr)>, - SplitStream>, - ) = framed.split(); - - writers.insert(connection_id, (sink, peer_address)); - readers.insert(connection_id, stream); - - Ok(DaemonConnect { - connection_id, - remote_address, - local_address - }) - }); - - let daemon_message = DaemonUdpOutgoing::Connect(daemon_connect); - debug!("interceptor_task -> daemon_message {:#?}", daemon_message); - daemon_tx.send(daemon_message).await? - } - // [user] -> [layer] -> [agent] -> [remote] - // `user` wrote some message to the remote host. - LayerUdpOutgoing::Write(LayerWrite { - connection_id, - bytes, - }) => { - let daemon_write = match writers - .get_mut(&connection_id) - .ok_or(ResponseError::NotFound(connection_id)) - { - Ok((mirror, remote_address)) => mirror - .send((BytesMut::from(bytes.as_slice()), *remote_address)) - .await - .map_err(ResponseError::from), - Err(fail) => Err(fail), - }; - - if let Err(fail) = daemon_write { - warn!("LayerUdpOutgoing::Write -> Failed with {:#?}", fail); - writers.remove(&connection_id); - readers.remove(&connection_id); - - let daemon_message = DaemonUdpOutgoing::Close(connection_id); - daemon_tx.send(daemon_message).await? - } - } - // [layer] -> [agent] - // `layer` closed their interceptor stream. - LayerUdpOutgoing::Close(LayerClose { ref connection_id }) => { - writers.remove(connection_id); - readers.remove(connection_id); - } - } - } - - // [remote] -> [agent] -> [layer] -> [user] - // Read the data from one of the connected remote hosts, and forward the result back - // to the `user`. - Some((connection_id, remote_read)) = readers.next() => { - trace!("interceptor_task -> read connection_id {:#?}", connection_id); - - match remote_read { - Some(read) => { - let daemon_read = read - .map_err(ResponseError::from) - .map(|(bytes, _)| DaemonRead { connection_id, bytes: bytes.to_vec() }); - - let daemon_message = DaemonUdpOutgoing::Read(daemon_read); - daemon_tx.send(daemon_message).await? - } - None => { - trace!("interceptor_task -> close connection {:#?}", connection_id); - writers.remove(&connection_id); - readers.remove(&connection_id); - - let daemon_message = DaemonUdpOutgoing::Close(connection_id); - daemon_tx.send(daemon_message).await? - } - } - } - else => { - // We have no more data coming from any of the remote hosts. - warn!("interceptor_task -> no messages left"); - break; - } - } - } - - Ok(()) - } - /// Sends a `UdpOutgoingRequest` to the `interceptor_task`. - pub(crate) async fn layer_message(&mut self, message: LayerUdpOutgoing) -> Result<()> { - trace!( - "UdpOutgoingApi::layer_message -> layer_message {:#?}", - message - ); - + #[tracing::instrument(level = Level::TRACE, skip(self), err)] + pub(crate) async fn send_to_task(&mut self, message: LayerUdpOutgoing) -> AgentResult<()> { if self.layer_tx.send(message).await.is_ok() { Ok(()) } else { @@ -246,7 +350,7 @@ impl UdpOutgoingApi { } /// Receives a `UdpOutgoingResponse` from the `interceptor_task`. - pub(crate) async fn daemon_message(&mut self) -> Result { + pub(crate) async fn recv_from_task(&mut self) -> AgentResult { match self.daemon_rx.recv().await { Some(msg) => Ok(msg), None => Err(self.task_status.unwrap_err().await), diff --git a/mirrord/agent/src/sniffer.rs b/mirrord/agent/src/sniffer.rs index b1c232eae6d..0d5cccfb584 100644 --- a/mirrord/agent/src/sniffer.rs +++ b/mirrord/agent/src/sniffer.rs @@ -24,8 +24,9 @@ use self::{ tcp_capture::RawSocketTcpCapture, }; use crate::{ - error::AgentError, + error::AgentResult, http::HttpVersion, + metrics::{MIRROR_CONNECTION_SUBSCRIPTION, MIRROR_PORT_SUBSCRIPTION}, util::{ChannelClosedFuture, ClientId, Subscriptions}, }; @@ -141,6 +142,13 @@ pub(crate) struct TcpConnectionSniffer { clients_closed: FuturesUnordered>, } +impl Drop for TcpConnectionSniffer { + fn drop(&mut self) { + MIRROR_PORT_SUBSCRIPTION.store(0, std::sync::atomic::Ordering::Relaxed); + MIRROR_CONNECTION_SUBSCRIPTION.store(0, std::sync::atomic::Ordering::Relaxed); + } +} + impl fmt::Debug for TcpConnectionSniffer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("TcpConnectionSniffer") @@ -163,7 +171,7 @@ impl TcpConnectionSniffer { command_rx: Receiver, network_interface: Option, is_mesh: bool, - ) -> Result { + ) -> AgentResult { let tcp_capture = RawSocketTcpCapture::new(network_interface, is_mesh).await?; Ok(Self { @@ -190,7 +198,7 @@ where /// Runs the sniffer loop, capturing packets. #[tracing::instrument(level = Level::DEBUG, skip(cancel_token), err)] - pub async fn start(mut self, cancel_token: CancellationToken) -> Result<(), AgentError> { + pub async fn start(mut self, cancel_token: CancellationToken) -> AgentResult<()> { loop { select! { command = self.command_rx.recv() => { @@ -232,7 +240,7 @@ where /// Removes the client with `client_id`, and also unsubscribes its port. /// Adjusts BPF filter if needed. #[tracing::instrument(level = Level::TRACE, err)] - fn handle_client_closed(&mut self, client_id: ClientId) -> Result<(), AgentError> { + fn handle_client_closed(&mut self, client_id: ClientId) -> AgentResult<()> { self.client_txs.remove(&client_id); if self.port_subscriptions.remove_client(client_id) { @@ -245,8 +253,9 @@ where /// Updates BPF filter used by [`Self::tcp_capture`] to match state of /// [`Self::port_subscriptions`]. #[tracing::instrument(level = Level::TRACE, err)] - fn update_packet_filter(&mut self) -> Result<(), AgentError> { + fn update_packet_filter(&mut self) -> AgentResult<()> { let ports = self.port_subscriptions.get_subscribed_topics(); + MIRROR_PORT_SUBSCRIPTION.store(ports.len() as i64, std::sync::atomic::Ordering::Relaxed); let filter = if ports.is_empty() { tracing::trace!("No ports subscribed, setting dummy bpf"); @@ -261,7 +270,7 @@ where } #[tracing::instrument(level = Level::TRACE, err)] - fn handle_command(&mut self, command: SnifferCommand) -> Result<(), AgentError> { + fn handle_command(&mut self, command: SnifferCommand) -> AgentResult<()> { match command { SnifferCommand { client_id, @@ -325,7 +334,7 @@ where &mut self, identifier: TcpSessionIdentifier, tcp_packet: TcpPacketData, - ) -> Result<(), AgentError> { + ) -> AgentResult<()> { let data_tx = match self.sessions.entry(identifier) { Entry::Occupied(e) => e, Entry::Vacant(e) => { @@ -394,6 +403,7 @@ where } } + MIRROR_CONNECTION_SUBSCRIPTION.fetch_add(1, std::sync::atomic::Ordering::Relaxed); e.insert_entry(data_tx) } }; @@ -448,6 +458,7 @@ mod test { async fn get_api(&mut self) -> TcpSnifferApi { let client_id = self.next_client_id; self.next_client_id += 1; + TcpSnifferApi::new(client_id, self.command_tx.clone(), self.task_status.clone()) .await .unwrap() diff --git a/mirrord/agent/src/sniffer/api.rs b/mirrord/agent/src/sniffer/api.rs index 31ec4107f97..08874e93124 100644 --- a/mirrord/agent/src/sniffer/api.rs +++ b/mirrord/agent/src/sniffer/api.rs @@ -14,12 +14,17 @@ use tokio_stream::{ StreamMap, StreamNotifyClose, }; -use super::messages::{SniffedConnection, SnifferCommand, SnifferCommandInner}; +use super::{ + messages::{SniffedConnection, SnifferCommand, SnifferCommandInner}, + AgentResult, +}; use crate::{error::AgentError, util::ClientId, watched_task::TaskStatus}; /// Interface used by clients to interact with the /// [`TcpConnectionSniffer`](super::TcpConnectionSniffer). Multiple instances of this struct operate /// on a single sniffer instance. +/// +/// Enabled by the `mirror` feature for incoming traffic. pub(crate) struct TcpSnifferApi { /// Id of the client using this struct. client_id: ClientId, @@ -55,7 +60,7 @@ impl TcpSnifferApi { client_id: ClientId, sniffer_sender: Sender, mut task_status: TaskStatus, - ) -> Result { + ) -> AgentResult { let (sender, receiver) = mpsc::channel(Self::CONNECTION_CHANNEL_SIZE); let command = SnifferCommand { @@ -79,7 +84,7 @@ impl TcpSnifferApi { /// Send the given command to the connected /// [`TcpConnectionSniffer`](super::TcpConnectionSniffer). - async fn send_command(&mut self, command: SnifferCommandInner) -> Result<(), AgentError> { + async fn send_command(&mut self, command: SnifferCommandInner) -> AgentResult<()> { let command = SnifferCommand { client_id: self.client_id, command, @@ -94,7 +99,7 @@ impl TcpSnifferApi { /// Return the next message from the connected /// [`TcpConnectionSniffer`](super::TcpConnectionSniffer). - pub async fn recv(&mut self) -> Result<(DaemonTcp, Option), AgentError> { + pub async fn recv(&mut self) -> AgentResult<(DaemonTcp, Option)> { tokio::select! { conn = self.receiver.recv() => match conn { Some(conn) => { @@ -158,27 +163,26 @@ impl TcpSnifferApi { } } - /// Tansform the given message into a [`SnifferCommand`] and pass it to the connected + /// Tansforms a [`LayerTcp`] message into a [`SnifferCommand`] and passes it to the connected /// [`TcpConnectionSniffer`](super::TcpConnectionSniffer). - pub async fn handle_client_message(&mut self, message: LayerTcp) -> Result<(), AgentError> { + pub async fn handle_client_message(&mut self, message: LayerTcp) -> AgentResult<()> { match message { LayerTcp::PortSubscribe(port) => { let (tx, rx) = oneshot::channel(); self.send_command(SnifferCommandInner::Subscribe(port, tx)) .await?; self.subscriptions_in_progress.push(rx); - Ok(()) } LayerTcp::PortUnsubscribe(port) => { self.send_command(SnifferCommandInner::UnsubscribePort(port)) - .await + .await?; + Ok(()) } LayerTcp::ConnectionUnsubscribe(connection_id) => { self.connections.remove(&connection_id); - Ok(()) } } diff --git a/mirrord/agent/src/sniffer/tcp_capture.rs b/mirrord/agent/src/sniffer/tcp_capture.rs index 1d8031d08b3..dc8fb2bba04 100644 --- a/mirrord/agent/src/sniffer/tcp_capture.rs +++ b/mirrord/agent/src/sniffer/tcp_capture.rs @@ -12,7 +12,7 @@ use rawsocket::{filter::SocketFilterProgram, RawCapture}; use tokio::net::UdpSocket; use tracing::Level; -use super::{TcpPacketData, TcpSessionIdentifier}; +use super::{AgentResult, TcpPacketData, TcpSessionIdentifier}; use crate::error::AgentError; /// Trait for structs that are able to sniff incoming Ethernet packets and filter TCP packets. @@ -36,7 +36,7 @@ impl RawSocketTcpCapture { /// /// Returned instance initially uses a BPF filter that drops every packet. #[tracing::instrument(level = Level::DEBUG, err)] - pub async fn new(network_interface: Option, is_mesh: bool) -> Result { + pub async fn new(network_interface: Option, is_mesh: bool) -> AgentResult { // Priority is whatever the user set as an option to mirrord, then we check if we're in a // mesh to use `lo` interface, otherwise we try to get the appropriate interface. let interface = match network_interface.or_else(|| is_mesh.then(|| "lo".to_string())) { diff --git a/mirrord/agent/src/steal/api.rs b/mirrord/agent/src/steal/api.rs index 15d2f265ba7..2fc5733f8fa 100644 --- a/mirrord/agent/src/steal/api.rs +++ b/mirrord/agent/src/steal/api.rs @@ -8,11 +8,11 @@ use mirrord_protocol::{ }; use tokio::sync::mpsc::{self, Receiver, Sender}; use tokio_stream::wrappers::ReceiverStream; +use tracing::Level; use super::{http::ReceiverStreamBody, *}; use crate::{ - error::{AgentError, Result}, - util::ClientId, + error::AgentResult, metrics::HTTP_REQUEST_IN_PROGRESS_COUNT, util::ClientId, watched_task::TaskStatus, }; @@ -50,17 +50,23 @@ pub(crate) struct TcpStealerApi { response_body_txs: HashMap<(ConnectionId, RequestId), ResponseBodyTx>, } +impl Drop for TcpStealerApi { + fn drop(&mut self) { + HTTP_REQUEST_IN_PROGRESS_COUNT.store(0, std::sync::atomic::Ordering::Relaxed); + } +} + impl TcpStealerApi { /// Initializes a [`TcpStealerApi`] and sends a message to [`TcpConnectionStealer`] signaling /// that we have a new client. - #[tracing::instrument(level = "trace")] + #[tracing::instrument(level = Level::TRACE, err)] pub(crate) async fn new( client_id: ClientId, command_tx: Sender, task_status: TaskStatus, channel_size: usize, protocol_version: semver::Version, - ) -> Result { + ) -> AgentResult { let (daemon_tx, daemon_rx) = mpsc::channel(channel_size); command_tx @@ -80,7 +86,7 @@ impl TcpStealerApi { } /// Send `command` to stealer, with the client id of the client that is using this API instance. - async fn send_command(&mut self, command: Command) -> Result<()> { + async fn send_command(&mut self, command: Command) -> AgentResult<()> { let command = StealerCommand { client_id: self.client_id, command, @@ -98,12 +104,16 @@ impl TcpStealerApi { /// /// Called in the `ClientConnectionHandler`. #[tracing::instrument(level = "trace", skip(self))] - pub(crate) async fn recv(&mut self) -> Result { + pub(crate) async fn recv(&mut self) -> AgentResult { match self.daemon_rx.recv().await { Some(msg) => { if let DaemonTcp::Close(close) = &msg { self.response_body_txs .retain(|(key_id, _), _| *key_id != close.connection_id); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); } Ok(msg) } @@ -115,7 +125,7 @@ impl TcpStealerApi { /// agent, to an internal stealer command [`Command::PortSubscribe`]. /// /// The actual handling of this message is done in [`TcpConnectionStealer`]. - pub(crate) async fn port_subscribe(&mut self, port_steal: StealType) -> Result<(), AgentError> { + pub(crate) async fn port_subscribe(&mut self, port_steal: StealType) -> AgentResult<()> { self.send_command(Command::PortSubscribe(port_steal)).await } @@ -123,7 +133,7 @@ impl TcpStealerApi { /// agent, to an internal stealer command [`Command::PortUnsubscribe`]. /// /// The actual handling of this message is done in [`TcpConnectionStealer`]. - pub(crate) async fn port_unsubscribe(&mut self, port: Port) -> Result<(), AgentError> { + pub(crate) async fn port_unsubscribe(&mut self, port: Port) -> AgentResult<()> { self.send_command(Command::PortUnsubscribe(port)).await } @@ -134,7 +144,7 @@ impl TcpStealerApi { pub(crate) async fn connection_unsubscribe( &mut self, connection_id: ConnectionId, - ) -> Result<(), AgentError> { + ) -> AgentResult<()> { self.send_command(Command::ConnectionUnsubscribe(connection_id)) .await } @@ -143,7 +153,7 @@ impl TcpStealerApi { /// agent, to an internal stealer command [`Command::ResponseData`]. /// /// The actual handling of this message is done in [`TcpConnectionStealer`]. - pub(crate) async fn client_data(&mut self, tcp_data: TcpData) -> Result<(), AgentError> { + pub(crate) async fn client_data(&mut self, tcp_data: TcpData) -> AgentResult<()> { self.send_command(Command::ResponseData(tcp_data)).await } @@ -154,24 +164,32 @@ impl TcpStealerApi { pub(crate) async fn http_response( &mut self, response: HttpResponseFallback, - ) -> Result<(), AgentError> { + ) -> AgentResult<()> { self.send_command(Command::HttpResponse(response)).await } pub(crate) async fn switch_protocol_version( &mut self, version: semver::Version, - ) -> Result<(), AgentError> { + ) -> AgentResult<()> { self.send_command(Command::SwitchProtocolVersion(version)) .await } - pub(crate) async fn handle_client_message(&mut self, message: LayerTcpSteal) -> Result<()> { + pub(crate) async fn handle_client_message( + &mut self, + message: LayerTcpSteal, + ) -> AgentResult<()> { match message { LayerTcpSteal::PortSubscribe(port_steal) => self.port_subscribe(port_steal).await, LayerTcpSteal::ConnectionUnsubscribe(connection_id) => { self.response_body_txs .retain(|(key_id, _), _| *key_id != connection_id); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); + self.connection_unsubscribe(connection_id).await } LayerTcpSteal::PortUnsubscribe(port) => self.port_unsubscribe(port).await, @@ -202,6 +220,10 @@ impl TcpStealerApi { let key = (response.connection_id, response.request_id); self.response_body_txs.insert(key, tx.clone()); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); self.http_response(HttpResponseFallback::Streamed(http_response)) .await?; @@ -209,6 +231,10 @@ impl TcpStealerApi { for frame in response.internal_response.body { if let Err(err) = tx.send(Ok(frame.into())).await { self.response_body_txs.remove(&key); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); tracing::trace!(?err, "error while sending streaming response frame"); } } @@ -231,12 +257,20 @@ impl TcpStealerApi { } if send_err || body.is_last { self.response_body_txs.remove(key); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); }; Ok(()) } ChunkedResponse::Error(err) => { self.response_body_txs .remove(&(err.connection_id, err.request_id)); + HTTP_REQUEST_IN_PROGRESS_COUNT.store( + self.response_body_txs.len() as i64, + std::sync::atomic::Ordering::Relaxed, + ); tracing::trace!(?err, "ChunkedResponse error received"); Ok(()) } diff --git a/mirrord/agent/src/steal/connection.rs b/mirrord/agent/src/steal/connection.rs index 5e4b6b1219a..6515f2ecfc9 100644 --- a/mirrord/agent/src/steal/connection.rs +++ b/mirrord/agent/src/steal/connection.rs @@ -28,11 +28,12 @@ use tokio::{ sync::mpsc::{Receiver, Sender}, }; use tokio_util::sync::CancellationToken; -use tracing::warn; +use tracing::{warn, Level}; use super::http::HttpResponseFallback; use crate::{ - error::{AgentError, Result}, + error::AgentResult, + metrics::HTTP_REQUEST_IN_PROGRESS_COUNT, steal::{ connections::{ ConnectionMessageIn, ConnectionMessageOut, StolenConnection, StolenConnections, @@ -55,6 +56,22 @@ struct MatchedHttpRequest { } impl MatchedHttpRequest { + fn new( + connection_id: ConnectionId, + port: Port, + request_id: RequestId, + request: Request, + ) -> Self { + HTTP_REQUEST_IN_PROGRESS_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + Self { + connection_id, + port, + request_id, + request, + } + } + async fn into_serializable(self) -> Result, hyper::Error> { let ( Parts { @@ -181,7 +198,7 @@ impl Client { let frames = frames .into_iter() .map(InternalHttpBodyFrame::try_from) - .filter_map(Result::ok) + .filter_map(AgentResult::ok) .collect(); let message = DaemonTcp::HttpRequestChunked(ChunkedRequest::Start(HttpRequest { @@ -210,7 +227,7 @@ impl Client { let frames = frames .into_iter() .map(InternalHttpBodyFrame::try_from) - .filter_map(Result::ok) + .filter_map(AgentResult::ok) .collect(); let message = DaemonTcp::HttpRequestChunked(ChunkedRequest::Body( ChunkedHttpBody { @@ -273,6 +290,8 @@ struct TcpStealerConfig { /// Meant to be run (see [`TcpConnectionStealer::start`]) in a separate thread while the agent /// lives. When handling port subscription requests, this struct manipulates iptables, so it should /// run in the same network namespace as the agent's target. +/// +/// Enabled by the `steal` feature for incoming traffic. pub(crate) struct TcpConnectionStealer { /// For managing active subscriptions and port redirections. port_subscriptions: PortSubscriptions, @@ -299,11 +318,11 @@ impl TcpConnectionStealer { /// Initializes a new [`TcpConnectionStealer`], but doesn't start the actual work. /// You need to call [`TcpConnectionStealer::start`] to do so. - #[tracing::instrument(level = "trace")] + #[tracing::instrument(level = Level::TRACE, err)] pub(crate) async fn new( command_rx: Receiver, support_ipv6: bool, - ) -> Result { + ) -> AgentResult { let config = envy::prefixed("MIRRORD_AGENT_") .from_env::() .unwrap_or_default(); @@ -341,10 +360,7 @@ impl TcpConnectionStealer { /// /// 4. Handling the cancellation of the whole stealer thread (given `cancellation_token`). #[tracing::instrument(level = "trace", skip(self))] - pub(crate) async fn start( - mut self, - cancellation_token: CancellationToken, - ) -> Result<(), AgentError> { + pub(crate) async fn start(mut self, cancellation_token: CancellationToken) -> AgentResult<()> { loop { tokio::select! { command = self.command_rx.recv() => { @@ -362,7 +378,9 @@ impl TcpConnectionStealer { }, accept = self.port_subscriptions.next_connection() => match accept { - Ok((stream, peer)) => self.incoming_connection(stream, peer).await?, + Ok((stream, peer)) => { + self.incoming_connection(stream, peer).await?; + } Err(error) => { tracing::error!(?error, "Failed to accept a stolen connection"); break Err(error); @@ -380,7 +398,11 @@ impl TcpConnectionStealer { /// Handles a new remote connection that was stolen by [`Self::port_subscriptions`]. #[tracing::instrument(level = "trace", skip(self))] - async fn incoming_connection(&mut self, stream: TcpStream, peer: SocketAddr) -> Result<()> { + async fn incoming_connection( + &mut self, + stream: TcpStream, + peer: SocketAddr, + ) -> AgentResult<()> { let mut real_address = orig_dst::orig_dst_addr(&stream)?; let localhost = if self.support_ipv6 && real_address.is_ipv6() { IpAddr::V6(Ipv6Addr::LOCALHOST) @@ -416,10 +438,7 @@ impl TcpConnectionStealer { /// Handles an update from one of the connections in [`Self::connections`]. #[tracing::instrument(level = "trace", skip(self))] - async fn handle_connection_update( - &mut self, - update: ConnectionMessageOut, - ) -> Result<(), AgentError> { + async fn handle_connection_update(&mut self, update: ConnectionMessageOut) -> AgentResult<()> { match update { ConnectionMessageOut::Closed { connection_id, @@ -526,12 +545,7 @@ impl TcpConnectionStealer { return Ok(()); } - let matched_request = MatchedHttpRequest { - connection_id, - request, - request_id: id, - port, - }; + let matched_request = MatchedHttpRequest::new(connection_id, port, id, request); if !client.send_request_async(matched_request) { self.connections @@ -550,11 +564,18 @@ impl TcpConnectionStealer { Ok(()) } - /// Helper function to handle [`Command::PortSubscribe`] messages. + /// Helper function to handle [`Command::PortSubscribe`] messages for the `TcpStealer`. /// - /// Inserts a subscription into [`Self::port_subscriptions`]. - #[tracing::instrument(level = "trace", skip(self))] - async fn port_subscribe(&mut self, client_id: ClientId, port_steal: StealType) -> Result<()> { + /// Checks if [`StealType`] is a valid [`HttpFilter`], then inserts a subscription into + /// [`Self::port_subscriptions`]. + /// + /// - Returns: `true` if this is an HTTP filtered subscription. + #[tracing::instrument(level = Level::TRACE, skip(self), err)] + async fn port_subscribe( + &mut self, + client_id: ClientId, + port_steal: StealType, + ) -> AgentResult { let spec = match port_steal { StealType::All(port) => Ok((port, None)), StealType::FilteredHttp(port, filter) => Regex::new(&format!("(?i){filter}")) @@ -565,6 +586,11 @@ impl TcpConnectionStealer { .map_err(|err| BadHttpFilterExRegex(filter, err.to_string())), }; + let filtered = spec + .as_ref() + .map(|(_, filter)| filter.is_some()) + .unwrap_or_default(); + let res = match spec { Ok((port, filter)) => self.port_subscriptions.add(client_id, port, filter).await?, Err(e) => Err(e.into()), @@ -573,18 +599,18 @@ impl TcpConnectionStealer { let client = self.clients.get(&client_id).expect("client not found"); let _ = client.tx.send(DaemonTcp::SubscribeResult(res)).await; - Ok(()) + Ok(filtered) } /// Removes the client with `client_id` from our list of clients (layers), and also removes /// their subscriptions from [`Self::port_subscriptions`] and all their open /// connections. #[tracing::instrument(level = "trace", skip(self))] - async fn close_client(&mut self, client_id: ClientId) -> Result<(), AgentError> { + async fn close_client(&mut self, client_id: ClientId) -> AgentResult<()> { self.port_subscriptions.remove_all(client_id).await?; let client = self.clients.remove(&client_id).expect("client not found"); - for connection in client.subscribed_connections.into_iter() { + for connection in client.subscribed_connections { self.connections .send(connection, ConnectionMessageIn::Unsubscribed { client_id }) .await; @@ -612,8 +638,8 @@ impl TcpConnectionStealer { } /// Handles [`Command`]s that were received by [`TcpConnectionStealer::command_rx`]. - #[tracing::instrument(level = "trace", skip(self))] - async fn handle_command(&mut self, command: StealerCommand) -> Result<(), AgentError> { + #[tracing::instrument(level = Level::TRACE, skip(self), err)] + async fn handle_command(&mut self, command: StealerCommand) -> AgentResult<()> { let StealerCommand { client_id, command } = command; match command { @@ -644,7 +670,7 @@ impl TcpConnectionStealer { } Command::PortSubscribe(port_steal) => { - self.port_subscribe(client_id, port_steal).await? + self.port_subscribe(client_id, port_steal).await?; } Command::PortUnsubscribe(port) => { diff --git a/mirrord/agent/src/steal/connections.rs b/mirrord/agent/src/steal/connections.rs index bc47eb80e4b..8e969b0a868 100644 --- a/mirrord/agent/src/steal/connections.rs +++ b/mirrord/agent/src/steal/connections.rs @@ -11,10 +11,14 @@ use tokio::{ sync::mpsc::{self, error::SendError, Receiver, Sender}, task::JoinSet, }; +use tracing::Level; use self::{filtered::DynamicBody, unfiltered::UnfilteredStealTask}; use super::{http::DefaultReversibleStream, subscriptions::PortSubscription}; -use crate::{http::HttpVersion, steal::connections::filtered::FilteredStealTask, util::ClientId}; +use crate::{ + http::HttpVersion, metrics::STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION, + steal::connections::filtered::FilteredStealTask, util::ClientId, +}; mod filtered; mod unfiltered; @@ -287,7 +291,7 @@ impl StolenConnections { /// Adds the given [`StolenConnection`] to this set. Spawns a new [`tokio::task`] that will /// manage it. - #[tracing::instrument(level = "trace", name = "manage_stolen_connection", skip(self))] + #[tracing::instrument(level = Level::TRACE, name = "manage_stolen_connection", skip(self))] pub fn manage(&mut self, connection: StolenConnection) { let connection_id = self.next_connection_id; self.next_connection_id += 1; @@ -458,13 +462,9 @@ impl ConnectionTask { }) .await?; - let task = UnfilteredStealTask { - connection_id: self.connection_id, - client_id, - stream: self.connection.stream, - }; - - task.run(self.tx, &mut self.rx).await + UnfilteredStealTask::new(self.connection_id, client_id, self.connection.stream) + .run(self.tx, &mut self.rx) + .await } PortSubscription::Filtered(filters) => { diff --git a/mirrord/agent/src/steal/connections/filtered.rs b/mirrord/agent/src/steal/connections/filtered.rs index b30e48a5757..ecc9f0064ad 100644 --- a/mirrord/agent/src/steal/connections/filtered.rs +++ b/mirrord/agent/src/steal/connections/filtered.rs @@ -1,5 +1,6 @@ use std::{ - collections::HashMap, future::Future, marker::PhantomData, net::SocketAddr, pin::Pin, sync::Arc, + collections::HashMap, future::Future, marker::PhantomData, net::SocketAddr, ops::Not, pin::Pin, + sync::Arc, }; use bytes::Bytes; @@ -28,9 +29,13 @@ use tokio::{ use tokio_util::sync::{CancellationToken, DropGuard}; use tracing::Level; -use super::{ConnectionMessageIn, ConnectionMessageOut, ConnectionTaskError}; +use super::{ + ConnectionMessageIn, ConnectionMessageOut, ConnectionTaskError, + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION, +}; use crate::{ http::HttpVersion, + metrics::STEAL_FILTERED_CONNECTION_SUBSCRIPTION, steal::{connections::unfiltered::UnfilteredStealTask, http::HttpFilter}, util::ClientId, }; @@ -368,6 +373,18 @@ pub struct FilteredStealTask { /// For safely downcasting the IO stream after an HTTP upgrade. See [`Upgraded::downcast`]. _io_type: PhantomData T>, + + /// Helps us figuring out if we should update some metrics in the `Drop` implementation. + metrics_updated: bool, +} + +impl Drop for FilteredStealTask { + fn drop(&mut self) { + if self.metrics_updated.not() { + STEAL_FILTERED_CONNECTION_SUBSCRIPTION + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } + } } impl FilteredStealTask @@ -443,6 +460,8 @@ where } }; + STEAL_FILTERED_CONNECTION_SUBSCRIPTION.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Self { connection_id, original_destination, @@ -453,6 +472,7 @@ where blocked_requests: Default::default(), next_request_id: Default::default(), _io_type: Default::default(), + metrics_updated: false, } } @@ -638,6 +658,8 @@ where queued_raw_data.remove(&client_id); self.subscribed.insert(client_id, false); self.blocked_requests.retain(|key, _| key.0 != client_id); + + STEAL_FILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); }, }, @@ -646,7 +668,10 @@ where // No more requests from the `FilteringService`. // HTTP connection is closed and possibly upgraded. - None => break, + None => { + STEAL_FILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + break + } } } } @@ -788,15 +813,18 @@ where ) -> Result<(), ConnectionTaskError> { let res = self.run_until_http_ends(tx.clone(), rx).await; + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + self.metrics_updated = true; + let res = match res { Ok(data) => self.run_after_http_ends(data, tx.clone(), rx).await, Err(e) => Err(e), }; - for (client_id, subscribed) in self.subscribed { - if subscribed { + for (client_id, subscribed) in self.subscribed.iter() { + if *subscribed { tx.send(ConnectionMessageOut::Closed { - client_id, + client_id: *client_id, connection_id: self.connection_id, }) .await?; diff --git a/mirrord/agent/src/steal/connections/unfiltered.rs b/mirrord/agent/src/steal/connections/unfiltered.rs index 5b6676094c3..ec54691315e 100644 --- a/mirrord/agent/src/steal/connections/unfiltered.rs +++ b/mirrord/agent/src/steal/connections/unfiltered.rs @@ -7,7 +7,10 @@ use tokio::{ sync::mpsc::{Receiver, Sender}, }; -use super::{ConnectionMessageIn, ConnectionMessageOut, ConnectionTaskError}; +use super::{ + ConnectionMessageIn, ConnectionMessageOut, ConnectionTaskError, + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION, +}; use crate::util::ClientId; /// Manages an unfiltered stolen connection. @@ -19,7 +22,23 @@ pub struct UnfilteredStealTask { pub stream: T, } +impl Drop for UnfilteredStealTask { + fn drop(&mut self) { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } +} + impl UnfilteredStealTask { + pub(crate) fn new(connection_id: ConnectionId, client_id: ClientId, stream: T) -> Self { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + Self { + connection_id, + client_id, + stream, + } + } + /// Runs this task until the managed connection is closed. /// /// # Note @@ -40,6 +59,8 @@ impl UnfilteredStealTask { read = self.stream.read_buf(&mut buf), if !reading_closed => match read { Ok(..) => { if buf.is_empty() { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + tracing::trace!( client_id = self.client_id, connection_id = self.connection_id, @@ -63,6 +84,8 @@ impl UnfilteredStealTask { Err(e) if e.kind() == ErrorKind::WouldBlock => {} Err(e) => { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + tx.send(ConnectionMessageOut::Closed { client_id: self.client_id, connection_id: self.connection_id @@ -85,6 +108,8 @@ impl UnfilteredStealTask { ConnectionMessageIn::Raw { data, .. } => { let res = if data.is_empty() { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + tracing::trace!( client_id = self.client_id, connection_id = self.connection_id, @@ -97,6 +122,8 @@ impl UnfilteredStealTask { }; if let Err(e) = res { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + tx.send(ConnectionMessageOut::Closed { client_id: self.client_id, connection_id: self.connection_id @@ -115,6 +142,8 @@ impl UnfilteredStealTask { }, ConnectionMessageIn::Unsubscribed { .. } => { + STEAL_UNFILTERED_CONNECTION_SUBSCRIPTION.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + return Ok(()); } } diff --git a/mirrord/agent/src/steal/ip_tables.rs b/mirrord/agent/src/steal/ip_tables.rs index 68bddb6a406..9c175220767 100644 --- a/mirrord/agent/src/steal/ip_tables.rs +++ b/mirrord/agent/src/steal/ip_tables.rs @@ -9,7 +9,7 @@ use rand::distributions::{Alphanumeric, DistString}; use tracing::warn; use crate::{ - error::{AgentError, Result}, + error::{AgentError, AgentResult}, steal::ip_tables::{ flush_connections::FlushConnections, mesh::{istio::AmbientRedirect, MeshRedirect, MeshVendorExt}, @@ -84,13 +84,13 @@ pub(crate) trait IPTables { where Self: Sized; - fn create_chain(&self, name: &str) -> Result<()>; - fn remove_chain(&self, name: &str) -> Result<()>; + fn create_chain(&self, name: &str) -> AgentResult<()>; + fn remove_chain(&self, name: &str) -> AgentResult<()>; - fn add_rule(&self, chain: &str, rule: &str) -> Result<()>; - fn insert_rule(&self, chain: &str, rule: &str, index: i32) -> Result<()>; - fn list_rules(&self, chain: &str) -> Result>; - fn remove_rule(&self, chain: &str, rule: &str) -> Result<()>; + fn add_rule(&self, chain: &str, rule: &str) -> AgentResult<()>; + fn insert_rule(&self, chain: &str, rule: &str, index: i32) -> AgentResult<()>; + fn list_rules(&self, chain: &str) -> AgentResult>; + fn remove_rule(&self, chain: &str, rule: &str) -> AgentResult<()>; } #[derive(Clone)] @@ -152,8 +152,13 @@ impl IPTables for IPTablesWrapper { } } - #[tracing::instrument(level = tracing::Level::TRACE, skip(self), ret, fields(table_name=%self.table_name))] - fn create_chain(&self, name: &str) -> Result<()> { + #[tracing::instrument( + level = tracing::Level::TRACE, + skip(self), + ret, + fields(table_name=%self.table_name + ))] + fn create_chain(&self, name: &str) -> AgentResult<()> { self.tables .new_chain(self.table_name, name) .map_err(|e| AgentError::IPTablesError(e.to_string()))?; @@ -165,7 +170,7 @@ impl IPTables for IPTablesWrapper { } #[tracing::instrument(level = "trace")] - fn remove_chain(&self, name: &str) -> Result<()> { + fn remove_chain(&self, name: &str) -> AgentResult<()> { self.tables .flush_chain(self.table_name, name) .map_err(|e| AgentError::IPTablesError(e.to_string()))?; @@ -177,28 +182,28 @@ impl IPTables for IPTablesWrapper { } #[tracing::instrument(level = "trace", ret)] - fn add_rule(&self, chain: &str, rule: &str) -> Result<()> { + fn add_rule(&self, chain: &str, rule: &str) -> AgentResult<()> { self.tables .append(self.table_name, chain, rule) .map_err(|e| AgentError::IPTablesError(e.to_string())) } #[tracing::instrument(level = "trace", ret)] - fn insert_rule(&self, chain: &str, rule: &str, index: i32) -> Result<()> { + fn insert_rule(&self, chain: &str, rule: &str, index: i32) -> AgentResult<()> { self.tables .insert(self.table_name, chain, rule, index) .map_err(|e| AgentError::IPTablesError(e.to_string())) } #[tracing::instrument(level = "trace")] - fn list_rules(&self, chain: &str) -> Result> { + fn list_rules(&self, chain: &str) -> AgentResult> { self.tables .list(self.table_name, chain) .map_err(|e| AgentError::IPTablesError(e.to_string())) } #[tracing::instrument(level = "trace")] - fn remove_rule(&self, chain: &str, rule: &str) -> Result<()> { + fn remove_rule(&self, chain: &str, rule: &str) -> AgentResult<()> { self.tables .delete(self.table_name, chain, rule) .map_err(|e| AgentError::IPTablesError(e.to_string())) @@ -233,7 +238,7 @@ where flush_connections: bool, pod_ips: Option<&str>, ipv6: bool, - ) -> Result { + ) -> AgentResult { let ipt = Arc::new(ipt); let mut redirect = if let Some(vendor) = MeshVendor::detect(ipt.as_ref())? { @@ -265,7 +270,7 @@ where Ok(Self { redirect }) } - pub(crate) async fn load(ipt: IPT, flush_connections: bool) -> Result { + pub(crate) async fn load(ipt: IPT, flush_connections: bool) -> AgentResult { let ipt = Arc::new(ipt); let mut redirect = if let Some(vendor) = MeshVendor::detect(ipt.as_ref())? { @@ -299,7 +304,7 @@ where &self, redirected_port: Port, target_port: Port, - ) -> Result<()> { + ) -> AgentResult<()> { self.redirect .add_redirect(redirected_port, target_port) .await @@ -314,13 +319,13 @@ where &self, redirected_port: Port, target_port: Port, - ) -> Result<()> { + ) -> AgentResult<()> { self.redirect .remove_redirect(redirected_port, target_port) .await } - pub(crate) async fn cleanup(&self) -> Result<()> { + pub(crate) async fn cleanup(&self) -> AgentResult<()> { self.redirect.unmount_entrypoint().await } } diff --git a/mirrord/agent/src/steal/ip_tables/chain.rs b/mirrord/agent/src/steal/ip_tables/chain.rs index c5bc6d65404..c1c34715c85 100644 --- a/mirrord/agent/src/steal/ip_tables/chain.rs +++ b/mirrord/agent/src/steal/ip_tables/chain.rs @@ -4,7 +4,7 @@ use std::sync::{ }; use crate::{ - error::{AgentError, Result}, + error::{AgentError, AgentResult}, steal::ip_tables::IPTables, }; @@ -19,7 +19,7 @@ impl IPTableChain where IPT: IPTables, { - pub fn create(inner: Arc, chain_name: String) -> Result { + pub fn create(inner: Arc, chain_name: String) -> AgentResult { inner.create_chain(&chain_name)?; // Start with 1 because the chain will allways have atleast `-A ` as a rule @@ -32,7 +32,7 @@ where }) } - pub fn load(inner: Arc, chain_name: String) -> Result { + pub fn load(inner: Arc, chain_name: String) -> AgentResult { let existing_rules = inner.list_rules(&chain_name)?.len(); if existing_rules == 0 { @@ -59,7 +59,7 @@ where &self.inner } - pub fn add_rule(&self, rule: &str) -> Result { + pub fn add_rule(&self, rule: &str) -> AgentResult { self.inner .insert_rule( &self.chain_name, @@ -72,7 +72,7 @@ where }) } - pub fn remove_rule(&self, rule: &str) -> Result<()> { + pub fn remove_rule(&self, rule: &str) -> AgentResult<()> { self.inner.remove_rule(&self.chain_name, rule)?; self.chain_size.fetch_sub(1, Ordering::Relaxed); diff --git a/mirrord/agent/src/steal/ip_tables/flush_connections.rs b/mirrord/agent/src/steal/ip_tables/flush_connections.rs index 6675a40651f..c0f19c20b8d 100644 --- a/mirrord/agent/src/steal/ip_tables/flush_connections.rs +++ b/mirrord/agent/src/steal/ip_tables/flush_connections.rs @@ -13,7 +13,7 @@ use tokio::process::Command; use tracing::warn; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{chain::IPTableChain, redirect::Redirect, IPTables, IPTABLE_INPUT}, }; @@ -33,7 +33,7 @@ where const ENTRYPOINT: &'static str = "INPUT"; #[tracing::instrument(level = "trace", skip(ipt, inner))] - pub fn create(ipt: Arc, inner: Box) -> Result { + pub fn create(ipt: Arc, inner: Box) -> AgentResult { let managed = IPTableChain::create(ipt.with_table("filter").into(), IPTABLE_INPUT.to_string())?; @@ -48,7 +48,7 @@ where } #[tracing::instrument(level = "trace", skip(ipt, inner))] - pub fn load(ipt: Arc, inner: Box) -> Result { + pub fn load(ipt: Arc, inner: Box) -> AgentResult { let managed = IPTableChain::load(ipt.with_table("filter").into(), IPTABLE_INPUT.to_string())?; @@ -63,7 +63,7 @@ where T: Redirect + Send + Sync, { #[tracing::instrument(level = "trace", skip(self), ret)] - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { self.inner.mount_entrypoint().await?; self.managed.inner().add_rule( @@ -75,7 +75,7 @@ where } #[tracing::instrument(level = "trace", skip(self), ret)] - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.inner.unmount_entrypoint().await?; self.managed.inner().remove_rule( @@ -87,7 +87,7 @@ where } #[tracing::instrument(level = "trace", skip(self), ret)] - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.inner .add_redirect(redirected_port, target_port) .await?; @@ -115,7 +115,7 @@ where } #[tracing::instrument(level = "trace", skip(self), ret)] - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.inner .remove_redirect(redirected_port, target_port) .await?; diff --git a/mirrord/agent/src/steal/ip_tables/mesh.rs b/mirrord/agent/src/steal/ip_tables/mesh.rs index 88fdff5d0b1..1a3e5acbe62 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh.rs @@ -5,7 +5,7 @@ use fancy_regex::Regex; use mirrord_protocol::{MeshVendor, Port}; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{ output::OutputRedirect, prerouting::PreroutingRedirect, redirect::Redirect, IPTables, IPTABLE_MESH, @@ -29,7 +29,7 @@ impl MeshRedirect where IPT: IPTables, { - pub fn create(ipt: Arc, vendor: MeshVendor, pod_ips: Option<&str>) -> Result { + pub fn create(ipt: Arc, vendor: MeshVendor, pod_ips: Option<&str>) -> AgentResult { let prerouting = PreroutingRedirect::create(ipt.clone())?; for port in Self::get_skip_ports(&ipt, &vendor)? { @@ -45,7 +45,7 @@ where }) } - pub fn load(ipt: Arc, vendor: MeshVendor) -> Result { + pub fn load(ipt: Arc, vendor: MeshVendor) -> AgentResult { let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; @@ -56,7 +56,7 @@ where }) } - fn get_skip_ports(ipt: &IPT, vendor: &MeshVendor) -> Result> { + fn get_skip_ports(ipt: &IPT, vendor: &MeshVendor) -> AgentResult> { let chain_name = vendor.input_chain(); let lookup_regex = if let Some(regex) = vendor.skip_ports_regex() { regex @@ -86,21 +86,21 @@ impl Redirect for MeshRedirect where IPT: IPTables + Send + Sync, { - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { self.prerouting.mount_entrypoint().await?; self.output.mount_entrypoint().await?; Ok(()) } - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.prerouting.unmount_entrypoint().await?; self.output.unmount_entrypoint().await?; Ok(()) } - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { if self.vendor != MeshVendor::IstioCni { self.prerouting .add_redirect(redirected_port, target_port) @@ -113,7 +113,7 @@ where Ok(()) } - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { if self.vendor != MeshVendor::IstioCni { self.prerouting .remove_redirect(redirected_port, target_port) @@ -129,13 +129,13 @@ where /// Extends the [`MeshVendor`] type with methods that are only relevant for the agent. pub(super) trait MeshVendorExt: Sized { - fn detect(ipt: &IPT) -> Result>; + fn detect(ipt: &IPT) -> AgentResult>; fn input_chain(&self) -> &str; fn skip_ports_regex(&self) -> Option<&Regex>; } impl MeshVendorExt for MeshVendor { - fn detect(ipt: &IPT) -> Result> { + fn detect(ipt: &IPT) -> AgentResult> { if let Ok(val) = std::env::var("MIRRORD_AGENT_ISTIO_CNI") && val.to_lowercase() == "true" { diff --git a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs index cd3d4b06fa9..01e513a6bf9 100644 --- a/mirrord/agent/src/steal/ip_tables/mesh/istio.rs +++ b/mirrord/agent/src/steal/ip_tables/mesh/istio.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use mirrord_protocol::Port; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{ output::OutputRedirect, prerouting::PreroutingRedirect, redirect::Redirect, IPTables, IPTABLE_IPV4_ROUTE_LOCALNET_ORIGINAL, IPTABLE_MESH, @@ -20,14 +20,14 @@ impl AmbientRedirect where IPT: IPTables, { - pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { + pub fn create(ipt: Arc, pod_ips: Option<&str>) -> AgentResult { let prerouting = PreroutingRedirect::create(ipt.clone())?; let output = OutputRedirect::create(ipt, IPTABLE_MESH.to_string(), pod_ips)?; Ok(AmbientRedirect { prerouting, output }) } - pub fn load(ipt: Arc) -> Result { + pub fn load(ipt: Arc) -> AgentResult { let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_MESH.to_string())?; @@ -40,7 +40,7 @@ impl Redirect for AmbientRedirect where IPT: IPTables + Send + Sync, { - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { tokio::fs::write("/proc/sys/net/ipv4/conf/all/route_localnet", "1".as_bytes()).await?; self.prerouting.mount_entrypoint().await?; @@ -49,7 +49,7 @@ where Ok(()) } - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.prerouting.unmount_entrypoint().await?; self.output.unmount_entrypoint().await?; @@ -62,7 +62,7 @@ where Ok(()) } - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.prerouting .add_redirect(redirected_port, target_port) .await?; @@ -73,7 +73,7 @@ where Ok(()) } - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.prerouting .remove_redirect(redirected_port, target_port) .await?; diff --git a/mirrord/agent/src/steal/ip_tables/output.rs b/mirrord/agent/src/steal/ip_tables/output.rs index 2286469c00c..9eebad0c9ae 100644 --- a/mirrord/agent/src/steal/ip_tables/output.rs +++ b/mirrord/agent/src/steal/ip_tables/output.rs @@ -6,7 +6,7 @@ use nix::unistd::getgid; use tracing::warn; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{chain::IPTableChain, IPTables, Redirect}, }; @@ -20,8 +20,8 @@ where { const ENTRYPOINT: &'static str = "OUTPUT"; - #[tracing::instrument(skip(ipt), level = tracing::Level::TRACE)] - pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> Result { + #[tracing::instrument(level = tracing::Level::TRACE, skip(ipt), err)] + pub fn create(ipt: Arc, chain_name: String, pod_ips: Option<&str>) -> AgentResult { let managed = IPTableChain::create(ipt, chain_name.clone()).inspect_err( |e| tracing::error!(%e, "Could not create iptables chain \"{chain_name}\"."), )?; @@ -42,7 +42,7 @@ where Ok(OutputRedirect { managed }) } - pub fn load(ipt: Arc, chain_name: String) -> Result { + pub fn load(ipt: Arc, chain_name: String) -> AgentResult { let managed = IPTableChain::load(ipt, chain_name)?; Ok(OutputRedirect { managed }) @@ -56,7 +56,7 @@ impl Redirect for OutputRedirect where IPT: IPTables + Send + Sync, { - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { if USE_INSERT { self.managed.inner().insert_rule( Self::ENTRYPOINT, @@ -73,7 +73,7 @@ where Ok(()) } - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.managed.inner().remove_rule( Self::ENTRYPOINT, &format!("-j {}", self.managed.chain_name()), @@ -82,7 +82,7 @@ where Ok(()) } - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { let redirect_rule = format!( "-o lo -m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}" ); @@ -92,7 +92,7 @@ where Ok(()) } - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { let redirect_rule = format!( "-o lo -m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}" ); diff --git a/mirrord/agent/src/steal/ip_tables/prerouting.rs b/mirrord/agent/src/steal/ip_tables/prerouting.rs index 486b0ca1b51..29d5de06103 100644 --- a/mirrord/agent/src/steal/ip_tables/prerouting.rs +++ b/mirrord/agent/src/steal/ip_tables/prerouting.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use mirrord_protocol::Port; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{chain::IPTableChain, IPTables, Redirect, IPTABLE_PREROUTING}, }; @@ -18,13 +18,13 @@ where { const ENTRYPOINT: &'static str = "PREROUTING"; - pub fn create(ipt: Arc) -> Result { + pub fn create(ipt: Arc) -> AgentResult { let managed = IPTableChain::create(ipt, IPTABLE_PREROUTING.to_string())?; Ok(PreroutingRedirect { managed }) } - pub fn load(ipt: Arc) -> Result { + pub fn load(ipt: Arc) -> AgentResult { let managed = IPTableChain::load(ipt, IPTABLE_PREROUTING.to_string())?; Ok(PreroutingRedirect { managed }) @@ -36,7 +36,7 @@ impl Redirect for PreroutingRedirect where IPT: IPTables + Send + Sync, { - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { self.managed.inner().add_rule( Self::ENTRYPOINT, &format!("-j {}", self.managed.chain_name()), @@ -45,7 +45,7 @@ where Ok(()) } - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.managed.inner().remove_rule( Self::ENTRYPOINT, &format!("-j {}", self.managed.chain_name()), @@ -54,7 +54,7 @@ where Ok(()) } - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { let redirect_rule = format!("-m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}"); @@ -63,7 +63,7 @@ where Ok(()) } - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { let redirect_rule = format!("-m tcp -p tcp --dport {redirected_port} -j REDIRECT --to-ports {target_port}"); diff --git a/mirrord/agent/src/steal/ip_tables/redirect.rs b/mirrord/agent/src/steal/ip_tables/redirect.rs index d18aeb1d7ea..fe52d90fc1e 100644 --- a/mirrord/agent/src/steal/ip_tables/redirect.rs +++ b/mirrord/agent/src/steal/ip_tables/redirect.rs @@ -2,17 +2,17 @@ use async_trait::async_trait; use enum_dispatch::enum_dispatch; use mirrord_protocol::Port; -use crate::error::Result; +use crate::error::AgentResult; #[async_trait] #[enum_dispatch] pub(crate) trait Redirect { - async fn mount_entrypoint(&self) -> Result<()>; + async fn mount_entrypoint(&self) -> AgentResult<()>; - async fn unmount_entrypoint(&self) -> Result<()>; + async fn unmount_entrypoint(&self) -> AgentResult<()>; /// Create port redirection - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()>; + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()>; /// Remove port redirection - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()>; + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()>; } diff --git a/mirrord/agent/src/steal/ip_tables/standard.rs b/mirrord/agent/src/steal/ip_tables/standard.rs index 3302b05c02e..47b9bf0c0af 100644 --- a/mirrord/agent/src/steal/ip_tables/standard.rs +++ b/mirrord/agent/src/steal/ip_tables/standard.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use mirrord_protocol::Port; use crate::{ - error::Result, + error::AgentResult, steal::ip_tables::{ output::OutputRedirect, prerouting::PreroutingRedirect, IPTables, Redirect, IPTABLE_STANDARD, @@ -20,14 +20,14 @@ impl StandardRedirect where IPT: IPTables, { - pub fn create(ipt: Arc, pod_ips: Option<&str>) -> Result { + pub fn create(ipt: Arc, pod_ips: Option<&str>) -> AgentResult { let prerouting = PreroutingRedirect::create(ipt.clone())?; let output = OutputRedirect::create(ipt, IPTABLE_STANDARD.to_string(), pod_ips)?; Ok(StandardRedirect { prerouting, output }) } - pub fn load(ipt: Arc) -> Result { + pub fn load(ipt: Arc) -> AgentResult { let prerouting = PreroutingRedirect::load(ipt.clone())?; let output = OutputRedirect::load(ipt, IPTABLE_STANDARD.to_string())?; @@ -42,21 +42,21 @@ impl Redirect for StandardRedirect where IPT: IPTables + Send + Sync, { - async fn mount_entrypoint(&self) -> Result<()> { + async fn mount_entrypoint(&self) -> AgentResult<()> { self.prerouting.mount_entrypoint().await?; self.output.mount_entrypoint().await?; Ok(()) } - async fn unmount_entrypoint(&self) -> Result<()> { + async fn unmount_entrypoint(&self) -> AgentResult<()> { self.prerouting.unmount_entrypoint().await?; self.output.unmount_entrypoint().await?; Ok(()) } - async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn add_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.prerouting .add_redirect(redirected_port, target_port) .await?; @@ -67,7 +67,7 @@ where Ok(()) } - async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> Result<()> { + async fn remove_redirect(&self, redirected_port: Port, target_port: Port) -> AgentResult<()> { self.prerouting .remove_redirect(redirected_port, target_port) .await?; diff --git a/mirrord/agent/src/steal/subscriptions.rs b/mirrord/agent/src/steal/subscriptions.rs index 0468719bc9c..901ecd725ef 100644 --- a/mirrord/agent/src/steal/subscriptions.rs +++ b/mirrord/agent/src/steal/subscriptions.rs @@ -16,7 +16,11 @@ use super::{ http::HttpFilter, ip_tables::{new_ip6tables, new_iptables, IPTablesWrapper, SafeIpTables}, }; -use crate::{error::AgentError, util::ClientId}; +use crate::{ + error::{AgentError, AgentResult}, + metrics::{STEAL_FILTERED_PORT_SUBSCRIPTION, STEAL_UNFILTERED_PORT_SUBSCRIPTION}, + util::ClientId, +}; /// For stealing incoming TCP connections. #[async_trait::async_trait] @@ -149,7 +153,7 @@ impl IpTablesRedirector { flush_connections: bool, pod_ips: Option, support_ipv6: bool, - ) -> Result { + ) -> AgentResult { let (pod_ips4, pod_ips6) = pod_ips.map_or_else( || (None, None), |ips| { @@ -310,6 +314,13 @@ pub struct PortSubscriptions { subscriptions: HashMap, } +impl Drop for PortSubscriptions { + fn drop(&mut self) { + STEAL_FILTERED_PORT_SUBSCRIPTION.store(0, std::sync::atomic::Ordering::Relaxed); + STEAL_UNFILTERED_PORT_SUBSCRIPTION.store(0, std::sync::atomic::Ordering::Relaxed); + } +} + impl PortSubscriptions { /// Create an empty instance of this struct. /// @@ -351,7 +362,16 @@ impl PortSubscriptions { ) -> Result, R::Error> { let add_redirect = match self.subscriptions.entry(port) { Entry::Occupied(mut e) => { + let filtered = filter.is_some(); if e.get_mut().try_extend(client_id, filter) { + if filtered { + STEAL_FILTERED_PORT_SUBSCRIPTION + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + STEAL_UNFILTERED_PORT_SUBSCRIPTION + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + Ok(false) } else { Err(ResponseError::PortAlreadyStolen(port)) @@ -359,6 +379,14 @@ impl PortSubscriptions { } Entry::Vacant(e) => { + if filter.is_some() { + STEAL_FILTERED_PORT_SUBSCRIPTION + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } else { + STEAL_UNFILTERED_PORT_SUBSCRIPTION + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + e.insert(PortSubscription::new(client_id, filter)); Ok(true) } @@ -395,11 +423,17 @@ impl PortSubscriptions { let remove_redirect = match e.get_mut() { PortSubscription::Unfiltered(subscribed_client) if *subscribed_client == client_id => { e.remove(); + STEAL_UNFILTERED_PORT_SUBSCRIPTION + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + true } PortSubscription::Unfiltered(..) => false, PortSubscription::Filtered(filters) => { - filters.remove(&client_id); + if filters.remove(&client_id).is_some() { + STEAL_FILTERED_PORT_SUBSCRIPTION + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } if filters.is_empty() { e.remove(); diff --git a/mirrord/agent/src/util.rs b/mirrord/agent/src/util.rs index 9dcbc6cd892..0c72cddd82f 100644 --- a/mirrord/agent/src/util.rs +++ b/mirrord/agent/src/util.rs @@ -12,7 +12,7 @@ use tokio::sync::mpsc; use tracing::error; use crate::{ - error::AgentError, + error::AgentResult, namespace::{set_namespace, NamespaceType}, }; @@ -151,7 +151,7 @@ where /// Many of the agent's TCP/UDP connections require that they're made from the `pid`'s namespace to /// work. #[tracing::instrument(level = "trace")] -pub(crate) fn enter_namespace(pid: Option, namespace: &str) -> Result<(), AgentError> { +pub(crate) fn enter_namespace(pid: Option, namespace: &str) -> AgentResult<()> { if let Some(pid) = pid { Ok(set_namespace(pid, NamespaceType::Net).inspect_err(|fail| { error!("Failed setting pid {pid:#?} namespace {namespace:#?} with {fail:#?}") diff --git a/mirrord/agent/src/vpn.rs b/mirrord/agent/src/vpn.rs index dd8c3a5133f..d7d30d5ca6f 100644 --- a/mirrord/agent/src/vpn.rs +++ b/mirrord/agent/src/vpn.rs @@ -17,7 +17,7 @@ use tokio::{ }; use crate::{ - error::{AgentError, Result}, + error::{AgentError, AgentResult}, util::run_thread_in_namespace, watched_task::{TaskStatus, WatchedTask}, }; @@ -75,7 +75,7 @@ impl VpnApi { /// Sends the [`ClientVpn`] message to the background task. #[tracing::instrument(level = "trace", skip(self))] - pub(crate) async fn layer_message(&mut self, message: ClientVpn) -> Result<()> { + pub(crate) async fn layer_message(&mut self, message: ClientVpn) -> AgentResult<()> { if self.layer_tx.send(message).await.is_ok() { Ok(()) } else { @@ -84,7 +84,7 @@ impl VpnApi { } /// Receives a [`ServerVpn`] message from the background task. - pub(crate) async fn daemon_message(&mut self) -> Result { + pub(crate) async fn daemon_message(&mut self) -> AgentResult { match self.daemon_rx.recv().await { Some(msg) => Ok(msg), None => Err(self.task_status.unwrap_err().await), @@ -121,7 +121,7 @@ impl AsyncRawSocket { } } -async fn create_raw_socket() -> Result { +async fn create_raw_socket() -> AgentResult { let index = nix::net::if_::if_nametoindex("eth0") .map_err(|err| AgentError::VpnError(err.to_string()))?; @@ -139,7 +139,7 @@ async fn create_raw_socket() -> Result { } #[tracing::instrument(level = "debug", ret)] -async fn resolve_interface() -> Result<(IpAddr, IpAddr, IpAddr)> { +async fn resolve_interface() -> AgentResult<(IpAddr, IpAddr, IpAddr)> { // Connect to a remote address so we can later get the default network interface. let temporary_socket = UdpSocket::bind("0.0.0.0:0").await?; temporary_socket.connect("8.8.8.8:53").await?; @@ -209,7 +209,7 @@ impl fmt::Debug for VpnTask { } } -fn interface_index_to_sock_addr(index: i32) -> Result { +fn interface_index_to_sock_addr(index: i32) -> AgentResult { let mut addr_storage: libc::sockaddr_storage = unsafe { std::mem::zeroed() }; let len = std::mem::size_of::() as libc::socklen_t; let macs = procfs::net::arp().map_err(|err| AgentError::VpnError(err.to_string()))?; @@ -245,7 +245,7 @@ impl VpnTask { } #[allow(clippy::indexing_slicing)] - async fn run(mut self) -> Result<()> { + async fn run(mut self) -> AgentResult<()> { // so host won't respond with RST to our packets. // TODO: need to do it for UDP as well to avoid ICMP unreachable. let output = std::process::Command::new("iptables") @@ -318,7 +318,7 @@ impl VpnTask { &mut self, message: ClientVpn, network_configuration: &NetworkConfiguration, - ) -> Result<()> { + ) -> AgentResult<()> { match message { // We make connection to the requested address, split the stream into halves with // `io::split`, and put them into respective maps. diff --git a/mirrord/agent/src/watched_task.rs b/mirrord/agent/src/watched_task.rs index 0212f279163..ad06bb238ee 100644 --- a/mirrord/agent/src/watched_task.rs +++ b/mirrord/agent/src/watched_task.rs @@ -2,7 +2,7 @@ use std::future::Future; use tokio::sync::watch::{self, Receiver, Sender}; -use crate::error::AgentError; +use crate::error::{AgentError, AgentResult}; /// A shared clonable view on a background task's status. #[derive(Debug, Clone)] @@ -83,7 +83,7 @@ impl WatchedTask { impl WatchedTask where - T: Future>, + T: Future>, { /// Execute the wrapped task. /// Store its result in the inner [`TaskStatus`]. diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md index 8e8b9ea6aee..0d1af19401d 100644 --- a/mirrord/config/configuration.md +++ b/mirrord/config/configuration.md @@ -68,7 +68,8 @@ configuration file containing all fields. "communication_timeout": 30, "startup_timeout": 360, "network_interface": "eth0", - "flush_connections": true + "flush_connections": true, + "metrics": "0.0.0.0:9000", }, "feature": { "env": { @@ -166,7 +167,11 @@ Allows setting up custom annotations for the agent Job and Pod. ```json { - "annotations": { "cats.io/inject": "enabled" } + "annotations": { + "cats.io/inject": "enabled" + "prometheus.io/scrape": "true", + "prometheus.io/port": "9000" + } } ``` @@ -299,6 +304,19 @@ with `RUST_LOG`. } ``` +### agent.metrics {#agent-metrics} + +Enables prometheus metrics for the agent pod. + +You might need to add annotations to the agent pod depending on how prometheus is +configured to scrape for metrics. + +```json +{ + "metrics": "0.0.0.0:9000" +} +``` + ### agent.namespace {#agent-namespace} Namespace where the agent shall live. diff --git a/mirrord/config/src/agent.rs b/mirrord/config/src/agent.rs index 3dff5adfefc..9600edfcd4d 100644 --- a/mirrord/config/src/agent.rs +++ b/mirrord/config/src/agent.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt, path::Path}; +use std::{collections::HashMap, fmt, net::SocketAddr, path::Path}; use k8s_openapi::api::core::v1::{ResourceRequirements, Toleration}; use mirrord_analytics::CollectAnalytics; @@ -322,7 +322,11 @@ pub struct AgentConfig { /// /// ```json /// { - /// "annotations": { "cats.io/inject": "enabled" } + /// "annotations": { + /// "cats.io/inject": "enabled" + /// "prometheus.io/scrape": "true", + /// "prometheus.io/port": "9000" + /// } /// } /// ``` pub annotations: Option>, @@ -350,6 +354,20 @@ pub struct AgentConfig { /// ``` pub service_account: Option, + /// ### agent.metrics {#agent-metrics} + /// + /// Enables prometheus metrics for the agent pod. + /// + /// You might need to add annotations to the agent pod depending on how prometheus is + /// configured to scrape for metrics. + /// + /// ```json + /// { + /// "metrics": "0.0.0.0:9000" + /// } + /// ``` + pub metrics: Option, + /// /// Create an agent that returns an error after accepting the first client. For testing /// purposes. Only supported with job agents (not with ephemeral agents). diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs index d3f8fae7bc6..7ca3e00dc39 100644 --- a/mirrord/config/src/lib.rs +++ b/mirrord/config/src/lib.rs @@ -119,7 +119,8 @@ pub static MIRRORD_RESOLVED_CONFIG_ENV: &str = "MIRRORD_RESOLVED_CONFIG"; /// "communication_timeout": 30, /// "startup_timeout": 360, /// "network_interface": "eth0", -/// "flush_connections": true +/// "flush_connections": true, +/// "metrics": "0.0.0.0:9000", /// }, /// "feature": { /// "env": { diff --git a/mirrord/kube/src/api/container/util.rs b/mirrord/kube/src/api/container/util.rs index 23fd752181b..d40949d268c 100644 --- a/mirrord/kube/src/api/container/util.rs +++ b/mirrord/kube/src/api/container/util.rs @@ -4,7 +4,9 @@ use futures::{AsyncBufReadExt, TryStreamExt}; use k8s_openapi::api::core::v1::{EnvVar, Pod, Toleration}; use kube::{api::LogParams, Api}; use mirrord_config::agent::{AgentConfig, LinuxCapability}; -use mirrord_protocol::{AGENT_IPV6_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV}; +use mirrord_protocol::{ + AGENT_IPV6_ENV, AGENT_METRICS_ENV, AGENT_NETWORK_INTERFACE_ENV, AGENT_OPERATOR_CERT_ENV, +}; use regex::Regex; use tracing::warn; @@ -59,7 +61,9 @@ pub(super) fn agent_env(agent: &AgentConfig, params: &&ContainerParams) -> Vec Vec = #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum ClientMessage { Close, + /// TCP sniffer message. + /// + /// These are the messages used by the `mirror` feature, and handled by the + /// `TcpSnifferApi` in the agent. Tcp(LayerTcp), + + /// TCP stealer message. + /// + /// These are the messages used by the `steal` feature, and handled by the `TcpStealerApi` in + /// the agent. TcpSteal(LayerTcpSteal), + /// TCP outgoing message. + /// + /// These are the messages used by the `outgoing` feature (tcp), and handled by the + /// `TcpOutgoingApi` in the agent. TcpOutgoing(LayerTcpOutgoing), + + /// UDP outgoing message. + /// + /// These are the messages used by the `outgoing` feature (udp), and handled by the + /// `UdpOutgoingApi` in the agent. UdpOutgoing(LayerUdpOutgoing), FileRequest(FileRequest), GetEnvVarsRequest(GetEnvVarsRequest), diff --git a/mirrord/protocol/src/error.rs b/mirrord/protocol/src/error.rs index efb7ff08198..67197d76843 100644 --- a/mirrord/protocol/src/error.rs +++ b/mirrord/protocol/src/error.rs @@ -44,7 +44,7 @@ pub enum ResponseError { #[error("Remote operation expected fd `{0}` to be a file, but it's a directory!")] NotFile(u64), - #[error("IO failed for remote operation with `{0}!")] + #[error("IO failed for remote operation: `{0}!")] RemoteIO(RemoteIOError), #[error(transparent)] diff --git a/mirrord/protocol/src/lib.rs b/mirrord/protocol/src/lib.rs index 983fcd3536b..f1a3cc1e5cc 100644 --- a/mirrord/protocol/src/lib.rs +++ b/mirrord/protocol/src/lib.rs @@ -112,4 +112,6 @@ pub const AGENT_OPERATOR_CERT_ENV: &str = "MIRRORD_AGENT_OPERATOR_CERT"; pub const AGENT_NETWORK_INTERFACE_ENV: &str = "MIRRORD_AGENT_INTERFACE"; +pub const AGENT_METRICS_ENV: &str = "MIRRORD_AGENT_METRICS"; + pub const AGENT_IPV6_ENV: &str = "MIRRORD_AGENT_SUPPORT_IPV6"; diff --git a/mirrord/protocol/src/outgoing/tcp.rs b/mirrord/protocol/src/outgoing/tcp.rs index e38fa0c44d0..877e0d2f6c0 100644 --- a/mirrord/protocol/src/outgoing/tcp.rs +++ b/mirrord/protocol/src/outgoing/tcp.rs @@ -3,14 +3,43 @@ use crate::RemoteResult; #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum LayerTcpOutgoing { + /// User is interested in connecting via tcp to some remote address, specified in + /// [`LayerConnect`]. + /// + /// The layer will get a mirrord managed address that it'll `connect` to, meanwhile + /// in the agent we `connect` to the actual remote address. Connect(LayerConnect), + + /// Write data to the remote address the agent is `connect`ed to. + /// + /// There's no `Read` message, as we're calling `read` in the agent, and we send + /// a [`DaemonTcpOutgoing::Read`] message in case we get some data from this connection. Write(LayerWrite), + + /// The layer closed the connection, this message syncs up the agent, closing it + /// over there as well. + /// + /// Connections in the agent may be closed in other ways, such as when an error happens + /// when reading or writing. Which means that this message is not the only way of + /// closing outgoing tcp connections. Close(LayerClose), } #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum DaemonTcpOutgoing { + /// The agent attempted a connection to the remote address specified by + /// [`LayerTcpOutgoing::Connect`], and it might've been successful or not. Connect(RemoteResult), + + /// Read data from the connection. + /// + /// There's no `Write` message, as `write`s come from the user (layer). The agent sending + /// a `write` to the layer like this would make no sense, since it could just `write` it + /// to the remote connection itself. Read(RemoteResult), + + /// Tell the layer that this connection has been `close`d, either by a request from + /// the user with [`LayerTcpOutgoing::Close`], or from some error in the agent when + /// writing or reading from the connection. Close(ConnectionId), } diff --git a/mirrord/protocol/src/outgoing/udp.rs b/mirrord/protocol/src/outgoing/udp.rs index 02b4d97f830..f58378beeea 100644 --- a/mirrord/protocol/src/outgoing/udp.rs +++ b/mirrord/protocol/src/outgoing/udp.rs @@ -3,14 +3,50 @@ use crate::RemoteResult; #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum LayerUdpOutgoing { + /// User is interested in connecting via udp to some remote address, specified in + /// [`LayerConnect`]. + /// + /// The layer will get a mirrord managed address that it'll `connect` to, meanwhile + /// in the agent we `connect` to the actual remote address. + /// + /// Saying that we have an _udp connection_ is a bit weird, considering it's a + /// _connectionless_ protocol, but in mirrord we use a _fakeish_ connection mechanism + /// when dealing with outgoing udp traffic. Connect(LayerConnect), + + /// Write data to the remote address the agent is `connect`ed to. + /// + /// There's no `Read` message, as we're calling `read` in the agent, and we send + /// a [`DaemonUdpOutgoing::Read`] message in case we get some data from this connection. Write(LayerWrite), + + /// The layer closed the connection, this message syncs up the agent, closing it + /// over there as well. + /// + /// Connections in the agent may be closed in other ways, such as when an error happens + /// when reading or writing. Which means that this message is not the only way of + /// closing outgoing udp connections. Close(LayerClose), } #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum DaemonUdpOutgoing { + /// The agent attempted a connection to the remote address specified by + /// [`LayerUdpOutgoing::Connect`], and it might've been successful or not. + /// + /// See the docs for [`LayerUdpOutgoing::Connect`] for a bit more information on the + /// weird idea of `connect` and udp in mirrord. Connect(RemoteResult), + + /// Read data from the connection. + /// + /// There's no `Write` message, as `write`s come from the user (layer). The agent sending + /// a `write` to the layer like this would make no sense, since it could just `write` it + /// to the remote connection itself. Read(RemoteResult), + + /// Tell the layer that this connection has been `close`d, either by a request from + /// the user with [`LayerUdpOutgoing::Close`], or from some error in the agent when + /// writing or reading from the connection. Close(ConnectionId), } diff --git a/mirrord/protocol/src/tcp.rs b/mirrord/protocol/src/tcp.rs index acf3d734121..e98077a62ec 100644 --- a/mirrord/protocol/src/tcp.rs +++ b/mirrord/protocol/src/tcp.rs @@ -52,14 +52,31 @@ pub struct TcpClose { } /// Messages related to Tcp handler from client. +/// +/// Part of the `mirror` feature. #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum LayerTcp { + /// User is interested in mirroring traffic on this `Port`, so add it to the list of + /// ports that the sniffer is filtering. PortSubscribe(Port), + + /// User is not interested in the connection with `ConnectionId` anymore. + /// + /// This means that their app has closed the connection they were `listen`ning on. + /// + /// There is no `ConnectionSubscribe` counter-part of this variant, the subscription + /// happens when the sniffer receives an (agent) internal `SniffedConnection`. ConnectionUnsubscribe(ConnectionId), + + /// Removes this `Port` from the sniffer's filter, the traffic won't be cloned to mirrord + /// anymore. PortUnsubscribe(Port), } /// Messages related to Tcp handler from server. +/// +/// They are the same for both `steal` and `mirror` modes, even though their layer +/// counterparts ([`LayerTcpSteal`] and [`LayerTcp`]) are different. #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum DaemonTcp { NewConnection(NewTcpConnection), @@ -214,10 +231,38 @@ impl StealType { } /// Messages related to Steal Tcp handler from client. +/// +/// `PortSubscribe`, `PortUnsubscribe`, and `ConnectionUnsubscribe` variants are similar +/// to what you'll find in the [`LayerTcp`], but they're handled by different tasks in +/// the agent. +/// +/// Stolen traffic might have an additional overhead when compared to mirrored traffic, as +/// we have an intermmediate HTTP server to handle filtering (based on HTTP headers, etc). #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub enum LayerTcpSteal { + /// User is interested in stealing traffic on this `Port`, so add it to the list of + /// ports that the stealer is filtering. + /// + /// The `TcpConnectionStealer` supports an [`HttpFilter`] granting the ability to steal + /// only traffic that matches the user configured filter. It's also possible to just steal + /// all traffic (which we refer as `Unfiltered`). For more info see [`StealType`]. + /// + /// This variant is somewhat related to [`LayerTcpSteal::ConnectionUnsubscribe`], since + /// we don't have a `ConnectionSubscribe` message anywhere, instead what we do is: when + /// a new connection comes in one of the ports we are subscribed to, we consider it a + /// connection subscription (so this mechanism represents the **non-existing** + /// `ConnectionSubscribe` variant). PortSubscribe(StealType), + + /// User has stopped stealing from this connection with [`ConnectionId`]. + /// + /// We do **not** have a `ConnectionSubscribe` variant/message. What happens instead is that we + /// call a _connection subscription_ the act of `accept`ing a new connection on one of the + /// ports we are subscribed to. See the [`LayerTcpSteal::PortSubscribe`] for more info. ConnectionUnsubscribe(ConnectionId), + + /// Removes this `Port` from the stealers's filter, the traffic won't be stolen by mirrord + /// anymore. PortUnsubscribe(Port), Data(TcpData), HttpResponse(HttpResponse>), From 0aa4f40a7afb478dd1d9cc3d988423f38cffed8f Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 23 Jan 2025 05:30:10 -0300 Subject: [PATCH 22/23] Add statfs support (#3018) * Add statfs hook * + changelog.d * Fix test statfs_fstatfs.rs * Fix statfs_fstatfs.c * Fix statfs_fstatfs.c * Fix statfs_fstatfs.rs * Fix statfs_fstatfs.rs * Fix statfs_fstatfs.rs * Update file.rs * PR comments --- Cargo.lock | 2 +- changelog.d/statfs.added.md | 1 + mirrord/agent/src/file.rs | 20 ++++++- mirrord/intproxy/protocol/src/lib.rs | 7 +++ mirrord/intproxy/src/proxies/files.rs | 14 +++-- mirrord/layer/src/file/hooks.rs | 31 ++++++++++- mirrord/layer/src/file/ops.rs | 13 ++++- mirrord/layer/src/go/linux_x64.rs | 2 + mirrord/layer/src/go/mod.rs | 2 + mirrord/layer/tests/apps/fileops/go/main.go | 13 ++++- .../apps/statfs_fstatfs/statfs_fstatfs.c | 52 +++++++++++++++++++ mirrord/layer/tests/common/mod.rs | 48 ++++++++++++++++- mirrord/layer/tests/fileops.rs | 3 ++ mirrord/layer/tests/statfs_fstatfs.rs | 45 ++++++++++++++++ mirrord/protocol/Cargo.toml | 2 +- mirrord/protocol/src/codec.rs | 1 + mirrord/protocol/src/file.rs | 8 +++ tests/python-e2e/ops.py | 14 ++++- 18 files changed, 265 insertions(+), 13 deletions(-) create mode 100644 changelog.d/statfs.added.md create mode 100644 mirrord/layer/tests/apps/statfs_fstatfs/statfs_fstatfs.c create mode 100644 mirrord/layer/tests/statfs_fstatfs.rs diff --git a/Cargo.lock b/Cargo.lock index 73c192236bf..36af01cf4a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4493,7 +4493,7 @@ dependencies = [ [[package]] name = "mirrord-protocol" -version = "1.15.1" +version = "1.16.0" dependencies = [ "actix-codec", "bincode", diff --git a/changelog.d/statfs.added.md b/changelog.d/statfs.added.md new file mode 100644 index 00000000000..b1cea16a410 --- /dev/null +++ b/changelog.d/statfs.added.md @@ -0,0 +1 @@ +Add statfs support \ No newline at end of file diff --git a/mirrord/agent/src/file.rs b/mirrord/agent/src/file.rs index 0fa945fbd85..571b2ad9d3c 100644 --- a/mirrord/agent/src/file.rs +++ b/mirrord/agent/src/file.rs @@ -223,8 +223,12 @@ impl FileManager { Some(FileResponse::Xstat(xstat_result)) } FileRequest::XstatFs(XstatFsRequest { fd }) => { - let xstat_result = self.xstatfs(fd); - Some(FileResponse::XstatFs(xstat_result)) + let xstatfs_result = self.xstatfs(fd); + Some(FileResponse::XstatFs(xstatfs_result)) + } + FileRequest::StatFs(StatFsRequest { path }) => { + let statfs_result = self.statfs(path); + Some(FileResponse::XstatFs(statfs_result)) } // dir operations @@ -769,6 +773,18 @@ impl FileManager { }) } + #[tracing::instrument(level = "trace", skip(self))] + pub(crate) fn statfs(&mut self, path: PathBuf) -> RemoteResult { + let path = resolve_path(path, &self.root_path)?; + + let statfs = nix::sys::statfs::statfs(&path) + .map_err(|err| std::io::Error::from_raw_os_error(err as i32))?; + + Ok(XstatFsResponse { + metadata: statfs.into(), + }) + } + #[tracing::instrument(level = Level::TRACE, skip(self), err(level = Level::DEBUG))] pub(crate) fn fdopen_dir(&mut self, fd: u64) -> RemoteResult { let path = match self diff --git a/mirrord/intproxy/protocol/src/lib.rs b/mirrord/intproxy/protocol/src/lib.rs index e51fcdf773e..7648f3d6cf6 100644 --- a/mirrord/intproxy/protocol/src/lib.rs +++ b/mirrord/intproxy/protocol/src/lib.rs @@ -387,6 +387,13 @@ impl_request!( res_path = ProxyToLayerMessage::File => FileResponse::XstatFs, ); +impl_request!( + req = StatFsRequest, + res = RemoteResult, + req_path = LayerToProxyMessage::File => FileRequest::StatFs, + res_path = ProxyToLayerMessage::File => FileResponse::XstatFs, +); + impl_request!( req = FdOpenDirRequest, res = RemoteResult, diff --git a/mirrord/intproxy/src/proxies/files.rs b/mirrord/intproxy/src/proxies/files.rs index 517c743ce12..55c1b2f98f0 100644 --- a/mirrord/intproxy/src/proxies/files.rs +++ b/mirrord/intproxy/src/proxies/files.rs @@ -6,7 +6,7 @@ use mirrord_protocol::{ file::{ CloseDirRequest, CloseFileRequest, DirEntryInternal, ReadDirBatchRequest, ReadDirResponse, ReadFileResponse, ReadLimitedFileRequest, SeekFromInternal, MKDIR_VERSION, - READDIR_BATCH_VERSION, READLINK_VERSION, RMDIR_VERSION, + READDIR_BATCH_VERSION, READLINK_VERSION, RMDIR_VERSION, STATFS_VERSION, }, ClientMessage, DaemonMessage, ErrorKindInternal, FileRequest, FileResponse, RemoteIOError, ResponseError, @@ -259,21 +259,27 @@ impl FilesProxy { match request { FileRequest::ReadLink(..) - if protocol_version.is_some_and(|version| !READLINK_VERSION.matches(version)) => + if protocol_version.is_none_or(|version| !READLINK_VERSION.matches(version)) => { Err(FileResponse::ReadLink(Err(ResponseError::NotImplemented))) } FileRequest::MakeDir(..) | FileRequest::MakeDirAt(..) - if protocol_version.is_some_and(|version| !MKDIR_VERSION.matches(version)) => + if protocol_version.is_none_or(|version| !MKDIR_VERSION.matches(version)) => { Err(FileResponse::MakeDir(Err(ResponseError::NotImplemented))) } FileRequest::RemoveDir(..) | FileRequest::Unlink(..) | FileRequest::UnlinkAt(..) if protocol_version - .is_some_and(|version: &Version| !RMDIR_VERSION.matches(version)) => + .is_none_or(|version: &Version| !RMDIR_VERSION.matches(version)) => { Err(FileResponse::RemoveDir(Err(ResponseError::NotImplemented))) } + FileRequest::StatFs(..) + if protocol_version + .is_none_or(|version: &Version| !STATFS_VERSION.matches(version)) => + { + Err(FileResponse::XstatFs(Err(ResponseError::NotImplemented))) + } _ => Ok(()), } } diff --git a/mirrord/layer/src/file/hooks.rs b/mirrord/layer/src/file/hooks.rs index 7c46165b37a..3fcfc3c1280 100644 --- a/mirrord/layer/src/file/hooks.rs +++ b/mirrord/layer/src/file/hooks.rs @@ -904,8 +904,9 @@ unsafe extern "C" fn fstatat_detour( }) } +/// Hook for `libc::fstatfs`. #[hook_guard_fn] -unsafe extern "C" fn fstatfs_detour(fd: c_int, out_stat: *mut statfs) -> c_int { +pub(crate) unsafe extern "C" fn fstatfs_detour(fd: c_int, out_stat: *mut statfs) -> c_int { if out_stat.is_null() { return HookError::BadPointer.into(); } @@ -919,6 +920,25 @@ unsafe extern "C" fn fstatfs_detour(fd: c_int, out_stat: *mut statfs) -> c_int { .unwrap_or_bypass_with(|_| FN_FSTATFS(fd, out_stat)) } +/// Hook for `libc::statfs`. +#[hook_guard_fn] +pub(crate) unsafe extern "C" fn statfs_detour( + raw_path: *const c_char, + out_stat: *mut statfs, +) -> c_int { + if out_stat.is_null() { + return HookError::BadPointer.into(); + } + + crate::file::ops::statfs(raw_path.checked_into()) + .map(|res| { + let res = res.metadata; + fill_statfs(out_stat, &res); + 0 + }) + .unwrap_or_bypass_with(|_| FN_STATFS(raw_path, out_stat)) +} + unsafe fn realpath_logic( source_path: *const c_char, output_path: *mut c_char, @@ -1333,6 +1353,8 @@ pub(crate) unsafe fn enable_file_hooks(hook_manager: &mut HookManager) { FnFstatfs, FN_FSTATFS ); + replace!(hook_manager, "statfs", statfs_detour, FnStatfs, FN_STATFS); + replace!( hook_manager, "fdopendir", @@ -1415,6 +1437,13 @@ pub(crate) unsafe fn enable_file_hooks(hook_manager: &mut HookManager) { FnFstatfs, FN_FSTATFS ); + replace!( + hook_manager, + "statfs$INODE64", + statfs_detour, + FnStatfs, + FN_STATFS + ); replace!( hook_manager, "fdopendir$INODE64", diff --git a/mirrord/layer/src/file/ops.rs b/mirrord/layer/src/file/ops.rs index fcc27507876..8ca2401101f 100644 --- a/mirrord/layer/src/file/ops.rs +++ b/mirrord/layer/src/file/ops.rs @@ -9,7 +9,8 @@ use mirrord_protocol::{ file::{ MakeDirAtRequest, MakeDirRequest, OpenFileRequest, OpenFileResponse, OpenOptionsInternal, ReadFileResponse, ReadLinkFileRequest, ReadLinkFileResponse, RemoveDirRequest, - SeekFileResponse, UnlinkAtRequest, WriteFileResponse, XstatFsResponse, XstatResponse, + SeekFileResponse, StatFsRequest, UnlinkAtRequest, WriteFileResponse, XstatFsResponse, + XstatResponse, }, ResponseError, }; @@ -736,6 +737,16 @@ pub(crate) fn xstatfs(fd: RawFd) -> Detour { Detour::Success(response) } +#[mirrord_layer_macro::instrument(level = "trace")] +pub(crate) fn statfs(path: Detour) -> Detour { + let path = path?; + let lstatfs = StatFsRequest { path }; + + let response = common::make_proxy_request_with_response(lstatfs)??; + + Detour::Success(response) +} + #[cfg(target_os = "linux")] #[mirrord_layer_macro::instrument(level = "trace")] pub(crate) fn getdents64(fd: RawFd, buffer_size: u64) -> Detour { diff --git a/mirrord/layer/src/go/linux_x64.rs b/mirrord/layer/src/go/linux_x64.rs index 18b36700cbe..622a24383d7 100644 --- a/mirrord/layer/src/go/linux_x64.rs +++ b/mirrord/layer/src/go/linux_x64.rs @@ -340,6 +340,8 @@ unsafe extern "C" fn c_abi_syscall_handler( faccessat_detour(param1 as _, param2 as _, param3 as _, 0) as i64 } libc::SYS_fstat => fstat_detour(param1 as _, param2 as _) as i64, + libc::SYS_statfs => statfs_detour(param1 as _, param2 as _) as i64, + libc::SYS_fstatfs => fstatfs_detour(param1 as _, param2 as _) as i64, libc::SYS_getdents64 => getdents64_detour(param1 as _, param2 as _, param3 as _) as i64, #[cfg(all(target_os = "linux", not(target_arch = "aarch64")))] libc::SYS_mkdir => mkdir_detour(param1 as _, param2 as _) as i64, diff --git a/mirrord/layer/src/go/mod.rs b/mirrord/layer/src/go/mod.rs index df810bbdcf9..6a28c3ebfd9 100644 --- a/mirrord/layer/src/go/mod.rs +++ b/mirrord/layer/src/go/mod.rs @@ -101,6 +101,8 @@ unsafe extern "C" fn c_abi_syscall6_handler( .into() } libc::SYS_fstat => fstat_detour(param1 as _, param2 as _) as i64, + libc::SYS_statfs => statfs_detour(param1 as _, param2 as _) as i64, + libc::SYS_fstatfs => fstatfs_detour(param1 as _, param2 as _) as i64, libc::SYS_fsync => fsync_detour(param1 as _) as i64, libc::SYS_fdatasync => fsync_detour(param1 as _) as i64, libc::SYS_openat => { diff --git a/mirrord/layer/tests/apps/fileops/go/main.go b/mirrord/layer/tests/apps/fileops/go/main.go index 5973e013d5e..69db336fb3e 100644 --- a/mirrord/layer/tests/apps/fileops/go/main.go +++ b/mirrord/layer/tests/apps/fileops/go/main.go @@ -7,10 +7,21 @@ import ( func main() { tempFile := "/tmp/test_file.txt" - syscall.Open(tempFile, syscall.O_CREAT|syscall.O_WRONLY, 0644) + fd, _ := syscall.Open(tempFile, syscall.O_CREAT|syscall.O_WRONLY, 0644) var stat syscall.Stat_t err := syscall.Stat(tempFile, &stat) if err != nil { panic(err) } + + var statfs syscall.Statfs_t + err = syscall.Statfs(tempFile, &statfs) + if err != nil { + panic(err) + } + + err = syscall.Fstatfs(fd, &statfs) + if err != nil { + panic(err) + } } diff --git a/mirrord/layer/tests/apps/statfs_fstatfs/statfs_fstatfs.c b/mirrord/layer/tests/apps/statfs_fstatfs/statfs_fstatfs.c new file mode 100644 index 00000000000..6c474cce4e3 --- /dev/null +++ b/mirrord/layer/tests/apps/statfs_fstatfs/statfs_fstatfs.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include +#include +#include +#include + +#if defined(__APPLE__) && defined(__MACH__) +#include +#include +#else +#include +#endif + +/// Test `statfs / fstatfs`. +/// +/// Gets information about a mounted filesystem +/// +int main() +{ + char *tmp_test_path = "/statfs_fstatfs_test_path"; + mkdir(tmp_test_path, 0777); + + // statfs + struct statfs statfs_buf; + if (statfs(tmp_test_path, &statfs_buf) == -1) + { + perror("statfs failed"); + return EXIT_FAILURE; + } + + // fstatfs + int fd = open(tmp_test_path, O_RDONLY); + + if (fd == -1) + { + perror("Error opening tmp_test_path"); + return 1; + } + + struct statfs fstatfs_buf; + if (fstatfs(fd, &fstatfs_buf) == -1) + { + perror("fstatfs failed"); + close(fd); + return EXIT_FAILURE; + } + + close(fd); + return 0; +} diff --git a/mirrord/layer/tests/common/mod.rs b/mirrord/layer/tests/common/mod.rs index 207f1c6c7a6..a4652433768 100644 --- a/mirrord/layer/tests/common/mod.rs +++ b/mirrord/layer/tests/common/mod.rs @@ -17,7 +17,7 @@ use mirrord_intproxy::{agent_conn::AgentConnection, IntProxy}; use mirrord_protocol::{ file::{ AccessFileRequest, AccessFileResponse, OpenFileRequest, OpenOptionsInternal, - ReadFileRequest, SeekFromInternal, XstatRequest, XstatResponse, + ReadFileRequest, SeekFromInternal, XstatFsResponse, XstatRequest, XstatResponse, }, tcp::{DaemonTcp, LayerTcp, NewTcpConnection, TcpClose, TcpData}, ClientMessage, DaemonCodec, DaemonMessage, FileRequest, FileResponse, @@ -489,6 +489,48 @@ impl TestIntProxy { .unwrap(); } + /// Makes a [`FileRequest::Statefs`] and answers it. + pub async fn expect_statfs(&mut self, expected_path: &str) { + // Expecting `statfs` call with path. + assert_matches!( + self.recv().await, + ClientMessage::FileRequest(FileRequest::StatFs( + mirrord_protocol::file::StatFsRequest { path } + )) if path.to_str().unwrap() == expected_path + ); + + // Answer `statfs`. + self.codec + .send(DaemonMessage::File(FileResponse::XstatFs(Ok( + XstatFsResponse { + metadata: Default::default(), + }, + )))) + .await + .unwrap(); + } + + /// Makes a [`FileRequest::Xstatefs`] and answers it. + pub async fn expect_fstatfs(&mut self, expected_fd: u64) { + // Expecting `fstatfs` call with path. + assert_matches!( + self.recv().await, + ClientMessage::FileRequest(FileRequest::XstatFs( + mirrord_protocol::file::XstatFsRequest { fd } + )) if expected_fd == fd + ); + + // Answer `fstatfs`. + self.codec + .send(DaemonMessage::File(FileResponse::XstatFs(Ok( + XstatFsResponse { + metadata: Default::default(), + }, + )))) + .await + .unwrap(); + } + /// Makes a [`FileRequest::RemoveDir`] and answers it. pub async fn expect_remove_dir(&mut self, expected_dir_name: &str) { // Expecting `rmdir` call with path. @@ -784,6 +826,7 @@ pub enum Application { Fork, ReadLink, MakeDir, + StatfsFstatfs, RemoveDir, OpenFile, CIssue2055, @@ -841,6 +884,7 @@ impl Application { Application::Fork => String::from("tests/apps/fork/out.c_test_app"), Application::ReadLink => String::from("tests/apps/readlink/out.c_test_app"), Application::MakeDir => String::from("tests/apps/mkdir/out.c_test_app"), + Application::StatfsFstatfs => String::from("tests/apps/statfs_fstatfs/out.c_test_app"), Application::RemoveDir => String::from("tests/apps/rmdir/out.c_test_app"), Application::Realpath => String::from("tests/apps/realpath/out.c_test_app"), Application::NodeHTTP | Application::NodeIssue2283 | Application::NodeIssue2807 => { @@ -1080,6 +1124,7 @@ impl Application { | Application::Fork | Application::ReadLink | Application::MakeDir + | Application::StatfsFstatfs | Application::RemoveDir | Application::Realpath | Application::RustFileOps @@ -1159,6 +1204,7 @@ impl Application { | Application::Fork | Application::ReadLink | Application::MakeDir + | Application::StatfsFstatfs | Application::RemoveDir | Application::Realpath | Application::Go21Issue834 diff --git a/mirrord/layer/tests/fileops.rs b/mirrord/layer/tests/fileops.rs index 5daacdadc50..de26b318f40 100644 --- a/mirrord/layer/tests/fileops.rs +++ b/mirrord/layer/tests/fileops.rs @@ -345,6 +345,9 @@ async fn go_stat( )))) .await; + intproxy.expect_statfs("/tmp/test_file.txt").await; + intproxy.expect_fstatfs(fd).await; + test_process.wait_assert_success().await; test_process.assert_no_error_in_stderr().await; } diff --git a/mirrord/layer/tests/statfs_fstatfs.rs b/mirrord/layer/tests/statfs_fstatfs.rs new file mode 100644 index 00000000000..38f48c8495f --- /dev/null +++ b/mirrord/layer/tests/statfs_fstatfs.rs @@ -0,0 +1,45 @@ +#![feature(assert_matches)] +use std::{path::Path, time::Duration}; + +use rstest::rstest; + +mod common; +pub use common::*; + +/// Test for the [`libc::statfs`] and [`libc::fstatfs`] functions. +#[rstest] +#[tokio::test] +#[timeout(Duration::from_secs(60))] +async fn mkdir(dylib_path: &Path) { + let application = Application::StatfsFstatfs; + + let (mut test_process, mut intproxy) = application + .start_process_with_layer(dylib_path, Default::default(), None) + .await; + + println!("waiting for file request (mkdir)."); + intproxy + .expect_make_dir("/statfs_fstatfs_test_path", 0o777) + .await; + + println!("waiting for file request (statfs)."); + intproxy.expect_statfs("/statfs_fstatfs_test_path").await; + + println!("waiting for file request (open)."); + let fd: u64 = 1; + intproxy + .expect_file_open_for_reading("/statfs_fstatfs_test_path", fd) + .await; + + println!("waiting for file request (fstatfs)."); + intproxy.expect_fstatfs(fd).await; + + println!("waiting for file request (close)."); + intproxy.expect_file_close(fd).await; + + assert_eq!(intproxy.try_recv().await, None); + + test_process.wait_assert_success().await; + test_process.assert_no_error_in_stderr().await; + test_process.assert_no_error_in_stdout().await; +} diff --git a/mirrord/protocol/Cargo.toml b/mirrord/protocol/Cargo.toml index 7daa0201505..67eab572e62 100644 --- a/mirrord/protocol/Cargo.toml +++ b/mirrord/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mirrord-protocol" -version = "1.15.1" +version = "1.16.0" authors.workspace = true description.workspace = true documentation.workspace = true diff --git a/mirrord/protocol/src/codec.rs b/mirrord/protocol/src/codec.rs index 2b55fa43b07..8e41d9acab1 100644 --- a/mirrord/protocol/src/codec.rs +++ b/mirrord/protocol/src/codec.rs @@ -94,6 +94,7 @@ pub enum FileRequest { RemoveDir(RemoveDirRequest), Unlink(UnlinkRequest), UnlinkAt(UnlinkAtRequest), + StatFs(StatFsRequest), } /// Minimal mirrord-protocol version that allows `ClientMessage::ReadyForLogs` message. diff --git a/mirrord/protocol/src/file.rs b/mirrord/protocol/src/file.rs index 9a8622731fb..4aa25069bb3 100644 --- a/mirrord/protocol/src/file.rs +++ b/mirrord/protocol/src/file.rs @@ -34,6 +34,9 @@ pub static RMDIR_VERSION: LazyLock = pub static OPEN_LOCAL_VERSION: LazyLock = LazyLock::new(|| ">=1.13.3".parse().expect("Bad Identifier")); +pub static STATFS_VERSION: LazyLock = + LazyLock::new(|| ">=1.16.0".parse().expect("Bad Identifier")); + /// Internal version of Metadata across operating system (macOS, Linux) /// Only mutual attributes #[derive(Encode, Decode, Debug, PartialEq, Clone, Copy, Eq, Default)] @@ -413,6 +416,11 @@ pub struct XstatFsRequest { pub fd: u64, } +#[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] +pub struct StatFsRequest { + pub path: PathBuf, +} + #[derive(Encode, Decode, Debug, PartialEq, Eq, Clone)] pub struct XstatResponse { pub metadata: MetadataInternal, diff --git a/tests/python-e2e/ops.py b/tests/python-e2e/ops.py index c107ebb2375..36c7ba5fb8c 100644 --- a/tests/python-e2e/ops.py +++ b/tests/python-e2e/ops.py @@ -88,6 +88,19 @@ def test_mkdir_errors(self): os.close(dir) + def test_statfs_and_fstatvfs_sucess(self): + """ + Test statfs / fstatfs + """ + file_path, _ = self._create_new_tmp_file() + + statvfs_result = os.statvfs(file_path) + self.assertIsNotNone(statvfs_result) + + fd = os.open(file_path, os.O_RDONLY) + fstatvfs_result = os.fstatvfs(fd) + self.assertIsNotNone(fstatvfs_result) + def test_rmdir(self): """ Creates a new directory in "/tmp" and removes it using rmdir. @@ -106,6 +119,5 @@ def _create_new_tmp_file(self): w_file.write(TEXT) return file_path, file_name - if __name__ == "__main__": unittest.main() From afccbc8dc4b9afc26a7034e19b74d01df4657e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Smolarek?= <34063647+Razz4780@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:03:34 +0100 Subject: [PATCH 23/23] Fixed HTTP filter stuck forever (#3031) * Fixed TcpConnectionStealer and ChannelClosedFuture, added unit tests * Changelog * use rstest timeout on cleanup_on_client_closed stealer test --- changelog.d/+http-filter-cleanup.fixed.md | 1 + mirrord/agent/src/sniffer.rs | 46 +++++- mirrord/agent/src/steal/connection.rs | 166 +++++++++++++++++++--- mirrord/agent/src/util.rs | 76 ++++++++-- mirrord/agent/src/watched_task.rs | 14 +- 5 files changed, 263 insertions(+), 40 deletions(-) create mode 100644 changelog.d/+http-filter-cleanup.fixed.md diff --git a/changelog.d/+http-filter-cleanup.fixed.md b/changelog.d/+http-filter-cleanup.fixed.md new file mode 100644 index 00000000000..92adcb9d93c --- /dev/null +++ b/changelog.d/+http-filter-cleanup.fixed.md @@ -0,0 +1 @@ +Agent now correctly clears incoming port subscriptions of disconnected clients. diff --git a/mirrord/agent/src/sniffer.rs b/mirrord/agent/src/sniffer.rs index 0d5cccfb584..94c40e0ce67 100644 --- a/mirrord/agent/src/sniffer.rs +++ b/mirrord/agent/src/sniffer.rs @@ -139,7 +139,7 @@ pub(crate) struct TcpConnectionSniffer { sessions: TCPSessionMap, client_txs: HashMap>, - clients_closed: FuturesUnordered>, + clients_closed: FuturesUnordered, } impl Drop for TcpConnectionSniffer { @@ -432,7 +432,7 @@ mod test { atomic::{AtomicUsize, Ordering}, Arc, }, - time::Duration, + time::{Duration, Instant}, }; use api::TcpSnifferApi; @@ -440,6 +440,7 @@ mod test { tcp::{DaemonTcp, LayerTcp, NewTcpConnection, TcpClose, TcpData}, ConnectionId, LogLevel, }; + use rstest::rstest; use tcp_capture::test::TcpPacketsChannel; use tokio::sync::mpsc; @@ -856,4 +857,45 @@ mod test { }), ); } + + /// Verifies that [`TcpConnectionSniffer`] reacts to [`TcpSnifferApi`] being dropped + /// and clears the packet filter. + #[rstest] + #[timeout(Duration::from_secs(5))] + #[tokio::test] + async fn cleanup_on_client_closed() { + let mut setup = TestSnifferSetup::new(); + + let mut api = setup.get_api().await; + + api.handle_client_message(LayerTcp::PortSubscribe(80)) + .await + .unwrap(); + assert_eq!( + api.recv().await.unwrap(), + (DaemonTcp::SubscribeResult(Ok(80)), None), + ); + assert_eq!(setup.times_filter_changed(), 1); + + std::mem::drop(api); + let dropped_at = Instant::now(); + + loop { + match setup.times_filter_changed() { + 1 => { + println!( + "filter still not changed {}ms after client closed", + dropped_at.elapsed().as_millis() + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + + 2 => { + break; + } + + other => panic!("unexpected times filter changed {other}"), + } + } + } } diff --git a/mirrord/agent/src/steal/connection.rs b/mirrord/agent/src/steal/connection.rs index 6515f2ecfc9..f6b4a9f2b7b 100644 --- a/mirrord/agent/src/steal/connection.rs +++ b/mirrord/agent/src/steal/connection.rs @@ -30,9 +30,9 @@ use tokio::{ use tokio_util::sync::CancellationToken; use tracing::{warn, Level}; -use super::http::HttpResponseFallback; +use super::{http::HttpResponseFallback, subscriptions::PortRedirector}; use crate::{ - error::AgentResult, + error::{AgentError, AgentResult}, metrics::HTTP_REQUEST_IN_PROGRESS_COUNT, steal::{ connections::{ @@ -292,9 +292,9 @@ struct TcpStealerConfig { /// run in the same network namespace as the agent's target. /// /// Enabled by the `steal` feature for incoming traffic. -pub(crate) struct TcpConnectionStealer { +pub(crate) struct TcpConnectionStealer { /// For managing active subscriptions and port redirections. - port_subscriptions: PortSubscriptions, + port_subscriptions: PortSubscriptions, /// For receiving commands. /// The other end of this channel belongs to [`TcpStealerApi`](super::api::TcpStealerApi). @@ -304,7 +304,7 @@ pub(crate) struct TcpConnectionStealer { clients: HashMap, /// [`Future`](std::future::Future)s that resolve when stealer clients close. - clients_closed: FuturesUnordered>, + clients_closed: FuturesUnordered, /// Set of active connections stolen by [`Self::port_subscriptions`]. connections: StolenConnections, @@ -313,7 +313,7 @@ pub(crate) struct TcpConnectionStealer { support_ipv6: bool, } -impl TcpConnectionStealer { +impl TcpConnectionStealer { pub const TASK_NAME: &'static str = "Stealer"; /// Initializes a new [`TcpConnectionStealer`], but doesn't start the actual work. @@ -327,25 +327,39 @@ impl TcpConnectionStealer { .from_env::() .unwrap_or_default(); - let port_subscriptions = { - let redirector = IpTablesRedirector::new( - config.stealer_flush_connections, - config.pod_ips, - support_ipv6, - ) - .await?; + let redirector = IpTablesRedirector::new( + config.stealer_flush_connections, + config.pod_ips, + support_ipv6, + ) + .await?; - PortSubscriptions::new(redirector, 4) - }; + Ok(Self::with_redirector(command_rx, support_ipv6, redirector)) + } +} - Ok(Self { - port_subscriptions, +impl TcpConnectionStealer +where + Redirector: PortRedirector, + Redirector::Error: std::error::Error + Into, + AgentError: From, +{ + /// Creates a new stealer. + /// + /// Given [`PortRedirector`] will be used to capture incoming connections. + pub(crate) fn with_redirector( + command_rx: Receiver, + support_ipv6: bool, + redirector: Redirector, + ) -> Self { + Self { + port_subscriptions: PortSubscriptions::new(redirector, 4), command_rx, clients: HashMap::with_capacity(8), clients_closed: Default::default(), connections: StolenConnections::with_capacity(8), support_ipv6, - }) + } } /// Runs the tcp traffic stealer loop. @@ -383,7 +397,7 @@ impl TcpConnectionStealer { } Err(error) => { tracing::error!(?error, "Failed to accept a stolen connection"); - break Err(error); + break Err(error.into()); } }, @@ -644,6 +658,8 @@ impl TcpConnectionStealer { match command { Command::NewClient(daemon_tx, protocol_version) => { + self.clients_closed + .push(ChannelClosedFuture::new(daemon_tx.clone(), client_id)); self.clients.insert( client_id, Client { @@ -708,7 +724,7 @@ impl TcpConnectionStealer { #[cfg(test)] mod test { - use std::net::SocketAddr; + use std::{net::SocketAddr, time::Duration}; use bytes::Bytes; use futures::{future::BoxFuture, FutureExt}; @@ -719,18 +735,75 @@ mod test { service::Service, }; use hyper_util::rt::TokioIo; - use mirrord_protocol::tcp::{ChunkedRequest, DaemonTcp, InternalHttpBodyFrame}; + use mirrord_protocol::{ + tcp::{ChunkedRequest, DaemonTcp, Filter, HttpFilter, InternalHttpBodyFrame, StealType}, + Port, + }; use rstest::rstest; use tokio::{ net::{TcpListener, TcpStream}, sync::{ mpsc::{self, Receiver, Sender}, - oneshot, + oneshot, watch, }, }; use tokio_stream::wrappers::ReceiverStream; + use tokio_util::sync::CancellationToken; + + use super::AgentError; + use crate::{ + steal::{ + connection::{Client, MatchedHttpRequest}, + subscriptions::PortRedirector, + TcpConnectionStealer, TcpStealerApi, + }, + watched_task::TaskStatus, + }; + + /// Notification about a requested redirection operation. + /// + /// Produced by [`NotifyingRedirector`]. + #[derive(Debug, PartialEq, Eq)] + enum RedirectNotification { + Added(Port), + Removed(Port), + Cleanup, + } + + /// Test [`PortRedirector`] that never fails and notifies about requested operations using an + /// [`mpsc::channel`]. + struct NotifyingRedirector(Sender); + + #[async_trait::async_trait] + impl PortRedirector for NotifyingRedirector { + type Error = AgentError; + + async fn add_redirection(&mut self, port: Port) -> Result<(), Self::Error> { + self.0 + .send(RedirectNotification::Added(port)) + .await + .unwrap(); + Ok(()) + } + + async fn remove_redirection(&mut self, port: Port) -> Result<(), Self::Error> { + self.0 + .send(RedirectNotification::Removed(port)) + .await + .unwrap(); + Ok(()) + } + + async fn cleanup(&mut self) -> Result<(), Self::Error> { + self.0.send(RedirectNotification::Cleanup).await.unwrap(); + Ok(()) + } + + async fn next_connection(&mut self) -> Result<(TcpStream, SocketAddr), Self::Error> { + std::future::pending().await + } + } - use crate::steal::connection::{Client, MatchedHttpRequest}; async fn prepare_dummy_service() -> ( SocketAddr, Receiver<(Request, oneshot::Sender>>)>, @@ -907,4 +980,51 @@ mod test { let _ = response_tx.send(Response::new(Empty::default())); } + + /// Verifies that [`TcpConnectionStealer`] removes client's port subscriptions + /// when client's [`TcpStealerApi`] is dropped. + #[rstest] + #[timeout(Duration::from_secs(5))] + #[tokio::test] + async fn cleanup_on_client_closed() { + let (command_tx, command_rx) = mpsc::channel(8); + let (redirect_tx, mut redirect_rx) = mpsc::channel(2); + let stealer = TcpConnectionStealer::with_redirector( + command_rx, + false, + NotifyingRedirector(redirect_tx), + ); + + tokio::spawn(stealer.start(CancellationToken::new())); + + let (_dummy_tx, dummy_rx) = watch::channel(None); + let task_status = TaskStatus::dummy(TcpConnectionStealer::TASK_NAME, dummy_rx); + let mut api = TcpStealerApi::new( + 0, + command_tx.clone(), + task_status, + 8, + mirrord_protocol::VERSION.clone(), + ) + .await + .unwrap(); + + api.port_subscribe(StealType::FilteredHttpEx( + 80, + HttpFilter::Header(Filter::new("user: test".into()).unwrap()), + )) + .await + .unwrap(); + + let response = api.recv().await.unwrap(); + assert_eq!(response, DaemonTcp::SubscribeResult(Ok(80))); + + let notification = redirect_rx.recv().await.unwrap(); + assert_eq!(notification, RedirectNotification::Added(80)); + + std::mem::drop(api); + + let notification = redirect_rx.recv().await.unwrap(); + assert_eq!(notification, RedirectNotification::Removed(80)); + } } diff --git a/mirrord/agent/src/util.rs b/mirrord/agent/src/util.rs index 0c72cddd82f..c5a002979e9 100644 --- a/mirrord/agent/src/util.rs +++ b/mirrord/agent/src/util.rs @@ -8,6 +8,7 @@ use std::{ thread::JoinHandle, }; +use futures::{future::BoxFuture, FutureExt}; use tokio::sync::mpsc; use tracing::error; @@ -162,27 +163,25 @@ pub(crate) fn enter_namespace(pid: Option, namespace: &str) -> AgentResult< } /// [`Future`] that resolves to [`ClientId`] when the client drops their [`mpsc::Receiver`]. -pub(crate) struct ChannelClosedFuture { - tx: mpsc::Sender, - client_id: ClientId, -} +pub(crate) struct ChannelClosedFuture(BoxFuture<'static, ClientId>); + +impl ChannelClosedFuture { + pub(crate) fn new(tx: mpsc::Sender, client_id: ClientId) -> Self { + let future = async move { + tx.closed().await; + client_id + } + .boxed(); -impl ChannelClosedFuture { - pub(crate) fn new(tx: mpsc::Sender, client_id: ClientId) -> Self { - Self { tx, client_id } + Self(future) } } -impl Future for ChannelClosedFuture { +impl Future for ChannelClosedFuture { type Output = ClientId; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let client_id = self.client_id; - - let future = std::pin::pin!(self.get_mut().tx.closed()); - std::task::ready!(future.poll(cx)); - - Poll::Ready(client_id) + self.get_mut().0.as_mut().poll(cx) } } @@ -264,3 +263,52 @@ mod subscription_tests { assert_eq!(subscriptions.get_subscribed_topics(), Vec::::new()); } } + +#[cfg(test)] +mod channel_closed_tests { + use std::time::Duration; + + use futures::{stream::FuturesUnordered, FutureExt, StreamExt}; + use rstest::rstest; + + use super::*; + + /// Verifies that [`ChannelClosedFuture`] resolves when the related [`mpsc::Receiver`] is + /// dropped. + #[rstest] + #[timeout(Duration::from_secs(5))] + #[tokio::test] + async fn channel_closed_resolves() { + let (tx, rx) = mpsc::channel::<()>(1); + let future = ChannelClosedFuture::new(tx, 0); + std::mem::drop(rx); + assert_eq!(future.await, 0); + } + + /// Verifies that [`ChannelClosedFuture`] works fine when used in [`FuturesUnordered`]. + /// + /// The future used to hold the [`mpsc::Sender`] and call poll [`mpsc::Sender::closed`] in it's + /// [`Future::poll`] implementation. This worked fine when the future was used in a simple way + /// ([`channel_closed_resolves`] test was passing). + /// + /// However, [`FuturesUnordered::next`] was hanging forever due to [`mpsc::Sender::closed`] + /// implementation details. + /// + /// New implementation of [`ChannelClosedFuture`] uses a [`BoxFuture`] internally, which works + /// fine. + #[rstest] + #[timeout(Duration::from_secs(5))] + #[tokio::test] + async fn channel_closed_works_in_futures_unordered() { + let mut unordered: FuturesUnordered = FuturesUnordered::new(); + + let (tx, rx) = mpsc::channel::<()>(1); + let future = ChannelClosedFuture::new(tx, 0); + + unordered.push(future); + + assert!(unordered.next().now_or_never().is_none()); + std::mem::drop(rx); + assert_eq!(unordered.next().await.unwrap(), 0); + } +} diff --git a/mirrord/agent/src/watched_task.rs b/mirrord/agent/src/watched_task.rs index ad06bb238ee..2e7370b262c 100644 --- a/mirrord/agent/src/watched_task.rs +++ b/mirrord/agent/src/watched_task.rs @@ -94,9 +94,21 @@ where } #[cfg(test)] -mod test { +pub(crate) mod test { use super::*; + impl TaskStatus { + pub fn dummy( + task_name: &'static str, + result_rx: Receiver>>, + ) -> Self { + Self { + task_name, + result_rx, + } + } + } + #[tokio::test] async fn simple_successful() { let task = WatchedTask::new("task", async move { Ok(()) });