Add support for manually setting listen ports for int/extproxy (#2793)

* .. * Changelog * Ops * Tiny * Update mirrord/config/src/external_proxy.rs Co-authored-by: Michał Smolarek <[email protected]> * Update --------- Co-authored-by: Michał Smolarek <[email protected]>
metalbear-co · Oct 7, 2024 · 12e3799 · 12e3799
1 parent be70a5a
commit 12e3799
Show file tree

Hide file tree

Showing 10 changed files with 62 additions and 25 deletions.
diff --git a/changelog.d/+int-extproxy-port.changed.md b/changelog.d/+int-extproxy-port.changed.md
@@ -0,0 +1 @@
+Allow setting port for int/extproxy from the command line.
diff --git a/mirrord-schema.json b/mirrord-schema.json
@@ -827,6 +827,13 @@
             "null"
           ]
         },
+        "tls_enable": {
+          "description": "<!--${internal}-->\n\nWhether to use TLS or a plain TCP connection.",
+          "type": [
+            "boolean",
+            "null"
+          ]
+        },
         "tls_key": {
           "description": "<!--${internal}-->\n\nPrivate Key path to be used for wrapping external proxy tcp listener with a tcp acceptor (self-signed one will be generated automaticaly if not specified)",
           "type": [

diff --git a/mirrord/cli/src/config.rs b/mirrord/cli/src/config.rs
@@ -59,11 +59,17 @@ pub(super) enum Commands {
 
     /// External Proxy - used for intproxy when it's running with `mirrord container` command.
     #[command(hide = true, name = "extproxy")]
-    ExternalProxy,
+    ExternalProxy {
+        #[arg(long, default_value_t = 0)]
+        port: u16,
+    },
 
     /// Internal proxy - used to aggregate connections from multiple layers
     #[command(hide = true, name = "intproxy")]
-    InternalProxy,
+    InternalProxy {
+        #[arg(long, default_value_t = 0)]
+        port: u16,
+    },
 
     /// Port forwarding - UNSTABLE FEATURE
     #[command(name = "port-forward")]

diff --git a/mirrord/cli/src/container.rs b/mirrord/cli/src/container.rs
@@ -228,8 +228,9 @@ pub(crate) async fn container_command(
         progress.warning(warning);
     }
 
-    let _internal_proxy_tls_guards = if config.internal_proxy.client_tls_certificate.is_none()
-        || config.internal_proxy.client_tls_key.is_none()
+    let _internal_proxy_tls_guards = if config.external_proxy.tls_enable
+        && (config.internal_proxy.client_tls_certificate.is_none()
+            || config.internal_proxy.client_tls_key.is_none())
     {
         let (internal_proxy_cert, internal_proxy_key) =
             create_self_signed_certificate(vec!["intproxy".to_owned()])?;
@@ -248,8 +249,9 @@ pub(crate) async fn container_command(
         None
     };
 
-    let _external_proxy_tls_guards = if config.external_proxy.tls_certificate.is_none()
-        || config.external_proxy.tls_key.is_none()
+    let _external_proxy_tls_guards = if config.external_proxy.tls_enable
+        && (config.external_proxy.tls_certificate.is_none()
+            || config.external_proxy.tls_key.is_none())
     {
         let external_proxy_subject_alt_names = local_ip()
             .map(|item| item.to_string())

diff --git a/mirrord/cli/src/external_proxy.rs b/mirrord/cli/src/external_proxy.rs
@@ -39,7 +39,7 @@ use mirrord_intproxy::agent_conn::{AgentConnection, ConnectionTlsError};
 use mirrord_protocol::DaemonCodec;
 use tokio::net::{TcpListener, TcpStream};
 use tokio_rustls::server::TlsStream;
-use tokio_util::sync::CancellationToken;
+use tokio_util::{either::Either, sync::CancellationToken};
 use tracing::Level;
 use tracing_subscriber::EnvFilter;
 
@@ -59,7 +59,7 @@ fn print_addr(listener: &TcpListener) -> io::Result<()> {
     Ok(())
 }
 
-pub async fn proxy(watch: drain::Watch) -> Result<()> {
+pub async fn proxy(listen_port: u16, watch: drain::Watch) -> Result<()> {
     let config = LayerConfig::from_env()?;
 
     tracing::info!(?config, "external_proxy starting");
@@ -103,12 +103,14 @@ pub async fn proxy(watch: drain::Watch) -> Result<()> {
     let tls_acceptor = create_external_proxy_tls_acceptor(&config).await?;
     let listener = create_listen_socket(SocketAddr::new(
         local_ip().unwrap_or_else(|_| Ipv4Addr::UNSPECIFIED.into()),
-        0,
+        listen_port,
     ))
     .map_err(ExternalProxyError::ListenerSetup)?;
     print_addr(&listener).map_err(ExternalProxyError::ListenerSetup)?;
 
-    unsafe { detach_io() }.map_err(ExternalProxyError::SetSid)?;
+    if let Err(error) = unsafe { detach_io() }.map_err(ExternalProxyError::SetSid) {
+        tracing::warn!(%error, "unable to detach io");
+    }
 
     let cancellation_token = CancellationToken::new();
     let connections = Arc::new(AtomicUsize::new(0));
@@ -133,7 +135,10 @@ pub async fn proxy(watch: drain::Watch) -> Result<()> {
                     connections.fetch_add(1, Ordering::Relaxed);
 
                     let fut = async move {
-                        let stream = tls_acceptor.accept(stream).await?;
+                        let stream = match tls_acceptor {
+                            Some(tls_acceptor) => Either::Right(tls_acceptor.accept(stream).await?),
+                            None => Either::Left(stream)
+                        };
 
                         handle_connection(stream, peer_addr, agent_conn, connection_cancelation_token).await;
 
@@ -180,7 +185,11 @@ pub async fn proxy(watch: drain::Watch) -> Result<()> {
 
 async fn create_external_proxy_tls_acceptor(
     config: &LayerConfig,
-) -> Result<tokio_rustls::TlsAcceptor, ExternalProxyError> {
+) -> Result<Option<tokio_rustls::TlsAcceptor>, ExternalProxyError> {
+    if !config.external_proxy.tls_enable {
+        return Ok(None);
+    }
+
     let (Some(client_tls_certificate), Some(tls_certificate), Some(tls_key)) = (
         config.internal_proxy.client_tls_certificate.as_ref(),
         config.external_proxy.tls_certificate.as_ref(),
@@ -229,12 +238,12 @@ async fn create_external_proxy_tls_acceptor(
         .with_single_cert(tls_certificate, tls_keys)
         .map_err(ConnectionTlsError::ServerConfig)?;
 
-    Ok(tokio_rustls::TlsAcceptor::from(Arc::new(tls_config)))
+    Ok(Some(tokio_rustls::TlsAcceptor::from(Arc::new(tls_config))))
 }
 
 #[tracing::instrument(level = Level::TRACE, skip(agent_conn))]
 async fn handle_connection(
-    stream: TlsStream<TcpStream>,
+    stream: Either<TcpStream, TlsStream<TcpStream>>,
     peer_addr: SocketAddr,
     mut agent_conn: AgentConnection,
     cancellation_token: CancellationToken,

diff --git a/mirrord/cli/src/internal_proxy.rs b/mirrord/cli/src/internal_proxy.rs
@@ -50,7 +50,7 @@ fn print_addr(listener: &TcpListener) -> io::Result<()> {
 
 /// Main entry point for the internal proxy.
 /// It listens for inbound layer connect and forwards to agent.
-pub(crate) async fn proxy(watch: drain::Watch) -> Result<(), InternalProxyError> {
+pub(crate) async fn proxy(listen_port: u16, watch: drain::Watch) -> Result<(), InternalProxyError> {
     let config = LayerConfig::from_env()?;
 
     tracing::info!(?config, "internal_proxy starting");
@@ -130,7 +130,7 @@ pub(crate) async fn proxy(watch: drain::Watch) -> Result<(), InternalProxyError>
     let agent_conn = connect_and_ping(&config, agent_connect_info, &mut analytics).await?;
 
     // Let it assign address for us then print it for the user.
-    let listener = create_listen_socket(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 0))
+    let listener = create_listen_socket(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), listen_port))
         .map_err(InternalProxyError::ListenerSetup)?;
     print_addr(&listener).map_err(InternalProxyError::ListenerSetup)?;
 

diff --git a/mirrord/cli/src/main.rs b/mirrord/cli/src/main.rs
@@ -671,7 +671,7 @@ fn main() -> miette::Result<()> {
             Commands::ExtensionExec(args) => {
                 extension_exec(*args, watch).await?;
             }
-            Commands::InternalProxy => internal_proxy::proxy(watch).await?,
+            Commands::InternalProxy { port } => internal_proxy::proxy(port, watch).await?,
             Commands::VerifyConfig(args) => verify_config(args).await?,
             Commands::Completions(args) => {
                 let mut cmd: clap::Command = Cli::command();
@@ -683,7 +683,7 @@ fn main() -> miette::Result<()> {
                 let (runtime_args, exec_params) = args.into_parts();
                 container_command(runtime_args, exec_params, watch).await?
             }
-            Commands::ExternalProxy => external_proxy::proxy(watch).await?,
+            Commands::ExternalProxy { port } => external_proxy::proxy(port, watch).await?,
             Commands::PortForward(args) => port_forward(&args, watch).await?,
             Commands::Vpn(args) => vpn::vpn_command(*args).await?,
         };
@@ -711,7 +711,7 @@ fn init_ext_error_handler(commands: &Commands) -> bool {
             let _ = miette::set_hook(Box::new(|_| Box::new(JSONReportHandler::new())));
             true
         }
-        Commands::InternalProxy | Commands::ExternalProxy => true,
+        Commands::InternalProxy { .. } | Commands::ExternalProxy { .. } => true,
         _ => false,
     }
 }

diff --git a/mirrord/config/src/external_proxy.rs b/mirrord/config/src/external_proxy.rs
@@ -27,6 +27,12 @@ pub static MIRRORD_EXTERNAL_TLS_KEY_ENV: &str = "MIRRORD_EXTERNAL_TLS_KEY";
 #[config(map_to = "ExternalProxyFileConfig", derive = "JsonSchema")]
 #[cfg_attr(test, config(derive = "PartialEq"))]
 pub struct ExternalProxyConfig {
+    /// <!--${internal}-->
+    ///
+    /// Whether to use TLS or a plain TCP connection.
+    #[config(default = true)]
+    pub tls_enable: bool,
+
     /// <!--${internal}-->
     ///
     /// Certificate path to be used for wrapping external proxy tcp listener with a tcp acceptor

diff --git a/mirrord/intproxy/src/agent_conn.rs b/mirrord/intproxy/src/agent_conn.rs
@@ -131,11 +131,17 @@ impl AgentConnection {
 
                 let stream = socket.connect(proxy_addr).await?;
 
-                if let (Some(tls_certificate), Some(client_tls_certificate), Some(client_tls_key)) = (
-                    config.external_proxy.tls_certificate.as_ref(),
-                    config.internal_proxy.client_tls_certificate.as_ref(),
-                    config.internal_proxy.client_tls_key.as_ref(),
-                ) {
+                if config.external_proxy.tls_enable
+                    && let (
+                        Some(tls_certificate),
+                        Some(client_tls_certificate),
+                        Some(client_tls_key),
+                    ) = (
+                        config.external_proxy.tls_certificate.as_ref(),
+                        config.internal_proxy.client_tls_certificate.as_ref(),
+                        config.internal_proxy.client_tls_key.as_ref(),
+                    )
+                {
                     wrap_connection_with_tls(
                         stream,
                         proxy_addr.ip(),

diff --git a/mirrord/intproxy/src/lib.rs b/mirrord/intproxy/src/lib.rs
@@ -1,4 +1,4 @@
-#![feature(map_try_insert)]
+#![feature(map_try_insert, let_chains)]
 #![warn(clippy::indexing_slicing)]
 
 use std::{collections::HashMap, time::Duration};
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Allow setting port for int/extproxy from the command line.