Hook execve and execv to allow shared sockets. (#2610)

* trying to send the sockets in execve, once again

* envp for the (almost) win

* bad file descriptor

* still bad

* new logs

* filtering

* little bit cleaner

* should work on macos

* changes

* cleaning up

* works, but can we do better than set_var

* Does this compile on macos

* add test file // less broken macos

* import exec

* fix imports

* disambiguate exec

* patch binaries is unused (question mark)

* remove unused

* remove unused fn

* trace logs

* different name for test

* uvicorn with reload

* more logs

* better log

* should work bash test

* works on all (question mark)

* clippy

* clippy

* exit from python

* leak on macos

* it was already leaking

* send sigterm

* wait for a bit

* actual message

* wait for longer

* add patch_binaries back

* readd function and fix macos

* wait for 10 secs before sigkil

* macos execve on posix_spawn

* compile macos

* remove dumb code

* please compile

* clippy macos

* no color in logs

* lgs

* cfg not macos

* dont hook execv for mac

* test typo

* macOS

* clippy

* coment

* docs

* remove dead code

* docs

* docs

* logs and docs

* fix docs

* fix url

* changelog

* insert sockets even if envp is empty

* clippy

* fix test

* dont test macos

* always patch sip

* fix macos

* clippy macos

* shadow variables

* trace

* chcked_into macos

* macos doesnt need checkedinto ...

---------

Co-authored-by: Aviram Hassan <[email protected]>

changelog.d/864.fixed.md

@@ -0,0 +1 @@
+Pass the list of UserSocket to child processes when exec is called through an env var MIRRORD_SHARED_SOCKETS.

mirrord/layer/Cargo.toml

@@ -27,6 +27,7 @@ mirrord-intproxy-protocol = { path = "../intproxy/protocol", features = ["codec"
 
 ctor = "0.2"
 libc.workspace = true
+bincode.workspace = true
 nix = { workspace = true, features = ["net", "process", "signal"]}
 tracing.workspace = true
 tracing-subscriber.workspace = true
@@ -56,16 +57,16 @@ dashmap = "5.4"
 hashbrown = "0.14"
 exec.workspace = true
 syscalls = { version = "0.6", features = ["full"] }
+null-terminated = "0.3"
+base64.workspace = true
 
 [target.'cfg(target_os = "macos")'.dependencies]
 mirrord-sip = { path = "../sip" }
-null-terminated = "0.3"
 
 [dev-dependencies]
 mirrord-intproxy = { path = "../intproxy" }
 k8s-openapi.workspace = true
 chrono = { version = "0.4", features = ["clock"] }
-base64.workspace = true
 http-body = { workspace = true }
 hyper = { workspace = true }
 rstest = "*"

mirrord/layer/src/common.rs

@@ -1,14 +1,16 @@
 //! Shared place for a few types and functions that are used everywhere by the layer.
-use std::{ffi::CStr, fmt::Debug, path::PathBuf};
+use std::{ffi::CStr, fmt::Debug, ops::Not, path::PathBuf};
 
 use libc::c_char;
 use mirrord_intproxy_protocol::{IsLayerRequest, IsLayerRequestWithResponse, MessageId};
 use mirrord_protocol::file::OpenOptionsInternal;
+use null_terminated::Nul;
 use tracing::warn;
 
 use crate::{
     detour::{Bypass, Detour},
     error::{HookError, HookResult},
+    exec_hooks::Argv,
     file::OpenOptionsInternalExt,
     PROXY_CONNECTION,
 };
@@ -116,6 +118,26 @@ impl CheckedInto<PathBuf> for *const c_char {
     }
 }
 
+// **Warning**: The implementation here expects that `*const *const c_char` be a valid,
+// null-terminated list! We're using `Nul::new_unchecked`, which doesn't check for this.
+impl CheckedInto<Argv> for *const *const c_char {
+    fn checked_into(self) -> Detour<Argv> {
+        let c_list = self
+            .is_null()
+            .not()
+            .then(|| unsafe { Nul::new_unchecked(self) })?;
+
+        let list = c_list
+            .iter()
+            // Remove the last `null` pointer.
+            .filter(|value| !value.is_null())
+            .map(|value| unsafe { CStr::from_ptr(*value) }.to_owned())
+            .collect::<Argv>();
+
+        Detour::Success(list)
+    }
+}
+
 impl CheckedInto<OpenOptionsInternal> for *const c_char {
     fn checked_into(self) -> Detour<OpenOptionsInternal> {
         CheckedInto::<String>::checked_into(self).map(OpenOptionsInternal::from_mode)

mirrord/layer/src/detour.rs

@@ -351,6 +351,15 @@ impl<S> Detour<S> {
             Detour::Error(e) => op(e),
         }
     }
+
+    #[inline]
+    pub fn or_bypass<O: FnOnce(Bypass) -> Detour<S>>(self, op: O) -> Detour<S> {
+        match self {
+            Detour::Success(s) => Detour::Success(s),
+            Detour::Bypass(b) => op(b),
+            Detour::Error(e) => Detour::Error(e),
+        }
+    }
 }
 
 impl<S> Detour<S>

mirrord/layer/src/error.rs

@@ -115,6 +115,9 @@ pub(crate) enum HookError {
 
     #[error("mirrord-layer: address passed to `bind` is not valid for the socket domain")]
     InvalidBindAddressForDomain,
+
+    #[error("mirrord-layer: Failed encoding value with `{0}`!")]
+    BincodeEncode(#[from] bincode::error::EncodeError),
 }
 
 /// Errors internal to mirrord-layer.
@@ -237,6 +240,7 @@ impl From<HookError> for i64 {
             HookError::ProxyError(_) => libc::EINVAL,
             HookError::IO(io_fail) => io_fail.raw_os_error().unwrap_or(libc::EIO),
             HookError::LockError => libc::EINVAL,
+            HookError::BincodeEncode(_) => libc::EINVAL,
             HookError::ResponseError(response_fail) => match response_fail {
                 ResponseError::AllocationFailure(_) => libc::ENOMEM,
                 ResponseError::NotFound(_) => libc::ENOENT,

mirrord/layer/src/exec_hooks.rs

@@ -0,0 +1,41 @@
+use std::{
+    ffi::{c_char, CString},
+    ptr,
+};
+
+pub(crate) mod hooks;
+
+/// Hold a vector of new CStrings to use instead of the original argv.
+#[derive(Default, Debug, Clone)]
+pub(crate) struct Argv(Vec<CString>);
+
+impl Argv {
+    /// Turns this list of [`CString`] into a C list of pointers (null-terminated).
+    ///
+    /// We leak the [`CString`]s, so that they may live in C-land.
+    pub(crate) fn leak(self) -> *const *const c_char {
+        // Leaks the strings.
+        let mut list = self
+            .0
+            .into_iter()
+            .map(|value| value.into_raw().cast_const())
+            .collect::<Vec<_>>();
+
+        // Null-terminated.
+        list.push(ptr::null());
+
+        // Leaks the list itself.
+        list.into_raw_parts().0.cast_const()
+    }
+
+    /// Convenience to [`Vec::push`] a new [`CString`].
+    pub(crate) fn push(&mut self, item: CString) {
+        self.0.push(item);
+    }
+}
+
+impl FromIterator<CString> for Argv {
+    fn from_iter<T: IntoIterator<Item = CString>>(iter: T) -> Self {
+        Argv(Vec::from_iter(iter))
+    }
+}

mirrord/layer/src/exec_hooks/hooks.rs

@@ -0,0 +1,144 @@
+use std::{ffi::CString, os::unix::process::parent_id};
+
+use base64::prelude::*;
+use libc::{c_char, c_int};
+use mirrord_layer_macro::hook_guard_fn;
+use tracing::Level;
+
+use super::*;
+#[cfg(not(target_os = "macos"))]
+use crate::common::CheckedInto;
+#[cfg(target_os = "macos")]
+use crate::exec_utils::*;
+use crate::{
+    detour::{Bypass, Detour},
+    hooks::HookManager,
+    replace,
+    socket::{UserSocket, SHARED_SOCKETS_ENV_VAR},
+    SOCKETS,
+};
+
+/// Converts the [`SOCKETS`] map into a vector of pairs `(Fd, UserSocket)`, so we can rebuild
+/// it as a map.
+#[mirrord_layer_macro::instrument(level = Level::TRACE, ret)]
+fn shared_sockets() -> Vec<(i32, UserSocket)> {
+    SOCKETS
+        .iter()
+        .map(|inner| (*inner.key(), UserSocket::clone(inner.value())))
+        .collect::<Vec<_>>()
+}
+
+/// Takes an [`Argv`] with the enviroment variables from an `exec` call, extending it with
+/// an encoded version of our [`SOCKETS`].
+///
+/// The check for [`libc::FD_CLOEXEC`] is performed during the [`SOCKETS`] initialization
+/// by the child process.
+#[mirrord_layer_macro::instrument(
+    level = Level::TRACE,
+    ret,
+    fields(
+        pid = std::process::id(),
+        parent_pid = parent_id(),
+    )
+)]
+pub(crate) fn prepare_execve_envp(env_vars: Detour<Argv>) -> Detour<Argv> {
+    let mut env_vars = env_vars.or_bypass(|reason| match reason {
+        Bypass::EmptyOption => Detour::Success(Argv(Vec::new())),
+        other => Detour::Bypass(other),
+    })?;
+
+    let encoded = bincode::encode_to_vec(shared_sockets(), bincode::config::standard())
+        .map(|bytes| BASE64_URL_SAFE.encode(bytes))?;
+
+    env_vars.push(CString::new(format!("{SHARED_SOCKETS_ENV_VAR}={encoded}"))?);
+
+    Detour::Success(env_vars)
+}
+
+/// Hook for `libc::execv` for linux only.
+///
+/// On macos this just calls `execve(path, argv, _environ)`, so we'll be handling it in our
+/// [`execve_detour`].
+#[cfg(not(target_os = "macos"))]
+#[hook_guard_fn]
+unsafe extern "C" fn execv_detour(path: *const c_char, argv: *const *const c_char) -> c_int {
+    let encoded = bincode::encode_to_vec(shared_sockets(), bincode::config::standard())
+        .map(|bytes| BASE64_URL_SAFE.encode(bytes))
+        .unwrap_or_default();
+
+    // `encoded` is emtpy if the encoding failed, so we don't set the env var.
+    if !encoded.is_empty() {
+        std::env::set_var("MIRRORD_SHARED_SOCKETS", encoded);
+    }
+
+    FN_EXECV(path, argv)
+}
+
+/// Hook for `libc::execve`.
+///
+/// We can't change the pointers, to get around that we create our own and **leak** them.
+#[cfg(not(target_os = "macos"))]
+#[hook_guard_fn]
+pub(crate) unsafe extern "C" fn execve_detour(
+    path: *const c_char,
+    argv: *const *const c_char,
+    envp: *const *const c_char,
+) -> c_int {
+    // Hopefully `envp` is a properly null-terminated list.
+    if let Detour::Success(envp) = prepare_execve_envp(envp.checked_into()) {
+        FN_EXECVE(path, argv, envp.leak())
+    } else {
+        FN_EXECVE(path, argv, envp)
+    }
+}
+
+/// Hook for `libc::execve`.
+///
+/// We can't change the pointers, to get around that we create our own and **leak** them.
+///
+/// - #[cfg(target_os = "macos")]
+///
+/// We change 3 arguments and then call the original functions:
+///
+/// 1. The executable path - we check it for SIP, create a patched binary and use the path to the
+/// new path instead of the original path. If there is no SIP, we use a new string with the same
+/// path.
+/// 2. argv - we strip mirrord's temporary directory from the start of arguments.
+/// So if `argv[1]` is "/var/folders/1337/mirrord-bin/opt/homebrew/bin/npx", switch it
+/// to "/opt/homebrew/bin/npx". Also here we create a new array with pointers
+/// to new strings, even if there are no changes needed (except for the case of an error).
+/// 3. envp - We found out that Turbopack (Vercel) spawns a clean "Node" instance without env,
+/// basically stripping all of the important mirrord env.
+/// [#2500](https://github.com/metalbear-co/mirrord/issues/2500)
+/// We restore the `DYLD_INSERT_LIBRARIES` environment variable and all env vars
+/// starting with `MIRRORD_` if the dyld var can't be found in `envp`.
+///
+/// If there is an error in the detour, we don't exit or anything, we just call the original libc
+/// function with the original passed arguments.
+#[cfg(target_os = "macos")]
+#[hook_guard_fn]
+pub(crate) unsafe extern "C" fn execve_detour(
+    path: *const c_char,
+    argv: *const *const c_char,
+    envp: *const *const c_char,
+) -> c_int {
+    match patch_sip_for_new_process(path, argv, envp) {
+        Detour::Success((path, argv, envp)) => {
+            match prepare_execve_envp(Detour::Success(envp.clone())) {
+                Detour::Success(envp) => {
+                    FN_EXECVE(path.into_raw().cast_const(), argv.leak(), envp.leak())
+                }
+                _ => FN_EXECVE(path.into_raw().cast_const(), argv.leak(), envp.leak()),
+            }
+        }
+        _ => FN_EXECVE(path, argv, envp),
+    }
+}
+
+/// Enables `exec` hooks.
+pub(crate) unsafe fn enable_exec_hooks(hook_manager: &mut HookManager) {
+    #[cfg(not(target_os = "macos"))]
+    replace!(hook_manager, "execv", execv_detour, FnExecv, FN_EXECV);
+
+    replace!(hook_manager, "execve", execve_detour, FnExecve, FN_EXECVE);
+}