Hook execve and execv to allow shared sockets.

mirrord/layer/Cargo.toml

@@ -27,6 +27,7 @@ mirrord-intproxy-protocol = { path = "../intproxy/protocol", features = ["codec"
 
 ctor = "0.2"
 libc.workspace = true
+bincode.workspace = true
 nix = { workspace = true, features = ["net", "process", "signal"]}
 tracing.workspace = true
 tracing-subscriber.workspace = true
@@ -56,16 +57,16 @@ dashmap = "5.4"
 hashbrown = "0.14"
 exec.workspace = true
 syscalls = { version = "0.6", features = ["full"] }
+null-terminated = "0.3"
+base64.workspace = true
 
 [target.'cfg(target_os = "macos")'.dependencies]
 mirrord-sip = { path = "../sip" }
-null-terminated = "0.3"
 
 [dev-dependencies]
 mirrord-intproxy = { path = "../intproxy" }
 k8s-openapi.workspace = true
 chrono = { version = "0.4", features = ["clock"] }
-base64.workspace = true
 http-body = { workspace = true }
 hyper = { workspace = true }
 rstest = "*"

mirrord/layer/src/common.rs

@@ -1,14 +1,16 @@
 //! Shared place for a few types and functions that are used everywhere by the layer.
-use std::{ffi::CStr, fmt::Debug, path::PathBuf};
+use std::{ffi::CStr, fmt::Debug, ops::Not, path::PathBuf};
 
 use libc::c_char;
 use mirrord_intproxy_protocol::{IsLayerRequest, IsLayerRequestWithResponse, MessageId};
 use mirrord_protocol::file::OpenOptionsInternal;
+use null_terminated::Nul;
 use tracing::warn;
 
 use crate::{
     detour::{Bypass, Detour},
     error::{HookError, HookResult},
+    exec_hooks::Argv,
     file::OpenOptionsInternalExt,
     PROXY_CONNECTION,
 };
@@ -116,6 +118,24 @@ impl CheckedInto<PathBuf> for *const c_char {
     }
 }
 
+impl CheckedInto<Argv> for *const *const c_char {
+    fn checked_into(self) -> Detour<Argv> {
+        let c_list = self
+            .is_null()
+            .not()
+            .then(|| unsafe { Nul::new_unchecked(self) })?;
+
+        let list = c_list
+            .iter()
+            // Remove the last `null` pointer.
+            .filter(|value| !value.is_null())
+            .map(|value| unsafe { CStr::from_ptr(*value) }.to_owned())
+            .collect::<Argv>();
+
+        Detour::Success(list)
+    }
+}
+
 impl CheckedInto<OpenOptionsInternal> for *const c_char {
     fn checked_into(self) -> Detour<OpenOptionsInternal> {
         CheckedInto::<String>::checked_into(self).map(OpenOptionsInternal::from_mode)

mirrord/layer/src/error.rs

@@ -115,6 +115,9 @@ pub(crate) enum HookError {
 
     #[error("mirrord-layer: address passed to `bind` is not valid for the socket domain")]
     InvalidBindAddressForDomain,
+
+    #[error("mirrord-layer: Failed encoding value with `{0}`!")]
+    BincodeEncode(#[from] bincode::error::EncodeError),
 }
 
 /// Errors internal to mirrord-layer.
@@ -237,6 +240,7 @@ impl From<HookError> for i64 {
             HookError::ProxyError(_) => libc::EINVAL,
             HookError::IO(io_fail) => io_fail.raw_os_error().unwrap_or(libc::EIO),
             HookError::LockError => libc::EINVAL,
+            HookError::BincodeEncode(_) => libc::EINVAL,
             HookError::ResponseError(response_fail) => match response_fail {
                 ResponseError::AllocationFailure(_) => libc::ENOMEM,
                 ResponseError::NotFound(_) => libc::ENOENT,

mirrord/layer/src/exec_hooks.rs

@@ -0,0 +1,69 @@
+#[cfg(target_os = "macos")]
+use std::marker::PhantomData;
+use std::{
+    ffi::{c_char, CString},
+    ptr,
+};
+
+pub(crate) mod hooks;
+
+/// Hold a vector of new CStrings to use instead of the original argv.
+#[derive(Default, Debug)]
+pub(crate) struct Argv(Vec<CString>);
+
+/// This must be memory-same as just a `*const c_char`.
+#[cfg(target_os = "macos")]
+#[repr(C)]
+pub(crate) struct StringPtr<'a> {
+    ptr: *const c_char,
+    _phantom: PhantomData<&'a ()>,
+}
+
+impl Argv {
+    /// Get a null-pointer [`StringPtr`].
+    #[cfg(target_os = "macos")]
+    pub(crate) fn null_string_ptr() -> StringPtr<'static> {
+        StringPtr {
+            ptr: ptr::null(),
+            _phantom: Default::default(),
+        }
+    }
+
+    /// Get a vector of pointers of which the data buffer is memory-same as a null-terminated array
+    /// of pointers to null-terminated strings.
+    #[cfg(target_os = "macos")]
+    pub(crate) fn null_vec(&mut self) -> Vec<StringPtr> {
+        let mut vec: Vec<StringPtr> = self
+            .0
+            .iter()
+            .map(|c_string| StringPtr {
+                ptr: c_string.as_ptr(),
+                _phantom: Default::default(),
+            })
+            .collect();
+        vec.push(Self::null_string_ptr());
+        vec
+    }
+
+    pub(crate) fn leak(self) -> *const *const c_char {
+        let mut list = self
+            .0
+            .into_iter()
+            .map(|value| value.into_raw().cast_const())
+            .collect::<Vec<_>>();
+
+        list.push(ptr::null());
+
+        list.into_raw_parts().0.cast_const()
+    }
+
+    pub(crate) fn push(&mut self, item: CString) {
+        self.0.push(item);
+    }
+}
+
+impl FromIterator<CString> for Argv {
+    fn from_iter<T: IntoIterator<Item = CString>>(iter: T) -> Self {
+        Argv(Vec::from_iter(iter))
+    }
+}

mirrord/layer/src/exec_hooks/hooks.rs

@@ -0,0 +1,114 @@
+use std::{ffi::CString, os::unix::process::parent_id};
+
+use base64::prelude::*;
+use libc::{c_char, c_int, FD_CLOEXEC};
+use mirrord_layer_macro::hook_guard_fn;
+use tracing::Level;
+
+use super::*;
+#[cfg(target_os = "macos")]
+use crate::exec_utils::*;
+use crate::{
+    common::CheckedInto,
+    detour::Detour,
+    hooks::HookManager,
+    replace,
+    socket::{hooks::FN_FCNTL, UserSocket, SHARED_SOCKETS_ENV_VAR},
+    SOCKETS,
+};
+
+#[tracing::instrument(level = Level::TRACE, ret)]
+fn shared_sockets() -> Vec<(i32, UserSocket)> {
+    SOCKETS
+        .iter()
+        .filter_map(|inner| {
+            // We only want to share the sockets that are not `FD_CLOEXEC`.
+            let is_shared = unsafe { FN_FCNTL(*inner.key(), libc::F_GETFD) } & FD_CLOEXEC > 0;
+            is_shared.then(|| (*inner.key(), UserSocket::clone(inner.value())))
+        })
+        .collect::<Vec<_>>()
+}
+
+#[mirrord_layer_macro::instrument(
+    level = Level::DEBUG,
+    ret,
+    fields(
+        pid = std::process::id(),
+        parent_pid = parent_id(),
+    )
+)]
+fn execve(env_vars: Detour<Argv>) -> Detour<*const *const c_char> {
+    let mut env_vars = env_vars?;
+    let encoded = bincode::encode_to_vec(shared_sockets(), bincode::config::standard())
+        .map(|bytes| BASE64_URL_SAFE.encode(bytes))?;
+
+    if !encoded.is_empty() {
+        env_vars.push(CString::new(format!("{SHARED_SOCKETS_ENV_VAR}={encoded}"))?);
+    }
+
+    Detour::Success(env_vars.leak())
+}
+
+#[hook_guard_fn]
+unsafe extern "C" fn execv_detour(path: *const c_char, argv: *const *const c_char) -> c_int {
+    let encoded = bincode::encode_to_vec(shared_sockets(), bincode::config::standard())
+        .map(|bytes| BASE64_URL_SAFE.encode(bytes))
+        .unwrap_or_default();
+
+    if !encoded.is_empty() {
+        std::env::set_var("MIRRORD_SHARED_SOCKETS", encoded);
+    }
+
+    FN_EXECV(path, argv)
+}
+
+/// Hook for `libc::execve`.
+///
+/// - #[cfg(target_os = "macos")]
+///
+/// We change 3 arguments and then call the original functions:
+/// 1. The executable path - we check it for SIP, create a patched binary and use the path to the
+///    new path instead of the original path. If there is no SIP, we use a new string with the same
+///    path.
+/// 2. argv - we strip mirrord's temporary directory from the start of arguments. So if argv[1] is
+///    "/var/folders/1337/mirrord-bin/opt/homebrew/bin/npx" Switch it to "/opt/homebrew/bin/npx"
+///    Also here we create a new array with pointers to new strings, even if there are no changes
+///    needed (except for the case of an error).
+/// 3. envp - We found out that Turbopack (Vercel) spawns a clean "Node" instance without env,
+///    basically stripping all of the important mirrord env.
+///    https://github.com/metalbear-co/mirrord/issues/2500
+///    We restore the `DYLD_INSERT_LIBRARIES` environment variable and all env vars starting with `MIRRORD_` if the dyld var can't be found in `envp`.
+/// If there is an error in the detour, we don't exit or anything, we just call the original libc
+/// function with the original passed arguments.
+#[hook_guard_fn]
+pub(crate) unsafe extern "C" fn execve_detour(
+    path: *const c_char,
+    argv: *const *const c_char,
+    envp: *const *const c_char,
+) -> c_int {
+    let checked_envp = envp.checked_into();
+
+    if let Detour::Success(modified_envp) = execve(checked_envp) {
+        #[cfg(target_os = "macos")]
+        match patch_sip_for_new_process(path, argv, modified_envp) {
+            Detour::Success((new_path, new_argv, new_envp)) => FN_EXECVE(
+                new_path.into_raw().cast_const(),
+                new_argv.leak(),
+                new_envp.leak(),
+            ),
+            _ => FN_EXECVE(path, argv, envp),
+        }
+
+        // tracing::info!("Success execve!");
+        #[cfg(target_os = "linux")]
+        FN_EXECVE(path, argv, modified_envp)
+    } else {
+        tracing::info!("Not success execve!");
+        FN_EXECVE(path, argv, envp)
+    }
+}
+
+pub(crate) unsafe fn enable_exec_hooks(hook_manager: &mut HookManager) {
+    replace!(hook_manager, "execv", execv_detour, FnExecv, FN_EXECV);
+    replace!(hook_manager, "execve", execve_detour, FnExecve, FN_EXECVE);
+}