Adds sendmsg and recvmsg hooks mainly for DNS resolving (#1776). (#1778)

* Adds sendmsg and recvmsg hooks mainly for DNS resolving (#1776).

* Looks like it works on linux

* Duplicate send_to as sendmsg to get more control over what happens.

* hook send/recv msg nocancel

* add sendmsg and recvmsg tests

* update ci with test

* add missing config file

* fix hook name

* sendmsg now modifies msghdr

* Try to connect to remote addr

* back to old send_to

* change test to mimick issue1458 dns test

* Send correct amount of bytes back

* Fix test stuff.

* fix issue 1776 port not 53 test.

* change ci cargo build

* print errno

* print more stuff

* Cleaning up recvmsg and sendmsg

* null destination is just send

* simplify sendmsg a bit

* dont leak message_header

* mostly docs

* test now more similar to 1458

* fix portnot53 test

* docs saying we ignore cmsghdr

* fix tests expect the right address

* raw -> rawish

* debug -> trace

* remove unused feature

* changelog

* test was using the wrong binary

* use the right test for 1776

.github/workflows/ci.yaml

@@ -359,6 +359,12 @@ jobs:
       - run: |
           cd mirrord/layer/tests/apps/listen_ports
           cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue1776
+          cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue1776portnot53
+          cargo build
       - run: ./scripts/build_c_apps.sh
       - name: download layer # Download layer lib built in the `build_mirrord` job.
         uses: actions/download-artifact@v3
@@ -424,6 +430,12 @@ jobs:
       - run: |
           cd mirrord/layer/tests/apps/dns_resolve
           cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue1776
+          cargo build
+      - run: |
+          cd mirrord/layer/tests/apps/issue1776portnot53
+          cargo build
       - run: ./scripts/build_c_apps.sh
       # For the `java_temurin_sip` test.
       - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f

Cargo.toml

@@ -7,6 +7,8 @@ members = [
     "mirrord/layer/tests/apps/listen_ports",
     "mirrord/layer/tests/apps/dns_resolve",
     "mirrord/layer/tests/apps/recv_from",
+    "mirrord/layer/tests/apps/issue1776",
+    "mirrord/layer/tests/apps/issue1776portnot53",
     "sample/rust",
     "medschool",
     "tests",

changelog.d/1776.added.md

@@ -0,0 +1 @@
+Introduced hooks for sendmsg and recvmsg, so mongodb+srv protocol (Csharp) may resolve DNS (implementation follows previous sendto and recvfrom patch).

mirrord/layer/src/socket/hooks.rs

@@ -370,6 +370,55 @@ pub(super) unsafe extern "C" fn send_to_detour(
         })
     }
 }
+
+/// Not a faithful reproduction of what [`libc::recvmsg`] is supposed to do, see [`recv_from`].
+///
+/// TODO(alex): We are ignoring the control message header [`libc::cmsghdr`].
+#[hook_guard_fn]
+pub(super) unsafe extern "C" fn recvmsg_detour(
+    sockfd: i32,
+    message_header: *mut libc::msghdr,
+    flags: c_int,
+) -> ssize_t {
+    let recvmsg_result = FN_RECVMSG(sockfd, message_header, flags);
+
+    if recvmsg_result == -1 && errno::errno() != errno::Errno(libc::EAGAIN) {
+        recvmsg_result
+    } else {
+        // Fills the address, similar to how `recv_from` works.
+        recv_from(
+            sockfd,
+            recvmsg_result,
+            (*message_header).msg_name as *mut _,
+            &mut (*message_header).msg_namelen,
+        )
+        .unwrap_or_bypass(recvmsg_result)
+    }
+}
+
+/// Not a faithful reproduction of what [`libc::sendmsg`] is supposed to do, see [`sendmsg`].
+///
+/// TODO(alex): We are ignoring the control message header [`libc::cmgshdr`].
+#[hook_guard_fn]
+pub(super) unsafe extern "C" fn sendmsg_detour(
+    sockfd: RawFd,
+    message_header: *const libc::msghdr,
+    flags: c_int,
+) -> ssize_t {
+    // When the whole header is null, the operation happens, but does basically nothing (afaik).
+    //
+    // If you ever hit an issue with this, maybe null here is meant to `libc::send` a 0-sized
+    // message?
+    //
+    // When `msg_name` is null, this is equivalent to `send`.
+    if message_header.is_null() || (*message_header).msg_name.is_null() {
+        FN_SENDMSG(sockfd, message_header, flags)
+    } else {
+        sendmsg(sockfd, message_header, flags)
+            .unwrap_or_bypass_with(|_| FN_SENDMSG(sockfd, message_header, flags))
+    }
+}
+
 pub(crate) unsafe fn enable_socket_hooks(hook_manager: &mut HookManager, enabled_remote_dns: bool) {
     replace!(hook_manager, "socket", socket_detour, FnSocket, FN_SOCKET);
 
@@ -387,6 +436,20 @@ pub(crate) unsafe fn enable_socket_hooks(hook_manager: &mut HookManager, enabled
         FnSend_to,
         FN_SEND_TO
     );
+    replace!(
+        hook_manager,
+        "recvmsg",
+        recvmsg_detour,
+        FnRecvmsg,
+        FN_RECVMSG
+    );
+    replace!(
+        hook_manager,
+        "sendmsg",
+        sendmsg_detour,
+        FnSendmsg,
+        FN_SENDMSG
+    );
 
     replace!(hook_manager, "bind", bind_detour, FnBind, FN_BIND);
     replace!(hook_manager, "listen", listen_detour, FnListen, FN_LISTEN);

mirrord/layer/src/socket/ops.rs

@@ -17,7 +17,7 @@ use mirrord_protocol::{
 };
 use socket2::SockAddr;
 use tokio::sync::oneshot;
-use tracing::{debug, error, info, trace};
+use tracing::{error, info, trace};
 
 use super::{hooks::*, *};
 use crate::{
@@ -304,7 +304,7 @@ pub(super) fn listen(sockfd: RawFd, backlog: c_int) -> Detour<i32> {
 /// interception procedure.
 /// This returns errno so we can restore the correct errno in case result is -1 (until we get
 /// back to the hook we might call functions that will corrupt errno)
-#[tracing::instrument(level = "debug", ret)]
+#[tracing::instrument(level = "trace", ret)]
 fn connect_outgoing<const PROTOCOL: ConnectProtocol, const CALL_CONNECT: bool>(
     sockfd: RawFd,
     remote_address: SockAddr,
@@ -364,6 +364,8 @@ fn connect_outgoing<const PROTOCOL: ConnectProtocol, const CALL_CONNECT: bool>(
             layer_address: Some(layer_address.try_into()?),
         };
 
+        trace!("we are connected {connected:#?}");
+
         Arc::get_mut(&mut user_socket_info).unwrap().state = SocketState::Connected(connected);
         SOCKETS.insert(sockfd, user_socket_info);
 
@@ -552,7 +554,7 @@ pub(super) fn getpeername(
             })?
     };
 
-    debug!("getpeername -> remote_address {:#?}", remote_address);
+    trace!("getpeername -> remote_address {:#?}", remote_address);
 
     fill_address(address, address_len, remote_address.try_into()?)
 }
@@ -793,7 +795,7 @@ pub(super) fn getaddrinfo(
         })
         .ok_or(HookError::DNSNoName)?;
 
-    debug!("getaddrinfo -> result {:#?}", result);
+    trace!("getaddrinfo -> result {:#?}", result);
 
     Detour::Success(result)
 }
@@ -864,6 +866,11 @@ pub(super) fn recv_from(
     raw_source: *mut sockaddr,
     source_length: *mut socklen_t,
 ) -> Detour<isize> {
+    if errno::errno() == errno::Errno(libc::EAGAIN) {
+        trace!("recvfrom/recvmsg hit EAGAIN, setting errno to 0");
+        errno::set_errno(errno::Errno(0));
+    }
+
     SOCKETS
         .get(&sockfd)
         .and_then(|socket| match &socket.state {
@@ -878,6 +885,55 @@ pub(super) fn recv_from(
     Detour::Success(recv_from_result)
 }
 
+/// Helps manually resolving DNS on port `53` with UDP, see [`send_to`] and [`sendmsg`].
+#[tracing::instrument(level = "trace", ret)]
+fn send_dns_patch(
+    sockfd: RawFd,
+    user_socket_info: Arc<UserSocket>,
+    destination: SocketAddr,
+) -> Detour<SockAddr> {
+    // We want to keep holding this socket.
+    SOCKETS.insert(sockfd, user_socket_info);
+
+    // Sending a packet on port NOT 53.
+    let destination = SOCKETS
+        .iter()
+        .filter(|socket| socket.kind.is_udp())
+        // Is the `destination` one of our sockets? If so, then we grab the actual address,
+        // instead of the, possibly fake address from mirrord.
+        .find_map(|receiver_socket| match &receiver_socket.state {
+            SocketState::Bound(Bound {
+                requested_address,
+                address,
+            }) => {
+                // Special case for port `0`, see `getsockname`.
+                if requested_address.port() == 0 {
+                    (SocketAddr::new(requested_address.ip(), address.port()) == destination)
+                        .then_some(*address)
+                } else {
+                    (*requested_address == destination).then_some(*address)
+                }
+            }
+            SocketState::Connected(Connected {
+                remote_address,
+                layer_address,
+                ..
+            }) => {
+                let remote_address: SocketAddr = remote_address.clone().try_into().ok()?;
+                let layer_address: SocketAddr = layer_address.clone()?.try_into().ok()?;
+
+                if remote_address == destination {
+                    Some(layer_address)
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        })?;
+
+    Detour::Success(SockAddr::from(destination))
+}
+
 /// ## DNS resolution on port `53`
 ///
 /// There is a bit of trickery going on here, as this function first triggers a _semantical_
@@ -932,57 +988,16 @@ pub(super) fn send_to(
         .as_socket()
         .filter(|destination| destination.port() != 53)
     {
-        // We want to keep holding this socket.
-        SOCKETS.insert(sockfd, user_socket_info);
-
-        // Sending a packet on port NOT 53.
-        let destination = SOCKETS
-            .iter()
-            .filter(|socket| socket.kind.is_udp())
-            // Is the `destination` one of our sockets? If so, then we grab the actual address,
-            // instead of the, possibly fake address from mirrord.
-            .find_map(|receiver_socket| match &receiver_socket.state {
-                SocketState::Bound(Bound {
-                    requested_address,
-                    address,
-                }) => {
-                    // Special case for port `0`, see `getsockname`.
-                    if requested_address.port() == 0 {
-                        (SocketAddr::new(requested_address.ip(), address.port()) == destination)
-                            .then_some(*address)
-                    } else {
-                        (*requested_address == destination).then_some(*address)
-                    }
-                }
-                SocketState::Connected(Connected {
-                    remote_address,
-                    layer_address,
-                    ..
-                }) => {
-                    let remote_address: SocketAddr = remote_address.clone().try_into().ok()?;
-                    let layer_address: SocketAddr = layer_address.clone()?.try_into().ok()?;
-
-                    if remote_address == destination {
-                        Some(layer_address)
-                    } else {
-                        None
-                    }
-                }
-                _ => None,
-            })?;
-
-        let rawish_true_destination = SockAddr::from(destination);
-        let raw_true_destination = rawish_true_destination.as_ptr();
-        let raw_true_destination_length = rawish_true_destination.len();
+        let rawish_true_destination = send_dns_patch(sockfd, user_socket_info, destination)?;
 
         unsafe {
             FN_SEND_TO(
                 sockfd,
                 raw_message,
                 message_length,
                 flags,
-                raw_true_destination,
-                raw_true_destination_length,
+                rawish_true_destination.as_ptr(),
+                rawish_true_destination.len(),
             )
         }
     } else {
@@ -1013,3 +1028,78 @@ pub(super) fn send_to(
 
     Detour::Success(sent_result)
 }
+
+/// Same behavior as [`send_to`], the only difference is that here we deal with [`libc::msghdr`],
+/// instead of directly with socket addresses.
+#[tracing::instrument(level = "trace", ret, skip(raw_message_header))]
+pub(super) fn sendmsg(
+    sockfd: RawFd,
+    raw_message_header: *const libc::msghdr,
+    flags: i32,
+) -> Detour<isize> {
+    // We have a destination, so apply our fake `connect` patch.
+    let destination = (!unsafe { *raw_message_header }.msg_name.is_null()).then(|| {
+        let raw_destination = unsafe { *raw_message_header }.msg_name as *const libc::sockaddr;
+        let destination_length = unsafe { *raw_message_header }.msg_namelen;
+        SockAddr::try_from_raw(raw_destination, destination_length)
+    })??;
+
+    trace!("destination {:?}", destination.as_socket());
+
+    let (_, user_socket_info) = SOCKETS
+        .remove(&sockfd)
+        .ok_or(Bypass::LocalFdNotFound(sockfd))?;
+
+    // Currently this flow only handles DNS resolution.
+    // So here we have to check for 2 things:
+    //
+    // 1. Are we sending something port 53? Then we use mirrord flow;
+    // 2. Is the destination a socket that we have bound? Then we send it to the real address that
+    // we've bound the destination socket.
+    //
+    // If none of the above are true, then the destination is some real address outside our scope.
+    let sent_result = if let Some(destination) = destination
+        .as_socket()
+        .filter(|destination| destination.port() != 53)
+    {
+        let rawish_true_destination = send_dns_patch(sockfd, user_socket_info, destination)?;
+
+        let mut true_message_header = Box::new(unsafe { *raw_message_header });
+
+        unsafe {
+            true_message_header.as_mut().msg_name.copy_from(
+                rawish_true_destination.as_ptr() as *const _,
+                rawish_true_destination.len() as usize,
+            )
+        };
+        true_message_header.as_mut().msg_namelen = rawish_true_destination.len();
+
+        unsafe { FN_SENDMSG(sockfd, true_message_header.as_ref(), flags) }
+    } else {
+        connect_outgoing::<UDP, false>(sockfd, destination, user_socket_info)?;
+
+        let layer_address: SockAddr = SOCKETS
+            .get(&sockfd)
+            .and_then(|socket| match &socket.state {
+                SocketState::Connected(connected) => connected.layer_address.clone(),
+                _ => unreachable!(),
+            })
+            .map(SocketAddress::try_into)??;
+
+        let raw_interceptor_address = layer_address.as_ptr() as *const _;
+        let raw_interceptor_length = layer_address.len();
+        let mut true_message_header = Box::new(unsafe { *raw_message_header });
+
+        unsafe {
+            true_message_header
+                .as_mut()
+                .msg_name
+                .copy_from(raw_interceptor_address, raw_interceptor_length as usize)
+        };
+        true_message_header.as_mut().msg_namelen = raw_interceptor_length;
+
+        unsafe { FN_SENDMSG(sockfd, true_message_header.as_ref(), flags) }
+    };
+
+    Detour::Success(sent_result)
+}