merge main

CHANGELOG.md

@@ -34,8 +34,6 @@ This project uses [*towncrier*](https://towncrier.readthedocs.io/) and the chang
   Improved handling of incoming connections on the local machine (e.g
   introduces reuse of local HTTP connections).
   [#3013](https://github.com/metalbear-co/mirrord/issues/3013)
-- Moved to an older builder base image for aarch64 to support centos-7 libc.
-  [#3024](https://github.com/metalbear-co/mirrord/issues/3024)
 
 
 ### Internal

CONTRIBUTING.md

@@ -131,6 +131,21 @@ In order to test IPv6 on a local cluster on macOS, you can use Kind:
 3. `kind create cluster --config kind-config.yaml`
 4. When you run `kubectl get svc -o wide --all-namespaces` you should see IPv6 addresses.
 
+In order to use an agent image from a local registry, you can load the image to kind's registry with:
+
+```
+kind load docker-image test:latest
+```
+
+In order to test on EKS, I used this blueprint: https://github.com/aws-ia/terraform-aws-eks-blueprints/tree/main/patterns/ipv6-eks-cluster
+
+After creating the cluster, I had to give myself permissions to the K8s objects, I did that via the AWS console (in the browser).
+Feel free to add instructions on how to make that "manual" step unnecessary.
+
+IPv6 tests (they currently don't run in the CI):
+- steal_http_ipv6_traffic
+- connect_to_kubernetes_api_service_over_ipv6
+
 
 ### Cleanup
 

Cross.toml

@@ -5,7 +5,7 @@ passthrough = [
 # Dockerfile used for building mirrord-layer for x64 with very old libc
 # this to support centos7 or Amazon Linux 2.
 [target.x86_64-unknown-linux-gnu]
-image = "ghcr.io/metalbear-co/ci-layer-build:8ca4a4e9757a5749c384c57c3435e56c2b442458e14e4cb7ecbaec4d557f5d69"
+image = "ghcr.io/metalbear-co/ci-layer-build:latest"
 
 [target.aarch64-unknown-linux-gnu]
-image = "ghcr.io/metalbear-co/ci-layer-build-aarch64:4c99d799d297ddec935914907b94d899bd0a2349155cec934492ef19a69ddbf0"
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main"

changelog.d/2958.added.md

@@ -0,0 +1 @@
+Support for in-cluster DNS resolution of IPv6 addresses.

mirrord/agent/src/dns.rs

@@ -3,7 +3,7 @@ use std::{future, path::PathBuf, time::Duration};
 use futures::{stream::FuturesOrdered, StreamExt};
 use hickory_resolver::{system_conf::parse_resolv_conf, Hosts, Resolver};
 use mirrord_protocol::{
-    dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoResponse},
+    dns::{DnsLookup, GetAddrInfoRequest, GetAddrInfoRequestV2, GetAddrInfoResponse},
     DnsLookupError, RemoteResult, ResolveErrorKindInternal, ResponseError,
 };
 use tokio::{
@@ -18,9 +18,24 @@ use tracing::Level;
 
 use crate::{error::AgentResult, metrics::DNS_REQUEST_COUNT, watched_task::TaskStatus};
 
+#[derive(Debug)]
+pub(crate) enum ClientGetAddrInfoRequest {
+    V1(GetAddrInfoRequest),
+    V2(GetAddrInfoRequestV2),
+}
+
+impl ClientGetAddrInfoRequest {
+    pub(crate) fn into_v2(self) -> GetAddrInfoRequestV2 {
+        match self {
+            ClientGetAddrInfoRequest::V1(old_req) => old_req.into(),
+            ClientGetAddrInfoRequest::V2(v2_req) => v2_req,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct DnsCommand {
-    request: GetAddrInfoRequest,
+    request: ClientGetAddrInfoRequest,
     response_tx: oneshot::Sender<RemoteResult<DnsLookup>>,
 }
 
@@ -31,6 +46,7 @@ pub(crate) struct DnsWorker {
     request_rx: Receiver<DnsCommand>,
     attempts: usize,
     timeout: Duration,
+    support_ipv6: bool,
 }
 
 impl DnsWorker {
@@ -42,7 +58,11 @@ impl DnsWorker {
     /// # Note
     ///
     /// `pid` is used to find the correct path of `etc` directory.
-    pub(crate) fn new(pid: Option<u64>, request_rx: Receiver<DnsCommand>) -> Self {
+    pub(crate) fn new(
+        pid: Option<u64>,
+        request_rx: Receiver<DnsCommand>,
+        support_ipv6: bool,
+    ) -> Self {
         let etc_path = pid
             .map(|pid| {
                 PathBuf::from("/proc")
@@ -63,6 +83,7 @@ impl DnsWorker {
                 .ok()
                 .and_then(|attempts| attempts.parse().ok())
                 .unwrap_or(1),
+            support_ipv6,
         }
     }
 
@@ -76,9 +97,10 @@ impl DnsWorker {
     #[tracing::instrument(level = Level::TRACE, ret, err(level = Level::TRACE))]
     async fn do_lookup(
         etc_path: PathBuf,
-        host: String,
+        request: GetAddrInfoRequestV2,
         attempts: usize,
         timeout: Duration,
+        support_ipv6: bool,
     ) -> RemoteResult<DnsLookup> {
         // Prepares the `Resolver` after reading some `/etc` DNS files.
         //
@@ -91,13 +113,32 @@ impl DnsWorker {
             let hosts_conf = fs::read(hosts_path).await?;
 
             let (config, mut options) = parse_resolv_conf(resolv_conf)?;
+            tracing::debug!(?config, ?options, "parsed config options");
             options.server_ordering_strategy =
                 hickory_resolver::config::ServerOrderingStrategy::UserProvidedOrder;
             options.timeout = timeout;
             options.attempts = attempts;
-            options.ip_strategy = hickory_resolver::config::LookupIpStrategy::Ipv4Only;
+            options.ip_strategy = if support_ipv6 {
+                tracing::debug!("IPv6 support enabled. Respecting client IP family.");
+                request
+                    .family
+                    .try_into()
+                    .inspect_err(|e| {
+                        tracing::error!(%e,
+                        "Unknown address family in addrinfo request. Using IPv4 and IPv6.")
+                    })
+                    // If the agent gets some new, unknown variant of family address, it's the
+                    // client's fault, so the agent queries both IPv4 and IPv6 and if that's not
+                    // good enough for the client, the client can error out.
+                    .unwrap_or(hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6)
+            } else {
+                tracing::debug!("IPv6 support disabled. Resolving IPv4 only.");
+                hickory_resolver::config::LookupIpStrategy::Ipv4Only
+            };
+            tracing::debug!(?config, ?options, "updated config options");
 
             let mut resolver = Resolver::tokio(config, options);
+            tracing::debug!(?resolver, "tokio resolver");
 
             let mut hosts = Hosts::default();
             hosts.read_hosts_conf(hosts_conf.as_slice())?;
@@ -108,9 +149,10 @@ impl DnsWorker {
 
         let lookup = resolver
             .inspect_err(|fail| tracing::error!(?fail, "Failed to build DNS resolver"))?
-            .lookup_ip(host)
+            .lookup_ip(request.node)
             .await
-            .inspect(|lookup| tracing::trace!(?lookup, "Lookup finished"))?
+            .inspect(|lookup| tracing::trace!(?lookup, "Lookup finished"))
+            .inspect_err(|e| tracing::trace!(%e, "lookup failed"))?
             .into();
 
         Ok(lookup)
@@ -125,8 +167,16 @@ impl DnsWorker {
 
         DNS_REQUEST_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 
+        let support_ipv6 = self.support_ipv6;
         let lookup_future = async move {
-            let result = Self::do_lookup(etc_path, message.request.node, attempts, timeout).await;
+            let result = Self::do_lookup(
+                etc_path,
+                message.request.into_v2(),
+                attempts,
+                timeout,
+                support_ipv6,
+            )
+            .await;
 
             if let Err(result) = message.response_tx.send(result) {
                 tracing::error!(?result, "Failed to send query response");
@@ -170,7 +220,10 @@ impl DnsApi {
 
     /// Schedules a new DNS request.
     /// Results of scheduled requests are available via [`Self::recv`] (order is preserved).
-    pub(crate) async fn make_request(&mut self, request: GetAddrInfoRequest) -> AgentResult<()> {
+    pub(crate) async fn make_request(
+        &mut self,
+        request: ClientGetAddrInfoRequest,
+    ) -> AgentResult<()> {
         let (response_tx, response_rx) = oneshot::channel();
 
         let command = DnsCommand {

mirrord/agent/src/entrypoint.rs

@@ -10,7 +10,7 @@ use std::{
 };
 
 use client_connection::AgentTlsConnector;
-use dns::{DnsCommand, DnsWorker};
+use dns::{ClientGetAddrInfoRequest, DnsCommand, DnsWorker};
 use futures::TryFutureExt;
 use metrics::{start_metrics, CLIENT_COUNT};
 use mirrord_protocol::{ClientMessage, DaemonMessage, GetEnvVarsRequest, LogMessage};
@@ -442,7 +442,14 @@ impl ClientConnectionHandler {
                     .await?
             }
             ClientMessage::GetAddrInfoRequest(request) => {
-                self.dns_api.make_request(request).await?;
+                self.dns_api
+                    .make_request(ClientGetAddrInfoRequest::V1(request))
+                    .await?;
+            }
+            ClientMessage::GetAddrInfoRequestV2(request) => {
+                self.dns_api
+                    .make_request(ClientGetAddrInfoRequest::V2(request))
+                    .await?;
             }
             ClientMessage::Ping => self.respond(DaemonMessage::Pong).await?,
             ClientMessage::Tcp(message) => {
@@ -634,7 +641,8 @@ async fn start_agent(args: Args) -> AgentResult<()> {
         let cancellation_token = cancellation_token.clone();
         let watched_task = WatchedTask::new(
             DnsWorker::TASK_NAME,
-            DnsWorker::new(state.container_pid(), dns_command_rx).run(cancellation_token),
+            DnsWorker::new(state.container_pid(), dns_command_rx, args.ipv6)
+                .run(cancellation_token),
         );
         let status = watched_task.status();
         let task = run_thread_in_namespace(

mirrord/config/src/feature/network.rs

@@ -10,7 +10,7 @@ use crate::{
     util::MirrordToggleableConfig,
 };
 
-const IPV6_ENV_VAR: &str = "MIRRORD_INCOMING_ENABLE_IPV6";
+const IPV6_ENV_VAR: &str = "MIRRORD_ENABLE_IPV6";
 
 pub mod dns;
 pub mod filter;

mirrord/intproxy/protocol/src/lib.rs

@@ -10,7 +10,7 @@ use std::{
 
 use bincode::{Decode, Encode};
 use mirrord_protocol::{
-    dns::{GetAddrInfoRequest, GetAddrInfoResponse},
+    dns::{GetAddrInfoRequestV2, GetAddrInfoResponse},
     file::*,
     outgoing::SocketAddress,
     tcp::StealType,
@@ -44,7 +44,7 @@ pub enum LayerToProxyMessage {
     /// A file operation request.
     File(FileRequest),
     /// A DNS request.
-    GetAddrInfo(GetAddrInfoRequest),
+    GetAddrInfo(GetAddrInfoRequestV2),
     /// A request to initiate a new outgoing connection.
     OutgoingConnect(OutgoingConnectRequest),
     /// Requests related to incoming connections.
@@ -210,7 +210,7 @@ pub enum ProxyToLayerMessage {
     NewSession(LayerId),
     /// A response to layer's [`FileRequest`].
     File(FileResponse),
-    /// A response to layer's [`GetAddrInfoRequest`].
+    /// A response to layer's [`GetAddrInfoRequestV2`].
     GetAddrInfo(GetAddrInfoResponse),
     /// A response to layer's [`OutgoingConnectRequest`].
     OutgoingConnect(RemoteResult<OutgoingConnectResponse>),
@@ -428,7 +428,7 @@ impl_request!(
 );
 
 impl_request!(
-    req = GetAddrInfoRequest,
+    req = GetAddrInfoRequestV2,
     res = GetAddrInfoResponse,
     req_path = LayerToProxyMessage::GetAddrInfo,
     res_path = ProxyToLayerMessage::GetAddrInfo,