Use target.namespace for targetless runs (#3059)

* Updated config docs * Using target namespace in the operator client * Added check to LayerConfig::verify * Changelog * Fix targetless Job agent namespace * Schema and configuration.md update
metalbear-co · Feb 4, 2025 · c4e24a4 · c4e24a4
1 parent dcb8bb9
commit c4e24a4
Show file tree

Hide file tree

Showing 9 changed files with 142 additions and 96 deletions.
diff --git a/changelog.d/+targetless-namespace.changed.md b/changelog.d/+targetless-namespace.changed.md
@@ -0,0 +1,2 @@
+Namespace for `targetless` runs is now be specified with the `target.namespace` config field (or the `MIRRORD_TARGET_NAMESPACE` environment variable).
+`agent.namespace` field is ignored in targetless runs.
diff --git a/mirrord-schema.json b/mirrord-schema.json
@@ -250,7 +250,7 @@
       "additionalProperties": false
     },
     "AgentFileConfig": {
-      "description": "Configuration for the mirrord-agent pod that is spawned in the Kubernetes cluster.\n\nWe provide sane defaults for this option, so you don't have to set up anything here.\n\n```json { \"agent\": { \"log_level\": \"info\", \"json_log\": false, \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": false } } ```",
+      "description": "Configuration for the mirrord-agent pod that is spawned in the Kubernetes cluster.\n\n**Note:** this configuration is ignored when using the mirrord Operator. Agent configuration is done by the cluster admin.\n\nWe provide sane defaults for this option, so you don't have to set up anything here.\n\n```json { \"agent\": { \"log_level\": \"info\", \"json_log\": false, \"namespace\": \"default\", \"image\": \"ghcr.io/metalbear-co/mirrord:latest\", \"image_pull_policy\": \"IfNotPresent\", \"image_pull_secrets\": [ { \"secret-key\": \"secret\" } ], \"ttl\": 30, \"ephemeral\": false, \"communication_timeout\": 30, \"startup_timeout\": 360, \"network_interface\": \"eth0\", \"flush_connections\": false } } ```",
       "type": "object",
       "properties": {
         "annotations": {
@@ -306,7 +306,7 @@
         },
         "ephemeral": {
           "title": "agent.ephemeral {#agent-ephemeral}",
-          "description": "Runs the agent as an [ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/)\n\nDefaults to `false`.",
+          "description": "Runs the agent as an [ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/).\n\nNot compatible with targetless runs.\n\nDefaults to `false`.",
           "type": [
             "boolean",
             "null"
@@ -388,7 +388,7 @@
         },
         "namespace": {
           "title": "agent.namespace {#agent-namespace}",
-          "description": "Namespace where the agent shall live. Note: Doesn't work with ephemeral containers. Defaults to the current kubernetes namespace.",
+          "description": "Namespace where the agent shall live.\n\n**Note:** ignored in targetless runs or when the agent is run as an ephemeral container.\n\nDefaults to the current kubernetes namespace.",
           "type": [
             "string",
             "null"

diff --git a/mirrord/config/configuration.md b/mirrord/config/configuration.md
@@ -140,6 +140,9 @@ If not provided, mirrord will use value from the kubeconfig.
 
 Configuration for the mirrord-agent pod that is spawned in the Kubernetes cluster.
 
+**Note:** this configuration is ignored when using the mirrord Operator.
+Agent configuration is done by the cluster admin.
+
 We provide sane defaults for this option, so you don't have to set up anything here.
 
 ```json
@@ -204,7 +207,9 @@ as targetless agent containers have no capabilities.
 ### agent.ephemeral {#agent-ephemeral}
 
 Runs the agent as an
-[ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/)
+[ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/).
+
+Not compatible with targetless runs.
 
 Defaults to `false`.
 
@@ -323,7 +328,9 @@ configured to scrape for metrics.
 ### agent.namespace {#agent-namespace}
 
 Namespace where the agent shall live.
-Note: Doesn't work with ephemeral containers.
+
+**Note:** ignored in targetless runs or when the agent is run as an ephemeral container.
+
 Defaults to the current kubernetes namespace.
 
 ### agent.network_interface {#agent-network_interface}
@@ -1587,7 +1594,7 @@ Please note that:
 - `job`, `cronjob`, `statefulset` and `service` targets require the mirrord Operator
 - `job` and `cronjob` targets require the [`copy_target`](#feature-copy_target) feature
 
-Shortened setup:
+Shortened setup with a target:
 
 ```json
 {
@@ -1598,7 +1605,7 @@ Shortened setup:
 The setup above will result in a session targeting the `bear-pod` Kubernetes pod
 in the user's default namespace. A target container will be chosen by mirrord.
 
-Shortened setup with target container:
+Shortened setup with a target container:
 
 ```json
 {
@@ -1609,7 +1616,7 @@ Shortened setup with target container:
 The setup above will result in a session targeting the `bear-pod-container` container
 in the `bear-pod` Kubernetes pod in the user's default namespace.
 
-Complete setup:
+Complete setup with a target container:
 
 ```json
 {
@@ -1626,16 +1633,34 @@ Complete setup:
 The setup above will result in a session targeting the `bear-pod-container` container
 in the `bear-pod` Kubernetes pod in the `bear-pod-namespace` namespace.
 
+Setup with a namespace for a targetless run:
+
+```json
+{
+  "target": {
+    "path": "targetless",
+    "namespace": "bear-namespace"
+  }
+}
+```
+
+The setup above will result in a session without any target.
+Remote outgoing traffic and DNS will be done from the `bear-namespace` namespace.
+
 ### target.namespace {#target-namespace}
 
 Namespace where the target lives.
 
+For targetless runs, this the namespace in which remote networking is done.
+
 Defaults to the Kubernetes user's default namespace (defined in Kubernetes context).
 
 ### target.path {#target-path}
 
 Specifies the Kubernetes resource to target.
 
+If not given, defaults to `targetless`.
+
 Note: targeting services and whole workloads is available only in mirrord for Teams.
 If you target a workload without the mirrord Operator, it will choose a random pod replica
 to work with.
@@ -1652,7 +1677,7 @@ Supports:
 - `statefulset/{statefulset-name}[/container/{container-name}]`; (requires mirrord
   Operator)
 - `service/{service-name}[/container/{container-name}]`; (requires mirrord Operator)
-- `replicaset/{replicaset-name}[/container/{replicaset-name}]`; (requires mirrord Operator)
+- `replicaset/{replicaset-name}[/container/{container-name}]`; (requires mirrord Operator)
 
 ## telemetry {#root-telemetry}
 Controls whether or not mirrord sends telemetry data to MetalBear cloud.

diff --git a/mirrord/config/src/agent.rs b/mirrord/config/src/agent.rs
@@ -47,6 +47,9 @@ impl fmt::Display for LinuxCapability {
 
 /// Configuration for the mirrord-agent pod that is spawned in the Kubernetes cluster.
 ///
+/// **Note:** this configuration is ignored when using the mirrord Operator.
+/// Agent configuration is done by the cluster admin.
+///
 /// We provide sane defaults for this option, so you don't have to set up anything here.
 ///
 /// ```json
@@ -106,7 +109,9 @@ pub struct AgentConfig {
     /// ### agent.namespace {#agent-namespace}
     ///
     /// Namespace where the agent shall live.
-    /// Note: Doesn't work with ephemeral containers.
+    ///
+    /// **Note:** ignored in targetless runs or when the agent is run as an ephemeral container.
+    ///
     /// Defaults to the current kubernetes namespace.
     #[config(env = "MIRRORD_AGENT_NAMESPACE")]
     pub namespace: Option<String>,
@@ -183,7 +188,9 @@ pub struct AgentConfig {
     /// ### agent.ephemeral {#agent-ephemeral}
     ///
     /// Runs the agent as an
-    /// [ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/)
+    /// [ephemeral container](https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/).
+    ///
+    /// Not compatible with targetless runs.
     ///
     /// Defaults to `false`.
     #[config(env = "MIRRORD_EPHEMERAL_CONTAINER", default = false)]

diff --git a/mirrord/config/src/lib.rs b/mirrord/config/src/lib.rs
@@ -500,14 +500,13 @@ impl LayerConfig {
             Err(ConfigError::TargetJobWithoutCopyTarget)?
         }
 
-        if self.target.path.is_none() && !context.ide {
-            // In the IDE, a target may be selected after `mirrord verify-config` is run, so we
-            // for this case we treat these as warnings. They'll become errors once mirrord proper
-            // tries to start (if the user somehow managed to not select a target by then).
-            if self.target.namespace.is_some() {
-                Err(ConfigError::TargetNamespaceWithoutTarget)?
-            }
+        let is_targetless = match self.target.path.as_ref() {
+            Some(Target::Targetless) => true,
+            None => !context.ide,
+            _ => false,
+        };
 
+        if is_targetless {
             if self.feature.network.incoming.is_steal() {
                 Err(ConfigError::Conflict("Steal mode is not compatible with a targetless agent, please either disable this option or specify a target.".into()))?
             }
@@ -520,6 +519,10 @@ impl LayerConfig {
                         .into(),
                 ))?
             }
+
+            if self.agent.namespace.is_some() {
+                context.add_warning("Agent namespace is ignored in targetless runs".into());
+            }
         }
 
         if self.feature.copy_target.enabled {
@@ -532,7 +535,7 @@ impl LayerConfig {
             }
 
             // Target may also be set later in the UI.
-            if self.target.path.is_none() && !context.ide {
+            if is_targetless {
                 return Err(ConfigError::Conflict(
                     "The copy target feature is not compatible with a targetless agent, \
                     please either disable this option or specify a target."

diff --git a/mirrord/config/src/target.rs b/mirrord/config/src/target.rs
@@ -87,7 +87,7 @@ fn make_simple_target_custom_schema(gen: &mut SchemaGenerator) -> schemars::sche
 /// - `job`, `cronjob`, `statefulset` and `service` targets require the mirrord Operator
 /// - `job` and `cronjob` targets require the [`copy_target`](#feature-copy_target) feature
 ///
-/// Shortened setup:
+/// Shortened setup with a target:
 ///
 ///```json
 /// {
@@ -98,7 +98,7 @@ fn make_simple_target_custom_schema(gen: &mut SchemaGenerator) -> schemars::sche
 /// The setup above will result in a session targeting the `bear-pod` Kubernetes pod
 /// in the user's default namespace. A target container will be chosen by mirrord.
 ///
-/// Shortened setup with target container:
+/// Shortened setup with a target container:
 ///
 /// ```json
 /// {
@@ -109,7 +109,7 @@ fn make_simple_target_custom_schema(gen: &mut SchemaGenerator) -> schemars::sche
 /// The setup above will result in a session targeting the `bear-pod-container` container
 /// in the `bear-pod` Kubernetes pod in the user's default namespace.
 ///
-/// Complete setup:
+/// Complete setup with a target container:
 ///
 /// ```json
 /// {
@@ -125,13 +125,29 @@ fn make_simple_target_custom_schema(gen: &mut SchemaGenerator) -> schemars::sche
 ///
 /// The setup above will result in a session targeting the `bear-pod-container` container
 /// in the `bear-pod` Kubernetes pod in the `bear-pod-namespace` namespace.
+///
+/// Setup with a namespace for a targetless run:
+///
+/// ```json
+/// {
+///   "target": {
+///     "path": "targetless",
+///     "namespace": "bear-namespace"
+///   }
+/// }
+/// ```
+///
+/// The setup above will result in a session without any target.
+/// Remote outgoing traffic and DNS will be done from the `bear-namespace` namespace.
 #[derive(Serialize, Deserialize, Clone, Eq, PartialEq, Hash, Debug)]
 #[serde(deny_unknown_fields)]
 pub struct TargetConfig {
     /// ### target.path {#target-path}
     ///
     /// Specifies the Kubernetes resource to target.
     ///
+    /// If not given, defaults to `targetless`.
+    ///
     /// Note: targeting services and whole workloads is available only in mirrord for Teams.
     /// If you target a workload without the mirrord Operator, it will choose a random pod replica
     /// to work with.
@@ -156,6 +172,8 @@ pub struct TargetConfig {
     ///
     /// Namespace where the target lives.
     ///
+    /// For targetless runs, this the namespace in which remote networking is done.
+    ///
     /// Defaults to the Kubernetes user's default namespace (defined in Kubernetes context).
     #[serde(skip_serializing_if = "Option::is_none")]
     pub namespace: Option<String>,
@@ -322,23 +340,6 @@ impl FromStr for Target {
 }
 
 impl Target {
-    /// Get the target name - pod name, deployment name, rollout name..
-    pub fn get_target_name(&self) -> String {
-        match self {
-            Target::Deployment(target) => target.deployment.clone(),
-            Target::Pod(target) => target.pod.clone(),
-            Target::Rollout(target) => target.rollout.clone(),
-            Target::Job(target) => target.job.clone(),
-            Target::CronJob(target) => target.cron_job.clone(),
-            Target::StatefulSet(target) => target.stateful_set.clone(),
-            Target::Service(target) => target.service.clone(),
-            Target::ReplicaSet(target) => target.replica_set.clone(),
-            Target::Targetless => {
-                unreachable!("this shouldn't happen - called from operator on a flow where it's not targetless.")
-            }
-        }
-    }
-
     /// `true` if this [`Target`] is only supported when the copy target feature is enabled.
     pub(super) fn requires_copy(&self) -> bool {
         matches!(self, Target::Job(_) | Target::CronJob(_))

diff --git a/mirrord/kube/src/api/kubernetes.rs b/mirrord/kube/src/api/kubernetes.rs
@@ -41,6 +41,10 @@ pub struct KubernetesAPI {
 }
 
 impl KubernetesAPI {
+    /// Creates a new instance from the given [`LayerConfig`].
+    ///
+    /// If [`LayerConfig::target`] specifies a targetless run,
+    /// replaces [`AgentConfig::namespace`] with the target namespace.
     pub async fn create(config: &LayerConfig) -> Result<Self> {
         let client = create_kube_config(
             config.accept_invalid_certificates,
@@ -50,7 +54,17 @@ impl KubernetesAPI {
         .await?
         .try_into()?;
 
-        Ok(KubernetesAPI::new(client, config.agent.clone()))
+        let mut agent = config.agent.clone();
+        if config
+            .target
+            .path
+            .as_ref()
+            .is_none_or(|path| matches!(path, Target::Targetless))
+        {
+            agent.namespace = config.target.namespace.clone();
+        }
+
+        Ok(KubernetesAPI::new(client, agent))
     }
 
     pub fn new(client: Client, agent: AgentConfig) -> Self {

diff --git a/mirrord/kube/src/resolved.rs b/mirrord/kube/src/resolved.rs
@@ -6,7 +6,7 @@ use k8s_openapi::api::{
     core::v1::{Pod, Service},
 };
 use kube::{Client, Resource, ResourceExt};
-use mirrord_config::{feature::network::incoming::ConcurrentSteal, target::Target};
+use mirrord_config::target::Target;
 use tracing::Level;
 
 use super::{
@@ -532,37 +532,3 @@ impl ResolvedTarget<false> {
         }
     }
 }
-
-impl ResolvedTarget<true> {
-    pub fn connect_url(
-        &self,
-        use_proxy: bool,
-        concurrent_steal: ConcurrentSteal,
-        api_version: &str,
-        plural: &str,
-        url_path: &str,
-    ) -> String {
-        let name = self.urlfied_name();
-        let namespace = self.namespace().unwrap_or("default");
-
-        if use_proxy {
-            format!("/apis/{api_version}/proxy/namespaces/{namespace}/{plural}/{name}?on_concurrent_steal={concurrent_steal}&connect=true")
-        } else {
-            format!("{url_path}/{name}?on_concurrent_steal={concurrent_steal}&connect=true")
-        }
-    }
-
-    pub fn urlfied_name(&self) -> String {
-        let mut url = self.type_().to_string();
-
-        if let Some(target_name) = self.name() {
-            url.push_str(&format!(".{target_name}"));
-        }
-
-        if let Some(container) = self.container() {
-            url.push_str(&format!(".container.{container}"));
-        }
-
-        url
-    }
-}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		Namespace for `targetless` runs is now be specified with the `target.namespace` config field (or the `MIRRORD_TARGET_NAMESPACE` environment variable).
		`agent.namespace` field is ignored in targetless runs.