Merge branch 'main' of github.com:DataDog/test-infra-definitions into…

… lenaic/revert_revert

.github/dependabot.yml

@@ -15,6 +15,13 @@ updates:
     directory: /
     schedule:
       interval: daily
+    groups:
+      aws-sdk-go-v2:
+        patterns:
+          - 'github.com/aws/aws-sdk-go-v2*'
+      pulumi-azure-native-sdk:
+        patterns:
+          - 'github.com/pulumi/pulumi-azure-native-sdk*'
 
   - package-ecosystem: pip
     directory: /
@@ -85,3 +92,5 @@ updates:
     directory: /components/datadog/apps/cws/images/cws-centos7
     schedule:
       interval: daily
+
+

.gitlab-ci.yml

@@ -104,10 +104,19 @@ bump-version-on-datadog-agent:
   stage: post-release
   image: ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA}
   tags: ["arch:amd64"]
-  needs: ["release-runner-image"]
   rules:
     - if: $CI_COMMIT_BRANCH == "main"
+      needs: ["release-runner-image"]
       when: on_success
+    - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
+      when: never
+    - when: manual
+      needs: ["build-runner-image"]
+      allow_failure: true
+      variables:
+        EXTRA_UPDATE_ARGS: "--is-dev-image"
+  variables:
+    EXTRA_UPDATE_ARGS: ""
   before_script:
     - set +x
     - export GITHUB_APP_USER_ID=153269286 # Can be found on https://api.github.com/users/agent-platform-auto-pr[bot]
@@ -126,8 +135,9 @@ bump-version-on-datadog-agent:
     - pushd datadog-agent
     - git checkout -b auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA
     - export PREVIOUS_SHA=$(cat .gitlab/common/test_infra_version.yml | grep 'TEST_INFRA_DEFINITIONS_BUILDIMAGES:' | awk -F " " '{print $NF}')
-    - inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA
-    - git add test/new-e2e/go.mod test/new-e2e/go.sum .gitlab/common/test_infra_version.yml
+    - inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA $EXTRA_UPDATE_ARGS
+    - inv -e tidy
+    - git add -u
     - git commit -m "[test-infra-definitions][automated] Bump test-infra-definitions to $CI_COMMIT_SHORT_SHA"
     - git push -f origin auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA
     - popd

components/datadog/agent/helm/kubernetes_agent.go

@@ -1,6 +1,7 @@
 package helm
 
 import (
+	componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes"
 	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 
@@ -43,23 +44,23 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne
 
 		baseName := "dda-" + platform
 
-		comp.LinuxNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
+		comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
 			"app": baseName + "-datadog",
 		})
 
 		if err != nil {
 			return err
 		}
 
-		comp.LinuxClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
+		comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
 			"app": baseName + "-datadog-cluster-agent",
 		})
 
 		if err != nil {
 			return err
 		}
 
-		comp.LinuxClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{
+		comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{
 			"app": baseName + "-datadog-clusterchecks",
 		})
 
@@ -70,21 +71,21 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne
 
 			baseName = "dda-" + platform
 
-			comp.WindowsNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
+			comp.WindowsNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
 				"app": baseName + "-datadog",
 			})
 			if err != nil {
 				return err
 			}
 
-			comp.WindowsClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
+			comp.WindowsClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{
 				"app": baseName + "-datadog-cluster-agent",
 			})
 			if err != nil {
 				return err
 			}
 
-			comp.WindowsClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{
+			comp.WindowsClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{
 				"app": baseName + "-datadog-clusterchecks",
 			})
 			if err != nil {

components/datadog/agent/kubernetes.go

@@ -1,6 +1,7 @@
 package agent
 
 import (
+	"github.com/DataDog/test-infra-definitions/components/kubernetes"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 
 	"github.com/DataDog/test-infra-definitions/components"
@@ -9,27 +10,27 @@ import (
 type KubernetesAgentOutput struct {
 	components.JSONImporter
 
-	LinuxNodeAgent     KubernetesObjRefOutput `json:"linuxNodeAgent"`
-	LinuxClusterAgent  KubernetesObjRefOutput `json:"linuxClusterAgent"`
-	LinuxClusterChecks KubernetesObjRefOutput `json:"linuxClusterChecks"`
+	LinuxNodeAgent     kubernetes.KubernetesObjRefOutput `json:"linuxNodeAgent"`
+	LinuxClusterAgent  kubernetes.KubernetesObjRefOutput `json:"linuxClusterAgent"`
+	LinuxClusterChecks kubernetes.KubernetesObjRefOutput `json:"linuxClusterChecks"`
 
-	WindowsNodeAgent     KubernetesObjRefOutput `json:"windowsNodeAgent"`
-	WindowsClusterAgent  KubernetesObjRefOutput `json:"windowsClusterAgent"`
-	WindowsClusterChecks KubernetesObjRefOutput `json:"windowsClusterChecks"`
+	WindowsNodeAgent     kubernetes.KubernetesObjRefOutput `json:"windowsNodeAgent"`
+	WindowsClusterAgent  kubernetes.KubernetesObjRefOutput `json:"windowsClusterAgent"`
+	WindowsClusterChecks kubernetes.KubernetesObjRefOutput `json:"windowsClusterChecks"`
 }
 
 // KubernetesAgent is an installer to install the Datadog Agent on a Kubernetes cluster.
 type KubernetesAgent struct {
 	pulumi.ResourceState
 	components.Component
 
-	LinuxNodeAgent     *KubernetesObjectRef `pulumi:"linuxNodeAgent"`
-	LinuxClusterAgent  *KubernetesObjectRef `pulumi:"linuxClusterAgent"`
-	LinuxClusterChecks *KubernetesObjectRef `pulumi:"linuxClusterChecks"`
+	LinuxNodeAgent     *kubernetes.KubernetesObjectRef `pulumi:"linuxNodeAgent"`
+	LinuxClusterAgent  *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterAgent"`
+	LinuxClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterChecks"`
 
-	WindowsNodeAgent     *KubernetesObjectRef `pulumi:"windowsNodeAgent"`
-	WindowsClusterAgent  *KubernetesObjectRef `pulumi:"windowsClusterAgent"`
-	WindowsClusterChecks *KubernetesObjectRef `pulumi:"windowsClusterChecks"`
+	WindowsNodeAgent     *kubernetes.KubernetesObjectRef `pulumi:"windowsNodeAgent"`
+	WindowsClusterAgent  *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterAgent"`
+	WindowsClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterChecks"`
 }
 
 func (h *KubernetesAgent) Export(ctx *pulumi.Context, out *KubernetesAgentOutput) error {

components/datadog/agent/kubernetes_helm.go

@@ -223,12 +223,23 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI
 			"namespaceLabelsAsTags": pulumi.Map{
 				"related_team": pulumi.String("team"),
 			},
-			"originDetectionUnified": pulumi.Map{
-				"enabled": pulumi.Bool(true),
-			},
 			"namespaceAnnotationsAsTags": pulumi.Map{
 				"related_email": pulumi.String("email"), // should be overridden by kubernetesResourcesAnnotationsAsTags
 			},
+			"kubernetesResourcesAnnotationsAsTags": pulumi.Map{
+				"deployments.apps": pulumi.Map{"x-sub-team": pulumi.String("sub-team")},
+				"pods":             pulumi.Map{"x-parent-name": pulumi.String("parent-name")},
+				"namespaces":       pulumi.Map{"related_email": pulumi.String("mail")},
+			},
+			"kubernetesResourcesLabelsAsTags": pulumi.Map{
+				"deployments.apps": pulumi.Map{"x-team": pulumi.String("team")},
+				"pods":             pulumi.Map{"x-parent-type": pulumi.String("domain")},
+				"namespaces":       pulumi.Map{"related_org": pulumi.String("org")},
+				"nodes":            pulumi.Map{"kubernetes.io/os": pulumi.String("os"), "kubernetes.io/arch": pulumi.String("arch")},
+			},
+			"originDetectionUnified": pulumi.Map{
+				"enabled": pulumi.Bool(true),
+			},
 			"logs": pulumi.Map{
 				"enabled":             pulumi.Bool(true),
 				"containerCollectAll": pulumi.Bool(logsContainerCollectAll),
@@ -304,14 +315,6 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI
 					"name":  pulumi.String("DD_TELEMETRY_CHECKS"),
 					"value": pulumi.String("*"),
 				},
-				pulumi.StringMap{
-					"name":  pulumi.String("DD_KUBERNETES_RESOURCES_LABELS_AS_TAGS"),
-					"value": pulumi.JSONMarshal(getResourcesLabelsAsTags().toJSONString()),
-				},
-				pulumi.StringMap{
-					"name":  pulumi.String("DD_KUBERNETES_RESOURCES_ANNOTATIONS_AS_TAGS"),
-					"value": pulumi.JSONMarshal(getResourcesAnnotationsAsTags().toJSONString()),
-				},
 			},
 		},
 		"agents": pulumi.Map{
@@ -457,6 +460,12 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI
 				"tag":           pulumi.String(agentImageTag),
 				"doNotCheckTag": pulumi.Bool(true),
 			},
+			"env": pulumi.StringMapArray{
+				pulumi.StringMap{
+					"name":  pulumi.String("DD_CLC_RUNNER_REMOTE_TAGGER_ENABLED"),
+					"value": pulumi.String("true"),
+				},
+			},
 			"resources": pulumi.StringMapMap{
 				"requests": pulumi.StringMap{
 					"cpu":    pulumi.String("20m"),

components/datadog/agent/kubernetes_operator.go

@@ -1,62 +1,48 @@
 package agent
 
 import (
-	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
-	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
-
 	"github.com/DataDog/test-infra-definitions/common/config"
-	"github.com/DataDog/test-infra-definitions/common/utils"
 	"github.com/DataDog/test-infra-definitions/components"
 	"github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams"
 	"github.com/DataDog/test-infra-definitions/components/datadog/apps/dda"
-	"github.com/DataDog/test-infra-definitions/components/datadog/operator"
-	"github.com/DataDog/test-infra-definitions/components/datadog/operatorparams"
+	componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes"
+	"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )
 
-func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, operatorOpts []operatorparams.Option, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) {
+func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) {
 	return components.NewComponent(e, resourceName, func(comp *KubernetesAgent) error {
-
-		operatorParams, err := operatorparams.NewParams(e, operatorOpts...)
-		if err != nil {
-			return err
-		}
-
 		ddaParams, err := agentwithoperatorparams.NewParams(ddaOptions...)
-
 		if err != nil {
 			return err
 		}
 
-		operatorComp, err := operator.NewOperator(e, resourceName, kubeProvider, operatorOpts...)
-
-		if err != nil {
-			return err
-		}
+		ddaParams.PulumiResourceOptions = append(ddaParams.PulumiResourceOptions, pulumi.Parent(comp))
 
-		_, err = dda.K8sAppDefinition(e, kubeProvider, "datadog", ddaParams.FakeIntake, ddaParams.KubeletTLSVerify, e.Ctx().Stack(), ddaParams.DDAConfig, utils.PulumiDependsOn(operatorComp))
+		_, err = dda.K8sAppDefinition(e, kubeProvider, ddaParams, ddaParams.PulumiResourceOptions...)
 
 		if err != nil {
 			return err
 		}
 
-		baseName := "dda-linux"
+		baseName := "dda-with-operator-linux"
 
-		comp.LinuxNodeAgent, err = NewKubernetesObjRef(e, baseName+"-nodeAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{"app": baseName + "-datadog"})
+		comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{"app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-agent"})
 
 		if err != nil {
 			return err
 		}
 
-		comp.LinuxClusterAgent, err = NewKubernetesObjRef(e, baseName+"-clusterAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{
-			"app": baseName + "-datadog-cluster-agent",
+		comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{
+			"app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-agent",
 		})
 
 		if err != nil {
 			return err
 		}
 
-		comp.LinuxClusterChecks, err = NewKubernetesObjRef(e, baseName+"-clusterChecks", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("version").ToStringOutput(), map[string]string{
-			"app": baseName + "-datadog-clusterchecks",
+		comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{
+			"app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-checks-runner",
 		})
 
 		if err != nil {

components/datadog/agentwithoperatorparams/params.go

@@ -11,17 +11,19 @@ import (
 type Params struct {
 	PulumiResourceOptions []pulumi.ResourceOption
 
-	Namespace        string
-	FakeIntake       *fakeintake.Fakeintake
-	DDAConfig        string
-	KubeletTLSVerify bool
+	Namespace  string
+	FakeIntake *fakeintake.Fakeintake
+	DDAConfig  DDAConfig
 }
 
 type Option = func(*Params) error
 
 func NewParams(options ...Option) (*Params, error) {
 	version := &Params{
 		Namespace: "datadog",
+		DDAConfig: DDAConfig{
+			Name: "dda",
+		},
 	}
 	return common.ApplyOption(version, options)
 }
@@ -34,14 +36,6 @@ func WithNamespace(namespace string) func(*Params) error {
 	}
 }
 
-// WithTLSKubeletVerify toggles kubelet TLS verification.
-func WithTLSKubeletVerify(verify bool) func(*Params) error {
-	return func(p *Params) error {
-		p.KubeletTLSVerify = verify
-		return nil
-	}
-}
-
 // WithPulumiResourceOptions sets the resources to depend on.
 func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) error {
 	return func(p *Params) error {
@@ -50,10 +44,10 @@ func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params)
 	}
 }
 
-// WithDDAConfig configures the DatadogAgent resource.
-func WithDDAConfig(config string) func(*Params) error {
+// WithDDAConfig configures the DatadogAgent custom resource.
+func WithDDAConfig(config DDAConfig) func(*Params) error {
 	return func(p *Params) error {
-		p.DDAConfig = p.DDAConfig + config
+		p.DDAConfig = config
 		return nil
 	}
 }
@@ -66,3 +60,15 @@ func WithFakeIntake(fakeintake *fakeintake.Fakeintake) func(*Params) error {
 		return nil
 	}
 }
+
+// DDAConfig is the DatadogAgent custom resource configuration.
+type DDAConfig struct {
+	// Name of the DatadogAgent custom resource
+	Name string `json:"name"`
+	// YamlFilePath file path to the DatadogAgent custom resource YAML
+	YamlFilePath string `json:"yamlFilePath,omitempty"`
+	// YamlConfig is the YAML string of the DatadogAgent custom resource
+	YamlConfig string `json:"YamlConfig,omitempty"`
+	// MapConfig is the map representation of the DatadogAgent custom resource
+	MapConfig map[string]interface{} `json:"MapConfig,omitempty"`
+}