From 1c333ed90674178de0564a34a599b26357ad1b5d Mon Sep 17 00:00:00 2001 From: Fanny Jiang Date: Thu, 12 Dec 2024 10:23:24 -0500 Subject: [PATCH] DDA w/operator component cleanups and improvements (#1251) --- .../datadog/agent/helm/kubernetes_agent.go | 13 +- components/datadog/agent/kubernetes.go | 25 +- .../datadog/agent/kubernetes_operator.go | 36 +- .../datadog/agentwithoperatorparams/params.go | 36 +- components/datadog/apps/dda/datadogagent.go | 406 ++++++++++-------- components/datadog/operator/component.go | 27 ++ components/datadog/operator/helm.go | 2 +- components/datadog/operator/operator.go | 24 +- components/datadog/operatorparams/params.go | 21 +- .../kubernetes_object_ref.go | 6 +- scenarios/aws/kindvm/run.go | 25 +- 11 files changed, 351 insertions(+), 270 deletions(-) create mode 100644 components/datadog/operator/component.go rename components/{datadog/agent => kubernetes}/kubernetes_object_ref.go (88%) diff --git a/components/datadog/agent/helm/kubernetes_agent.go b/components/datadog/agent/helm/kubernetes_agent.go index b56a621cc..684f5c6a4 100644 --- a/components/datadog/agent/helm/kubernetes_agent.go +++ b/components/datadog/agent/helm/kubernetes_agent.go @@ -1,6 +1,7 @@ package helm import ( + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -43,7 +44,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne baseName := "dda-" + platform - comp.LinuxNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog", }) @@ -51,7 +52,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne return err } - comp.LinuxClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-cluster-agent", }) @@ -59,7 +60,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne return err } - comp.LinuxClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-clusterchecks", }) @@ -70,21 +71,21 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne baseName = "dda-" + platform - comp.WindowsNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog", }) if err != nil { return err } - comp.WindowsClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-cluster-agent", }) if err != nil { return err } - comp.WindowsClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-clusterchecks", }) if err != nil { diff --git a/components/datadog/agent/kubernetes.go b/components/datadog/agent/kubernetes.go index 3da247593..64b529db2 100644 --- a/components/datadog/agent/kubernetes.go +++ b/components/datadog/agent/kubernetes.go @@ -1,6 +1,7 @@ package agent import ( + "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/DataDog/test-infra-definitions/components" @@ -9,13 +10,13 @@ import ( type KubernetesAgentOutput struct { components.JSONImporter - LinuxNodeAgent KubernetesObjRefOutput `json:"linuxNodeAgent"` - LinuxClusterAgent KubernetesObjRefOutput `json:"linuxClusterAgent"` - LinuxClusterChecks KubernetesObjRefOutput `json:"linuxClusterChecks"` + LinuxNodeAgent kubernetes.KubernetesObjRefOutput `json:"linuxNodeAgent"` + LinuxClusterAgent kubernetes.KubernetesObjRefOutput `json:"linuxClusterAgent"` + LinuxClusterChecks kubernetes.KubernetesObjRefOutput `json:"linuxClusterChecks"` - WindowsNodeAgent KubernetesObjRefOutput `json:"windowsNodeAgent"` - WindowsClusterAgent KubernetesObjRefOutput `json:"windowsClusterAgent"` - WindowsClusterChecks KubernetesObjRefOutput `json:"windowsClusterChecks"` + WindowsNodeAgent kubernetes.KubernetesObjRefOutput `json:"windowsNodeAgent"` + WindowsClusterAgent kubernetes.KubernetesObjRefOutput `json:"windowsClusterAgent"` + WindowsClusterChecks kubernetes.KubernetesObjRefOutput `json:"windowsClusterChecks"` } // KubernetesAgent is an installer to install the Datadog Agent on a Kubernetes cluster. @@ -23,13 +24,13 @@ type KubernetesAgent struct { pulumi.ResourceState components.Component - LinuxNodeAgent *KubernetesObjectRef `pulumi:"linuxNodeAgent"` - LinuxClusterAgent *KubernetesObjectRef `pulumi:"linuxClusterAgent"` - LinuxClusterChecks *KubernetesObjectRef `pulumi:"linuxClusterChecks"` + LinuxNodeAgent *kubernetes.KubernetesObjectRef `pulumi:"linuxNodeAgent"` + LinuxClusterAgent *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterAgent"` + LinuxClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterChecks"` - WindowsNodeAgent *KubernetesObjectRef `pulumi:"windowsNodeAgent"` - WindowsClusterAgent *KubernetesObjectRef `pulumi:"windowsClusterAgent"` - WindowsClusterChecks *KubernetesObjectRef `pulumi:"windowsClusterChecks"` + WindowsNodeAgent *kubernetes.KubernetesObjectRef `pulumi:"windowsNodeAgent"` + WindowsClusterAgent *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterAgent"` + WindowsClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterChecks"` } func (h *KubernetesAgent) Export(ctx *pulumi.Context, out *KubernetesAgentOutput) error { diff --git a/components/datadog/agent/kubernetes_operator.go b/components/datadog/agent/kubernetes_operator.go index 8b666ab0b..c04b0b3ee 100644 --- a/components/datadog/agent/kubernetes_operator.go +++ b/components/datadog/agent/kubernetes_operator.go @@ -1,39 +1,25 @@ package agent import ( - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "github.com/DataDog/test-infra-definitions/common/config" - "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components" "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" "github.com/DataDog/test-infra-definitions/components/datadog/apps/dda" - "github.com/DataDog/test-infra-definitions/components/datadog/operator" - "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, operatorOpts []operatorparams.Option, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) { +func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) { return components.NewComponent(e, resourceName, func(comp *KubernetesAgent) error { - - operatorParams, err := operatorparams.NewParams(e, operatorOpts...) - if err != nil { - return err - } - ddaParams, err := agentwithoperatorparams.NewParams(ddaOptions...) - if err != nil { return err } - operatorComp, err := operator.NewOperator(e, resourceName, kubeProvider, operatorOpts...) - - if err != nil { - return err - } + pulumiResourceOptions := append(ddaParams.PulumiResourceOptions, pulumi.Parent(comp)) - _, err = dda.K8sAppDefinition(e, kubeProvider, "datadog", ddaParams.FakeIntake, ddaParams.KubeletTLSVerify, e.Ctx().Stack(), ddaParams.DDAConfig, utils.PulumiDependsOn(operatorComp)) + _, err = dda.K8sAppDefinition(e, kubeProvider, ddaParams, pulumiResourceOptions...) if err != nil { return err @@ -41,22 +27,22 @@ func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kuberne baseName := "dda-linux" - comp.LinuxNodeAgent, err = NewKubernetesObjRef(e, baseName+"-nodeAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{"app": baseName + "-datadog"}) + comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{"app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-agent"}) if err != nil { return err } - comp.LinuxClusterAgent, err = NewKubernetesObjRef(e, baseName+"-clusterAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{ - "app": baseName + "-datadog-cluster-agent", + comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{ + "app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-agent", }) if err != nil { return err } - comp.LinuxClusterChecks, err = NewKubernetesObjRef(e, baseName+"-clusterChecks", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("version").ToStringOutput(), map[string]string{ - "app": baseName + "-datadog-clusterchecks", + comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{ + "app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-checks-runner", }) if err != nil { diff --git a/components/datadog/agentwithoperatorparams/params.go b/components/datadog/agentwithoperatorparams/params.go index df0a39ce0..4b71162d4 100644 --- a/components/datadog/agentwithoperatorparams/params.go +++ b/components/datadog/agentwithoperatorparams/params.go @@ -11,10 +11,9 @@ import ( type Params struct { PulumiResourceOptions []pulumi.ResourceOption - Namespace string - FakeIntake *fakeintake.Fakeintake - DDAConfig string - KubeletTLSVerify bool + Namespace string + FakeIntake *fakeintake.Fakeintake + DDAConfig DDAConfig } type Option = func(*Params) error @@ -22,6 +21,9 @@ type Option = func(*Params) error func NewParams(options ...Option) (*Params, error) { version := &Params{ Namespace: "datadog", + DDAConfig: DDAConfig{ + Name: "dda", + }, } return common.ApplyOption(version, options) } @@ -34,14 +36,6 @@ func WithNamespace(namespace string) func(*Params) error { } } -// WithTLSKubeletVerify toggles kubelet TLS verification. -func WithTLSKubeletVerify(verify bool) func(*Params) error { - return func(p *Params) error { - p.KubeletTLSVerify = verify - return nil - } -} - // WithPulumiResourceOptions sets the resources to depend on. func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) error { return func(p *Params) error { @@ -50,10 +44,10 @@ func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) } } -// WithDDAConfig configures the DatadogAgent resource. -func WithDDAConfig(config string) func(*Params) error { +// WithDDAConfig configures the DatadogAgent custom resource. +func WithDDAConfig(config DDAConfig) func(*Params) error { return func(p *Params) error { - p.DDAConfig = p.DDAConfig + config + p.DDAConfig = config return nil } } @@ -66,3 +60,15 @@ func WithFakeIntake(fakeintake *fakeintake.Fakeintake) func(*Params) error { return nil } } + +// DDAConfig is the DatadogAgent custom resource configuration. +type DDAConfig struct { + // Name of the DatadogAgent custom resource + Name string `json:"name"` + // YamlFilePath file path to the DatadogAgent custom resource YAML + YamlFilePath string `json:"yamlFilePath,omitempty"` + // YamlConfig is the YAML string of the DatadogAgent custom resource + YamlConfig string `json:"YamlConfig,omitempty"` + // MapConfig is the map representation of the DatadogAgent custom resource + MapConfig map[string]interface{} `json:"MapConfig,omitempty"` +} diff --git a/components/datadog/apps/dda/datadogagent.go b/components/datadog/apps/dda/datadogagent.go index c83a943e4..926a53b6f 100644 --- a/components/datadog/apps/dda/datadogagent.go +++ b/components/datadog/apps/dda/datadogagent.go @@ -1,27 +1,40 @@ package dda import ( - "encoding/json" "fmt" "dario.cat/mergo" + "github.com/DataDog/test-infra-definitions/common/config" + "github.com/DataDog/test-infra-definitions/common/utils" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "gopkg.in/yaml.v3" +) - "github.com/DataDog/test-infra-definitions/common/config" - "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" - componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" +const ( + baseName = "dda" ) -func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace string, fakeIntake *fakeintake.Fakeintake, kubeletTLSVerify bool, clusterName string, customDda string, opts ...pulumi.ResourceOption) (*componentskube.Workload, error) { +type datadogAgentWorkload struct { + ctx *pulumi.Context + opts *agentwithoperatorparams.Params + name string + clusterName string + imagePullSecret *corev1.Secret +} + +func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, params *agentwithoperatorparams.Params, opts ...pulumi.ResourceOption) (*componentskube.Workload, error) { + if params == nil { + return nil, nil + } apiKey := e.AgentAPIKey() appKey := e.AgentAPPKey() - baseName := "dda-with-operator" + clusterName := e.Ctx().Stack() + opts = append(opts, pulumi.Provider(kubeProvider), pulumi.Parent(kubeProvider), pulumi.DeletedWith(kubeProvider)) k8sComponent := &componentskube.Workload{} @@ -31,26 +44,10 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace opts = append(opts, pulumi.Parent(k8sComponent)) - ns, err := corev1.NewNamespace( - e.Ctx(), - namespace, - &corev1.NamespaceArgs{ - Metadata: metav1.ObjectMetaArgs{ - Name: pulumi.String(namespace), - }, - }, - opts..., - ) - if err != nil { - return nil, err - } - - opts = append(opts, utils.PulumiDependsOn(ns)) - - // Create secret if necessary + // Create datadog-credentials secret if necessary secret, err := corev1.NewSecret(e.Ctx(), "datadog-credentials", &corev1.SecretArgs{ Metadata: metav1.ObjectMetaArgs{ - Namespace: ns.Metadata.Name(), + Namespace: pulumi.String(params.Namespace), Name: pulumi.Sprintf("%s-datadog-credentials", baseName), }, StringData: pulumi.StringMap{ @@ -63,201 +60,242 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } opts = append(opts, utils.PulumiDependsOn(secret)) - ddaConfig := buildDDAConfig(baseName, clusterName, kubeletTLSVerify) - if fakeIntake != nil { - configureFakeIntake(ddaConfig, fakeIntake) - } - ddaConfig, err = mergeYamlToConfig(ddaConfig, customDda) - - if err != nil { - return nil, err - } - - // Image pull secrets need to be configured after custom DDA config merge because pulumi.StringOutput cannot be marshalled to JSON + // Create imagePullSecret var imagePullSecret *corev1.Secret if e.ImagePullRegistry() != "" { - imagePullSecret, err = utils.NewImagePullSecret(e, namespace, opts...) + imagePullSecret, err = utils.NewImagePullSecret(e, params.Namespace, opts...) if err != nil { return nil, err } opts = append(opts, utils.PulumiDependsOn(imagePullSecret)) - configureImagePullSecret(ddaConfig, imagePullSecret) } - _, err = apiextensions.NewCustomResource(e.Ctx(), "datadog-agent", &apiextensions.CustomResourceArgs{ - ApiVersion: pulumi.String("datadoghq.com/v2alpha1"), - Kind: pulumi.String("DatadogAgent"), - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String("datadog"), - Namespace: pulumi.String("datadog"), - }, - OtherFields: ddaConfig, - }, opts...) - if err != nil { + ddaWorkload := datadogAgentWorkload{ + ctx: e.Ctx(), + opts: params, + name: params.DDAConfig.Name, + clusterName: clusterName, + imagePullSecret: imagePullSecret, + } + + if err = ddaWorkload.buildDDAConfig(opts...); err != nil { + e.Ctx().Log.Debug(fmt.Sprintf("Error building DDA config: %v", err), nil) return nil, err } return k8sComponent, nil } -func buildDDAConfig(baseName string, clusterName string, kubeletTLSVerify bool) kubernetes.UntypedArgs { - return kubernetes.UntypedArgs{ - "spec": pulumi.Map{ - "global": pulumi.Map{ - "clusterName": pulumi.String(clusterName), - "kubelet": pulumi.Map{ - "tlsVerify": pulumi.Bool(kubeletTLSVerify), - }, - "credentials": pulumi.Map{ - "apiSecret": pulumi.Map{ - "secretName": pulumi.String(baseName + "-datadog-credentials"), - "keyName": pulumi.String("api-key"), - }, - "appSecret": pulumi.Map{ - "secretName": pulumi.String(baseName + "-datadog-credentials"), - "keyName": pulumi.String("app-key"), - }, - }, - }, - "features": pulumi.Map{ - "clusterChecks": pulumi.Map{ - "enabled": pulumi.Bool(true), - "useClusterChecksRunners": pulumi.Bool(true), - }, - "dogstatsd": pulumi.Map{ - "tagCardinality": pulumi.String("high"), - }, - "logCollection": pulumi.Map{ - "enabled": pulumi.Bool(true), - "containerCollectAll": pulumi.Bool(true), - "containerCollectUsingFiles": pulumi.Bool(true), - }, - "prometheusScrape": pulumi.Map{ - "enabled": pulumi.Bool(true), - "version": pulumi.Int(2), - }, - "liveProcessCollection": pulumi.Map{ - "enabled": pulumi.Bool(true), - }, - "eventCollection": pulumi.Map{ - "collectKubernetesEvents": pulumi.Bool(false), +func (d datadogAgentWorkload) buildDDAConfig(opts ...pulumi.ResourceOption) error { + ctx := d.ctx + defaultYamlTransformations := d.defaultDDAYamlTransformations() + + if d.opts.DDAConfig.YamlFilePath != "" { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Files: []string{d.opts.DDAConfig.YamlFilePath}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig yaml file path: %v", err), nil) + return err + } + } else if d.opts.DDAConfig.YamlConfig != "" { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + YAML: []string{d.opts.DDAConfig.YamlConfig}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig yaml: %v", err), nil) + return err + } + } else if d.opts.DDAConfig.MapConfig != nil { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Objs: []map[string]interface{}{d.opts.DDAConfig.MapConfig}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig map config: %v", err), nil) + return err + } + } else { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Objs: []map[string]interface{}{d.defaultDDAConfig()}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error creating default DDA config: %v", err), nil) + return err + } + + } + return nil +} + +func (d datadogAgentWorkload) defaultDDAConfig() map[string]interface{} { + return map[string]interface{}{ + "apiVersion": "datadoghq.com/v2alpha1", + "kind": "DatadogAgent", + "metadata": map[string]interface{}{ + "name": d.opts.DDAConfig.Name, + "namespace": d.opts.Namespace, + }, + "spec": map[string]interface{}{ + "features": map[string]interface{}{ + "clusterChecks": map[string]interface{}{ + "enabled": true, + "useClusterChecksRunners": true, }, }, }, } } -func configureFakeIntake(config kubernetes.UntypedArgs, fakeintake *fakeintake.Fakeintake) { - if fakeintake == nil { - return - } - endpointsEnvVar := pulumi.StringMapArray{ - pulumi.StringMap{ - "name": pulumi.String("DD_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), +func (d datadogAgentWorkload) fakeIntakeEnvVars() []map[string]interface{} { + return []map[string]interface{}{ + { + "name": "DD_DD_URL", + "value": d.opts.FakeIntake.URL, + }, + { + "name": "DD_PROCESS_CONFIG_PROCESS_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_PROCESS_CONFIG_PROCESS_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), + { + "name": "DD_APM_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_APM_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), + { + "name": "DD_LOGS_CONFIG_LOGS_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_SKIP_SSL_VALIDATION"), - "value": pulumi.String("true"), + { + "name": "DD_LOGS_CONFIG_USE_HTTP", + "value": "true", }, - pulumi.StringMap{ - "name": pulumi.String("DD_REMOTE_CONFIGURATION_NO_TLS_VALIDATION"), - "value": pulumi.String("true"), + { + "name": "DD_SKIP_SSL_VALIDATION", + "value": "true", }, - pulumi.StringMap{ - "name": pulumi.String("DD_LOGS_CONFIG_USE_HTTP"), - "value": pulumi.String("true"), + { + "name": "DD_REMOTE_CONFIGURATION_NO_TLS_VALIDATION", + "value": "true", }, } - for _, section := range []string{"nodeAgent", "clusterAgent", "clusterChecksRunner"} { - if _, found := config["spec"].(pulumi.Map)["override"]; !found { - config["spec"].(pulumi.Map)["override"] = pulumi.Map{ - section: pulumi.Map{ - "env": endpointsEnvVar, +} + +func (d datadogAgentWorkload) defaultDDAYamlTransformations() []yaml.Transformation { + return []yaml.Transformation{ + // Configure metadata + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + defaultMetadata := map[string]interface{}{ + "name": d.opts.DDAConfig.Name, + "namespace": d.opts.Namespace, + } + if state["metadata"] == nil { + state["metadata"] = defaultMetadata + } else { + stateMetadata := state["metadata"].(map[string]interface{}) + err := mergo.Merge(&stateMetadata, defaultMetadata) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging DDA metadata YAML: %v", err), nil) + } + + } + }, + // Configure global + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + defaultGlobal := map[string]interface{}{ + "clusterName": d.clusterName, + "credentials": map[string]interface{}{ + "apiSecret": map[string]interface{}{ + "secretName": baseName + "-datadog-credentials", + "keyName": "api-key", + }, + "appSecret": map[string]interface{}{ + "secretName": baseName + "-datadog-credentials", + "keyName": "app-key", + }, }, } - } else if _, found = config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section]; !found { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section] = pulumi.Map{ - "env": endpointsEnvVar, + if state["spec"].(map[string]interface{})["global"] == nil { + state["spec"].(map[string]interface{})["global"] = defaultGlobal + } else { + stateGlobal := state["spec"].(map[string]interface{})["global"].(map[string]interface{}) + err := mergo.Map(&stateGlobal, defaultGlobal) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging DDA global YAML: %v", err), nil) + } + } + }, + // Configure Fake Intake + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + if d.opts.FakeIntake == nil { + return + } + fakeIntakeOverride := map[string]interface{}{ + "nodeAgent": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + "clusterAgent": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + "clusterChecksRunner": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + } + if state["spec"].(map[string]interface{})["override"] == nil { + state["spec"].(map[string]interface{})["override"] = fakeIntakeOverride + } else { + stateOverride := state["spec"].(map[string]interface{})["override"].(map[string]interface{}) + err := mergo.Map(&stateOverride, fakeIntakeOverride) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging fakeintake override YAML: %v", err), nil) + } + } + }, + // Configure Image pull secret + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + if d.imagePullSecret == nil { + return } - } else if _, found = config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"]; !found { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"] = endpointsEnvVar - } else { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"] = append(config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"].(pulumi.StringMapArray), endpointsEnvVar...) - } - } -} - -func configureImagePullSecret(config kubernetes.UntypedArgs, secret *corev1.Secret) { - if secret == nil { - return - } - for _, section := range []string{"nodeAgent", "clusterAgent", "clusterChecksRunner"} { - if _, found := config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section]; !found { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section] = pulumi.Map{ - "image": pulumi.Map{ - "pullSecrets": pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + imgPullSecretOverride := map[string]interface{}{ + "nodeAgent": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), }, }, }, - } - } else if _, found = config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"]; !found { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"] = pulumi.Map{ - "pullSecrets": pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + "clusterAgent": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), + }, }, }, - } - } else { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"].(map[string]interface{})["pullSecrets"] = pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + "clusterChecksRunner": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), + }, + }, }, } - } - } -} -func mergeYamlToConfig(config kubernetes.UntypedArgs, yamlConfig string) (kubernetes.UntypedArgs, error) { - var configMap, yamlMap map[string]interface{} - configJSON, err := json.Marshal(config) - if err != nil { - fmt.Println(fmt.Sprintf("Error marshalling original DDA config: %v)", err)) - return config, err - } - - if err := json.Unmarshal(configJSON, &configMap); err != nil { - return config, fmt.Errorf("error unmarshalling original DDA config: %v", err) - } - if err := yaml.Unmarshal([]byte(yamlConfig), &yamlMap); err != nil { - return config, fmt.Errorf("error unmarshalling new DDA yaml config: %v", err) - } - - if err := mergo.Map(&configMap, yamlMap, mergo.WithOverride); err != nil { - return config, fmt.Errorf("error merging DDA configs: %v", err) - } - - merged, err := json.Marshal(configMap) - if err != nil { - return config, fmt.Errorf("error marshalling merged DDA config: %v", err) - } - - var mergedConfig kubernetes.UntypedArgs - if err = json.Unmarshal(merged, &mergedConfig); err != nil { - return config, fmt.Errorf("error ummarshalling merged DDA config: %v", err) + if state["spec"].(map[string]interface{})["override"] == nil { + state["spec"].(map[string]interface{})["override"] = imgPullSecretOverride + } else { + stateOverride := state["spec"].(map[string]interface{})["override"].(map[string]interface{}) + err := mergo.Map(&stateOverride, imgPullSecretOverride) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging imagePullSecrets override YAML: %v", err), nil) + } + } + }, } - - return mergedConfig, nil } diff --git a/components/datadog/operator/component.go b/components/datadog/operator/component.go new file mode 100644 index 000000000..6d5546ac9 --- /dev/null +++ b/components/datadog/operator/component.go @@ -0,0 +1,27 @@ +package operator + +import ( + compkubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/test-infra-definitions/components" +) + +// OperatorOutput is used to import the Operator component +type OperatorOutput struct { // nolint:revive, We want to keep the name as Output + components.JSONImporter + + Operator compkubernetes.KubernetesObjRefOutput `json:"operator"` +} + +// Operator represents an Operator installation +type Operator struct { + pulumi.ResourceState + components.Component + + Operator *compkubernetes.KubernetesObjectRef `pulumi:"operator"` +} + +func (o *Operator) Export(ctx *pulumi.Context, out *OperatorOutput) error { + return components.Export(ctx, o, out) +} diff --git a/components/datadog/operator/helm.go b/components/datadog/operator/helm.go index 15701ebe1..7bda78e3b 100644 --- a/components/datadog/operator/helm.go +++ b/components/datadog/operator/helm.go @@ -93,7 +93,7 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi operatorImagePath = args.OperatorFullImagePath } operatorImagePath, operatorImageTag := utils.ParseImageReference(operatorImagePath) - linuxInstallName := baseName + "-linux" + linuxInstallName := baseName + "-operator-linux" values := buildLinuxHelmValues(baseName, operatorImagePath, operatorImageTag) values.configureImagePullSecret(imgPullSecret) diff --git a/components/datadog/operator/operator.go b/components/datadog/operator/operator.go index 69c90043d..cf8325594 100644 --- a/components/datadog/operator/operator.go +++ b/components/datadog/operator/operator.go @@ -1,6 +1,7 @@ package operator import ( + compkubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -9,21 +10,6 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" ) -// OperatorOutput is used to import the Operator component -type Output struct { - components.JSONImporter -} - -// Operator represents an Operator installation -type Operator struct { - pulumi.ResourceState - components.Component -} - -func (h *Operator) Export(ctx *pulumi.Context, out *Output) error { - return components.Export(ctx, h, out) -} - func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, options ...operatorparams.Option) (*Operator, error) { return components.NewComponent(e, resourceName, func(comp *Operator) error { params, err := operatorparams.NewParams(e, options...) @@ -32,7 +18,7 @@ func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Pro } pulumiResourceOptions := append(params.PulumiResourceOptions, pulumi.Parent(comp)) - _, err = NewHelmInstallation(e, HelmInstallationArgs{ + release, err := NewHelmInstallation(e, HelmInstallationArgs{ KubeProvider: kubeProvider, Namespace: params.Namespace, ValuesYAML: params.HelmValues, @@ -42,6 +28,12 @@ func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Pro return err } + comp.Operator, err = compkubernetes.NewKubernetesObjRef(e, "datadog-operator", params.Namespace, "Pod", release.LinuxHelmReleaseStatus.AppVersion().Elem(), release.LinuxHelmReleaseStatus.Version().Elem(), map[string]string{"app.kubernetes.io/name": "datadog-operator"}) + + if err != nil { + return err + } + return nil }) } diff --git a/components/datadog/operatorparams/params.go b/components/datadog/operatorparams/params.go index 188fb01b3..5a54cd7d4 100644 --- a/components/datadog/operatorparams/params.go +++ b/components/datadog/operatorparams/params.go @@ -11,11 +11,11 @@ import ( ) type Params struct { - // OperatorFullImagePath is the full path of the docker agent image to use. + // OperatorFullImagePath is the full path of the operator image to use. OperatorFullImagePath string - // Namespace is the namespace to deploy the agent to. + // Namespace is the namespace to deploy the operator to. Namespace string - // HelmValues is the Helm values to use for the agent installation. + // HelmValues is the Helm values to use for the operator installation. HelmValues pulumi.AssetOrArchiveArray // PulumiResourceOptions is a list of resources to depend on. PulumiResourceOptions []pulumi.ResourceOption @@ -25,8 +25,7 @@ type Option = func(*Params) error func NewParams(e config.Env, options ...Option) (*Params, error) { version := &Params{ - Namespace: "datadog", - OperatorFullImagePath: "gcr.io/datadoghq/operator:latest", + Namespace: "datadog", } if e.PipelineID() != "" && e.CommitSHA() != "" { @@ -52,11 +51,21 @@ func WithOperatorFullImagePath(path string) func(*Params) error { } } -// WithHelmValues adds helm values to the agent installation. If used several times, the helm values are merged together +// WithHelmValues adds helm values to the operator installation. If used several times, the helm values are merged together // If the same values is defined several times the latter call will override the previous one. +// Accepts a string for single-line values (e.g. installCRDs: true) or a string literal in yaml format +// for multi-line values func WithHelmValues(values string) func(*Params) error { return func(p *Params) error { p.HelmValues = append(p.HelmValues, pulumi.NewStringAsset(values)) return nil } } + +// WithPulumiResourceOptions sets the resources to depend on. +func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) error { + return func(p *Params) error { + p.PulumiResourceOptions = append(p.PulumiResourceOptions, resources...) + return nil + } +} diff --git a/components/datadog/agent/kubernetes_object_ref.go b/components/kubernetes/kubernetes_object_ref.go similarity index 88% rename from components/datadog/agent/kubernetes_object_ref.go rename to components/kubernetes/kubernetes_object_ref.go index 7bdcf8b2e..f3ff3e1f2 100644 --- a/components/datadog/agent/kubernetes_object_ref.go +++ b/components/kubernetes/kubernetes_object_ref.go @@ -1,4 +1,4 @@ -package agent +package kubernetes import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -7,7 +7,7 @@ import ( "github.com/DataDog/test-infra-definitions/components" ) -type KubernetesObjRefOutput struct { +type KubernetesObjRefOutput struct { // nolint:revive, We want to keep the name as ObjRefOutput components.JSONImporter Namespace string `json:"namespace"` @@ -18,7 +18,7 @@ type KubernetesObjRefOutput struct { LabelSelectors map[string]string `json:"labelSelectors"` } -type KubernetesObjectRef struct { +type KubernetesObjectRef struct { // nolint:revive, We want to keep the name as ObjectRef pulumi.ResourceState components.Component diff --git a/scenarios/aws/kindvm/run.go b/scenarios/aws/kindvm/run.go index 9a89e67f6..f58e2cb39 100644 --- a/scenarios/aws/kindvm/run.go +++ b/scenarios/aws/kindvm/run.go @@ -15,6 +15,7 @@ import ( dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + "github.com/DataDog/test-infra-definitions/components/datadog/operator" "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" @@ -145,11 +146,31 @@ agents: operatorparams.WithNamespace("datadog"), ) + operatorComp, err := operator.NewOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator"), kindKubeProvider, operatorOpts...) + if err != nil { + return err + } + + if err := operatorComp.Export(awsEnv.Ctx(), nil); err != nil { + return err + } + + ddaConfig := agentwithoperatorparams.DDAConfig{ + Name: "dda-with-operator", + YamlConfig: ` +apiVersion: datadoghq.com/v2alpha1 +kind: DatadogAgent +spec: + global: + kubelet: + tlsVerify: false +`} + ddaOptions := make([]agentwithoperatorparams.Option, 0) ddaOptions = append( ddaOptions, agentwithoperatorparams.WithNamespace("datadog"), - agentwithoperatorparams.WithTLSKubeletVerify(false), + agentwithoperatorparams.WithDDAConfig(ddaConfig), ) if fakeIntake != nil { @@ -159,7 +180,7 @@ agents: ) } - operatorAgentComponent, err := agent.NewDDAWithOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator-agent"), kindKubeProvider, operatorOpts, ddaOptions...) + operatorAgentComponent, err := agent.NewDDAWithOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator-agent"), kindKubeProvider, ddaOptions...) if err != nil { return err