diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a295d3dc1..16e8e01ac 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -15,6 +15,13 @@ updates: directory: / schedule: interval: daily + groups: + aws-sdk-go-v2: + patterns: + - 'github.com/aws/aws-sdk-go-v2*' + pulumi-azure-native-sdk: + patterns: + - 'github.com/pulumi/pulumi-azure-native-sdk*' - package-ecosystem: pip directory: / @@ -85,3 +92,5 @@ updates: directory: /components/datadog/apps/cws/images/cws-centos7 schedule: interval: daily + + \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ddc2d5ce1..726f65d76 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -104,10 +104,19 @@ bump-version-on-datadog-agent: stage: post-release image: ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA} tags: ["arch:amd64"] - needs: ["release-runner-image"] rules: - if: $CI_COMMIT_BRANCH == "main" + needs: ["release-runner-image"] when: on_success + - if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/ + when: never + - when: manual + needs: ["build-runner-image"] + allow_failure: true + variables: + EXTRA_UPDATE_ARGS: "--is-dev-image" + variables: + EXTRA_UPDATE_ARGS: "" before_script: - set +x - export GITHUB_APP_USER_ID=153269286 # Can be found on https://api.github.com/users/agent-platform-auto-pr[bot] @@ -126,8 +135,9 @@ bump-version-on-datadog-agent: - pushd datadog-agent - git checkout -b auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA - export PREVIOUS_SHA=$(cat .gitlab/common/test_infra_version.yml | grep 'TEST_INFRA_DEFINITIONS_BUILDIMAGES:' | awk -F " " '{print $NF}') - - inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA - - git add test/new-e2e/go.mod test/new-e2e/go.sum .gitlab/common/test_infra_version.yml + - inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA $EXTRA_UPDATE_ARGS + - inv -e tidy + - git add -u - git commit -m "[test-infra-definitions][automated] Bump test-infra-definitions to $CI_COMMIT_SHORT_SHA" - git push -f origin auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA - popd diff --git a/components/datadog/agent/helm/kubernetes_agent.go b/components/datadog/agent/helm/kubernetes_agent.go index b56a621cc..684f5c6a4 100644 --- a/components/datadog/agent/helm/kubernetes_agent.go +++ b/components/datadog/agent/helm/kubernetes_agent.go @@ -1,6 +1,7 @@ package helm import ( + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -43,7 +44,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne baseName := "dda-" + platform - comp.LinuxNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog", }) @@ -51,7 +52,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne return err } - comp.LinuxClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-cluster-agent", }) @@ -59,7 +60,7 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne return err } - comp.LinuxClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-clusterchecks", }) @@ -70,21 +71,21 @@ func NewKubernetesAgent(e config.Env, resourceName string, kubeProvider *kuberne baseName = "dda-" + platform - comp.WindowsNodeAgent, err = agent.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog", }) if err != nil { return err } - comp.WindowsClusterAgent, err = agent.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-cluster-agent", }) if err != nil { return err } - comp.WindowsClusterChecks, err = agent.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ + comp.WindowsClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", params.Namespace, "Pod", appVersion, version, map[string]string{ "app": baseName + "-datadog-clusterchecks", }) if err != nil { diff --git a/components/datadog/agent/kubernetes.go b/components/datadog/agent/kubernetes.go index 3da247593..64b529db2 100644 --- a/components/datadog/agent/kubernetes.go +++ b/components/datadog/agent/kubernetes.go @@ -1,6 +1,7 @@ package agent import ( + "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" "github.com/DataDog/test-infra-definitions/components" @@ -9,13 +10,13 @@ import ( type KubernetesAgentOutput struct { components.JSONImporter - LinuxNodeAgent KubernetesObjRefOutput `json:"linuxNodeAgent"` - LinuxClusterAgent KubernetesObjRefOutput `json:"linuxClusterAgent"` - LinuxClusterChecks KubernetesObjRefOutput `json:"linuxClusterChecks"` + LinuxNodeAgent kubernetes.KubernetesObjRefOutput `json:"linuxNodeAgent"` + LinuxClusterAgent kubernetes.KubernetesObjRefOutput `json:"linuxClusterAgent"` + LinuxClusterChecks kubernetes.KubernetesObjRefOutput `json:"linuxClusterChecks"` - WindowsNodeAgent KubernetesObjRefOutput `json:"windowsNodeAgent"` - WindowsClusterAgent KubernetesObjRefOutput `json:"windowsClusterAgent"` - WindowsClusterChecks KubernetesObjRefOutput `json:"windowsClusterChecks"` + WindowsNodeAgent kubernetes.KubernetesObjRefOutput `json:"windowsNodeAgent"` + WindowsClusterAgent kubernetes.KubernetesObjRefOutput `json:"windowsClusterAgent"` + WindowsClusterChecks kubernetes.KubernetesObjRefOutput `json:"windowsClusterChecks"` } // KubernetesAgent is an installer to install the Datadog Agent on a Kubernetes cluster. @@ -23,13 +24,13 @@ type KubernetesAgent struct { pulumi.ResourceState components.Component - LinuxNodeAgent *KubernetesObjectRef `pulumi:"linuxNodeAgent"` - LinuxClusterAgent *KubernetesObjectRef `pulumi:"linuxClusterAgent"` - LinuxClusterChecks *KubernetesObjectRef `pulumi:"linuxClusterChecks"` + LinuxNodeAgent *kubernetes.KubernetesObjectRef `pulumi:"linuxNodeAgent"` + LinuxClusterAgent *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterAgent"` + LinuxClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"linuxClusterChecks"` - WindowsNodeAgent *KubernetesObjectRef `pulumi:"windowsNodeAgent"` - WindowsClusterAgent *KubernetesObjectRef `pulumi:"windowsClusterAgent"` - WindowsClusterChecks *KubernetesObjectRef `pulumi:"windowsClusterChecks"` + WindowsNodeAgent *kubernetes.KubernetesObjectRef `pulumi:"windowsNodeAgent"` + WindowsClusterAgent *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterAgent"` + WindowsClusterChecks *kubernetes.KubernetesObjectRef `pulumi:"windowsClusterChecks"` } func (h *KubernetesAgent) Export(ctx *pulumi.Context, out *KubernetesAgentOutput) error { diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index 94c6ea7c7..2501312aa 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -223,12 +223,23 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI "namespaceLabelsAsTags": pulumi.Map{ "related_team": pulumi.String("team"), }, - "originDetectionUnified": pulumi.Map{ - "enabled": pulumi.Bool(true), - }, "namespaceAnnotationsAsTags": pulumi.Map{ "related_email": pulumi.String("email"), // should be overridden by kubernetesResourcesAnnotationsAsTags }, + "kubernetesResourcesAnnotationsAsTags": pulumi.Map{ + "deployments.apps": pulumi.Map{"x-sub-team": pulumi.String("sub-team")}, + "pods": pulumi.Map{"x-parent-name": pulumi.String("parent-name")}, + "namespaces": pulumi.Map{"related_email": pulumi.String("mail")}, + }, + "kubernetesResourcesLabelsAsTags": pulumi.Map{ + "deployments.apps": pulumi.Map{"x-team": pulumi.String("team")}, + "pods": pulumi.Map{"x-parent-type": pulumi.String("domain")}, + "namespaces": pulumi.Map{"related_org": pulumi.String("org")}, + "nodes": pulumi.Map{"kubernetes.io/os": pulumi.String("os"), "kubernetes.io/arch": pulumi.String("arch")}, + }, + "originDetectionUnified": pulumi.Map{ + "enabled": pulumi.Bool(true), + }, "logs": pulumi.Map{ "enabled": pulumi.Bool(true), "containerCollectAll": pulumi.Bool(logsContainerCollectAll), @@ -304,14 +315,6 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI "name": pulumi.String("DD_TELEMETRY_CHECKS"), "value": pulumi.String("*"), }, - pulumi.StringMap{ - "name": pulumi.String("DD_KUBERNETES_RESOURCES_LABELS_AS_TAGS"), - "value": pulumi.JSONMarshal(getResourcesLabelsAsTags().toJSONString()), - }, - pulumi.StringMap{ - "name": pulumi.String("DD_KUBERNETES_RESOURCES_ANNOTATIONS_AS_TAGS"), - "value": pulumi.JSONMarshal(getResourcesAnnotationsAsTags().toJSONString()), - }, }, }, "agents": pulumi.Map{ @@ -457,6 +460,12 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI "tag": pulumi.String(agentImageTag), "doNotCheckTag": pulumi.Bool(true), }, + "env": pulumi.StringMapArray{ + pulumi.StringMap{ + "name": pulumi.String("DD_CLC_RUNNER_REMOTE_TAGGER_ENABLED"), + "value": pulumi.String("true"), + }, + }, "resources": pulumi.StringMapMap{ "requests": pulumi.StringMap{ "cpu": pulumi.String("20m"), diff --git a/components/datadog/agent/kubernetes_helm_utils.go b/components/datadog/agent/kubernetes_helm_utils.go deleted file mode 100644 index 2bbeb9b71..000000000 --- a/components/datadog/agent/kubernetes_helm_utils.go +++ /dev/null @@ -1,33 +0,0 @@ -package agent - -import "encoding/json" - -// TODO: Remove these defaults when kubernetes_resource_labels_as_tags and kubernetes_resource_annotations_as_tags are added to the helm chart - -type KubernetesResourcesMetadataAsTags map[string]map[string]string - -func (k KubernetesResourcesMetadataAsTags) toJSONString() string { - bytes, err := json.Marshal(k) - if err != nil { - return "" - } - - return string(bytes) -} - -func getResourcesLabelsAsTags() KubernetesResourcesMetadataAsTags { - return KubernetesResourcesMetadataAsTags{ - "deployments.apps": {"x-team": "team"}, - "pods": {"x-parent-type": "domain"}, - "namespaces": {"kubernetes.io/metadata.name": "metadata-name"}, - "nodes": {"kubernetes.io/os": "os", "kubernetes.io/arch": "arch"}, - } -} - -func getResourcesAnnotationsAsTags() KubernetesResourcesMetadataAsTags { - return KubernetesResourcesMetadataAsTags{ - "deployments.apps": {"x-sub-team": "sub-team"}, - "pods": {"x-parent-name": "parent-name"}, - "namespaces": {"related_email": "mail"}, - } -} diff --git a/components/datadog/agent/kubernetes_operator.go b/components/datadog/agent/kubernetes_operator.go index 8b666ab0b..304effe40 100644 --- a/components/datadog/agent/kubernetes_operator.go +++ b/components/datadog/agent/kubernetes_operator.go @@ -1,62 +1,48 @@ package agent import ( - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "github.com/DataDog/test-infra-definitions/common/config" - "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components" "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" "github.com/DataDog/test-infra-definitions/components/datadog/apps/dda" - "github.com/DataDog/test-infra-definitions/components/datadog/operator" - "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) -func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, operatorOpts []operatorparams.Option, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) { +func NewDDAWithOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, ddaOptions ...agentwithoperatorparams.Option) (*KubernetesAgent, error) { return components.NewComponent(e, resourceName, func(comp *KubernetesAgent) error { - - operatorParams, err := operatorparams.NewParams(e, operatorOpts...) - if err != nil { - return err - } - ddaParams, err := agentwithoperatorparams.NewParams(ddaOptions...) - if err != nil { return err } - operatorComp, err := operator.NewOperator(e, resourceName, kubeProvider, operatorOpts...) - - if err != nil { - return err - } + ddaParams.PulumiResourceOptions = append(ddaParams.PulumiResourceOptions, pulumi.Parent(comp)) - _, err = dda.K8sAppDefinition(e, kubeProvider, "datadog", ddaParams.FakeIntake, ddaParams.KubeletTLSVerify, e.Ctx().Stack(), ddaParams.DDAConfig, utils.PulumiDependsOn(operatorComp)) + _, err = dda.K8sAppDefinition(e, kubeProvider, ddaParams, ddaParams.PulumiResourceOptions...) if err != nil { return err } - baseName := "dda-linux" + baseName := "dda-with-operator-linux" - comp.LinuxNodeAgent, err = NewKubernetesObjRef(e, baseName+"-nodeAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{"app": baseName + "-datadog"}) + comp.LinuxNodeAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-nodeAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{"app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-agent"}) if err != nil { return err } - comp.LinuxClusterAgent, err = NewKubernetesObjRef(e, baseName+"-clusterAgent", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("Version").ToStringOutput(), map[string]string{ - "app": baseName + "-datadog-cluster-agent", + comp.LinuxClusterAgent, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterAgent", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{ + "app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-agent", }) if err != nil { return err } - comp.LinuxClusterChecks, err = NewKubernetesObjRef(e, baseName+"-clusterChecks", operatorParams.Namespace, "Pod", pulumi.String("appVersion").ToStringOutput(), pulumi.String("version").ToStringOutput(), map[string]string{ - "app": baseName + "-datadog-clusterchecks", + comp.LinuxClusterChecks, err = componentskube.NewKubernetesObjRef(e, baseName+"-clusterChecks", ddaParams.Namespace, "Pod", pulumi.String("").ToStringOutput(), pulumi.String("datadoghq/v2alpha1").ToStringOutput(), map[string]string{ + "app.kubernetes.io/instance": ddaParams.DDAConfig.Name + "-cluster-checks-runner", }) if err != nil { diff --git a/components/datadog/agentwithoperatorparams/params.go b/components/datadog/agentwithoperatorparams/params.go index df0a39ce0..4b71162d4 100644 --- a/components/datadog/agentwithoperatorparams/params.go +++ b/components/datadog/agentwithoperatorparams/params.go @@ -11,10 +11,9 @@ import ( type Params struct { PulumiResourceOptions []pulumi.ResourceOption - Namespace string - FakeIntake *fakeintake.Fakeintake - DDAConfig string - KubeletTLSVerify bool + Namespace string + FakeIntake *fakeintake.Fakeintake + DDAConfig DDAConfig } type Option = func(*Params) error @@ -22,6 +21,9 @@ type Option = func(*Params) error func NewParams(options ...Option) (*Params, error) { version := &Params{ Namespace: "datadog", + DDAConfig: DDAConfig{ + Name: "dda", + }, } return common.ApplyOption(version, options) } @@ -34,14 +36,6 @@ func WithNamespace(namespace string) func(*Params) error { } } -// WithTLSKubeletVerify toggles kubelet TLS verification. -func WithTLSKubeletVerify(verify bool) func(*Params) error { - return func(p *Params) error { - p.KubeletTLSVerify = verify - return nil - } -} - // WithPulumiResourceOptions sets the resources to depend on. func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) error { return func(p *Params) error { @@ -50,10 +44,10 @@ func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) } } -// WithDDAConfig configures the DatadogAgent resource. -func WithDDAConfig(config string) func(*Params) error { +// WithDDAConfig configures the DatadogAgent custom resource. +func WithDDAConfig(config DDAConfig) func(*Params) error { return func(p *Params) error { - p.DDAConfig = p.DDAConfig + config + p.DDAConfig = config return nil } } @@ -66,3 +60,15 @@ func WithFakeIntake(fakeintake *fakeintake.Fakeintake) func(*Params) error { return nil } } + +// DDAConfig is the DatadogAgent custom resource configuration. +type DDAConfig struct { + // Name of the DatadogAgent custom resource + Name string `json:"name"` + // YamlFilePath file path to the DatadogAgent custom resource YAML + YamlFilePath string `json:"yamlFilePath,omitempty"` + // YamlConfig is the YAML string of the DatadogAgent custom resource + YamlConfig string `json:"YamlConfig,omitempty"` + // MapConfig is the map representation of the DatadogAgent custom resource + MapConfig map[string]interface{} `json:"MapConfig,omitempty"` +} diff --git a/components/datadog/apps/dda/datadogagent.go b/components/datadog/apps/dda/datadogagent.go index c83a943e4..926a53b6f 100644 --- a/components/datadog/apps/dda/datadogagent.go +++ b/components/datadog/apps/dda/datadogagent.go @@ -1,27 +1,40 @@ package dda import ( - "encoding/json" "fmt" "dario.cat/mergo" + "github.com/DataDog/test-infra-definitions/common/config" + "github.com/DataDog/test-infra-definitions/common/utils" + "github.com/DataDog/test-infra-definitions/components/datadog/agentwithoperatorparams" + componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" - "gopkg.in/yaml.v3" +) - "github.com/DataDog/test-infra-definitions/common/config" - "github.com/DataDog/test-infra-definitions/common/utils" - "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" - componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" +const ( + baseName = "dda" ) -func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace string, fakeIntake *fakeintake.Fakeintake, kubeletTLSVerify bool, clusterName string, customDda string, opts ...pulumi.ResourceOption) (*componentskube.Workload, error) { +type datadogAgentWorkload struct { + ctx *pulumi.Context + opts *agentwithoperatorparams.Params + name string + clusterName string + imagePullSecret *corev1.Secret +} + +func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, params *agentwithoperatorparams.Params, opts ...pulumi.ResourceOption) (*componentskube.Workload, error) { + if params == nil { + return nil, nil + } apiKey := e.AgentAPIKey() appKey := e.AgentAPPKey() - baseName := "dda-with-operator" + clusterName := e.Ctx().Stack() + opts = append(opts, pulumi.Provider(kubeProvider), pulumi.Parent(kubeProvider), pulumi.DeletedWith(kubeProvider)) k8sComponent := &componentskube.Workload{} @@ -31,26 +44,10 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace opts = append(opts, pulumi.Parent(k8sComponent)) - ns, err := corev1.NewNamespace( - e.Ctx(), - namespace, - &corev1.NamespaceArgs{ - Metadata: metav1.ObjectMetaArgs{ - Name: pulumi.String(namespace), - }, - }, - opts..., - ) - if err != nil { - return nil, err - } - - opts = append(opts, utils.PulumiDependsOn(ns)) - - // Create secret if necessary + // Create datadog-credentials secret if necessary secret, err := corev1.NewSecret(e.Ctx(), "datadog-credentials", &corev1.SecretArgs{ Metadata: metav1.ObjectMetaArgs{ - Namespace: ns.Metadata.Name(), + Namespace: pulumi.String(params.Namespace), Name: pulumi.Sprintf("%s-datadog-credentials", baseName), }, StringData: pulumi.StringMap{ @@ -63,201 +60,242 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } opts = append(opts, utils.PulumiDependsOn(secret)) - ddaConfig := buildDDAConfig(baseName, clusterName, kubeletTLSVerify) - if fakeIntake != nil { - configureFakeIntake(ddaConfig, fakeIntake) - } - ddaConfig, err = mergeYamlToConfig(ddaConfig, customDda) - - if err != nil { - return nil, err - } - - // Image pull secrets need to be configured after custom DDA config merge because pulumi.StringOutput cannot be marshalled to JSON + // Create imagePullSecret var imagePullSecret *corev1.Secret if e.ImagePullRegistry() != "" { - imagePullSecret, err = utils.NewImagePullSecret(e, namespace, opts...) + imagePullSecret, err = utils.NewImagePullSecret(e, params.Namespace, opts...) if err != nil { return nil, err } opts = append(opts, utils.PulumiDependsOn(imagePullSecret)) - configureImagePullSecret(ddaConfig, imagePullSecret) } - _, err = apiextensions.NewCustomResource(e.Ctx(), "datadog-agent", &apiextensions.CustomResourceArgs{ - ApiVersion: pulumi.String("datadoghq.com/v2alpha1"), - Kind: pulumi.String("DatadogAgent"), - Metadata: &metav1.ObjectMetaArgs{ - Name: pulumi.String("datadog"), - Namespace: pulumi.String("datadog"), - }, - OtherFields: ddaConfig, - }, opts...) - if err != nil { + ddaWorkload := datadogAgentWorkload{ + ctx: e.Ctx(), + opts: params, + name: params.DDAConfig.Name, + clusterName: clusterName, + imagePullSecret: imagePullSecret, + } + + if err = ddaWorkload.buildDDAConfig(opts...); err != nil { + e.Ctx().Log.Debug(fmt.Sprintf("Error building DDA config: %v", err), nil) return nil, err } return k8sComponent, nil } -func buildDDAConfig(baseName string, clusterName string, kubeletTLSVerify bool) kubernetes.UntypedArgs { - return kubernetes.UntypedArgs{ - "spec": pulumi.Map{ - "global": pulumi.Map{ - "clusterName": pulumi.String(clusterName), - "kubelet": pulumi.Map{ - "tlsVerify": pulumi.Bool(kubeletTLSVerify), - }, - "credentials": pulumi.Map{ - "apiSecret": pulumi.Map{ - "secretName": pulumi.String(baseName + "-datadog-credentials"), - "keyName": pulumi.String("api-key"), - }, - "appSecret": pulumi.Map{ - "secretName": pulumi.String(baseName + "-datadog-credentials"), - "keyName": pulumi.String("app-key"), - }, - }, - }, - "features": pulumi.Map{ - "clusterChecks": pulumi.Map{ - "enabled": pulumi.Bool(true), - "useClusterChecksRunners": pulumi.Bool(true), - }, - "dogstatsd": pulumi.Map{ - "tagCardinality": pulumi.String("high"), - }, - "logCollection": pulumi.Map{ - "enabled": pulumi.Bool(true), - "containerCollectAll": pulumi.Bool(true), - "containerCollectUsingFiles": pulumi.Bool(true), - }, - "prometheusScrape": pulumi.Map{ - "enabled": pulumi.Bool(true), - "version": pulumi.Int(2), - }, - "liveProcessCollection": pulumi.Map{ - "enabled": pulumi.Bool(true), - }, - "eventCollection": pulumi.Map{ - "collectKubernetesEvents": pulumi.Bool(false), +func (d datadogAgentWorkload) buildDDAConfig(opts ...pulumi.ResourceOption) error { + ctx := d.ctx + defaultYamlTransformations := d.defaultDDAYamlTransformations() + + if d.opts.DDAConfig.YamlFilePath != "" { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Files: []string{d.opts.DDAConfig.YamlFilePath}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig yaml file path: %v", err), nil) + return err + } + } else if d.opts.DDAConfig.YamlConfig != "" { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + YAML: []string{d.opts.DDAConfig.YamlConfig}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig yaml: %v", err), nil) + return err + } + } else if d.opts.DDAConfig.MapConfig != nil { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Objs: []map[string]interface{}{d.opts.DDAConfig.MapConfig}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error transforming DDAConfig map config: %v", err), nil) + return err + } + } else { + _, err := yaml.NewConfigGroup(ctx, d.name, &yaml.ConfigGroupArgs{ + Objs: []map[string]interface{}{d.defaultDDAConfig()}, + Transformations: defaultYamlTransformations, + }, opts...) + + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error creating default DDA config: %v", err), nil) + return err + } + + } + return nil +} + +func (d datadogAgentWorkload) defaultDDAConfig() map[string]interface{} { + return map[string]interface{}{ + "apiVersion": "datadoghq.com/v2alpha1", + "kind": "DatadogAgent", + "metadata": map[string]interface{}{ + "name": d.opts.DDAConfig.Name, + "namespace": d.opts.Namespace, + }, + "spec": map[string]interface{}{ + "features": map[string]interface{}{ + "clusterChecks": map[string]interface{}{ + "enabled": true, + "useClusterChecksRunners": true, }, }, }, } } -func configureFakeIntake(config kubernetes.UntypedArgs, fakeintake *fakeintake.Fakeintake) { - if fakeintake == nil { - return - } - endpointsEnvVar := pulumi.StringMapArray{ - pulumi.StringMap{ - "name": pulumi.String("DD_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), +func (d datadogAgentWorkload) fakeIntakeEnvVars() []map[string]interface{} { + return []map[string]interface{}{ + { + "name": "DD_DD_URL", + "value": d.opts.FakeIntake.URL, + }, + { + "name": "DD_PROCESS_CONFIG_PROCESS_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_PROCESS_CONFIG_PROCESS_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), + { + "name": "DD_APM_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_APM_DD_URL"), - "value": pulumi.String(fmt.Sprintf("%v", fakeintake.URL)), + { + "name": "DD_LOGS_CONFIG_LOGS_DD_URL", + "value": d.opts.FakeIntake.URL, }, - pulumi.StringMap{ - "name": pulumi.String("DD_SKIP_SSL_VALIDATION"), - "value": pulumi.String("true"), + { + "name": "DD_LOGS_CONFIG_USE_HTTP", + "value": "true", }, - pulumi.StringMap{ - "name": pulumi.String("DD_REMOTE_CONFIGURATION_NO_TLS_VALIDATION"), - "value": pulumi.String("true"), + { + "name": "DD_SKIP_SSL_VALIDATION", + "value": "true", }, - pulumi.StringMap{ - "name": pulumi.String("DD_LOGS_CONFIG_USE_HTTP"), - "value": pulumi.String("true"), + { + "name": "DD_REMOTE_CONFIGURATION_NO_TLS_VALIDATION", + "value": "true", }, } - for _, section := range []string{"nodeAgent", "clusterAgent", "clusterChecksRunner"} { - if _, found := config["spec"].(pulumi.Map)["override"]; !found { - config["spec"].(pulumi.Map)["override"] = pulumi.Map{ - section: pulumi.Map{ - "env": endpointsEnvVar, +} + +func (d datadogAgentWorkload) defaultDDAYamlTransformations() []yaml.Transformation { + return []yaml.Transformation{ + // Configure metadata + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + defaultMetadata := map[string]interface{}{ + "name": d.opts.DDAConfig.Name, + "namespace": d.opts.Namespace, + } + if state["metadata"] == nil { + state["metadata"] = defaultMetadata + } else { + stateMetadata := state["metadata"].(map[string]interface{}) + err := mergo.Merge(&stateMetadata, defaultMetadata) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging DDA metadata YAML: %v", err), nil) + } + + } + }, + // Configure global + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + defaultGlobal := map[string]interface{}{ + "clusterName": d.clusterName, + "credentials": map[string]interface{}{ + "apiSecret": map[string]interface{}{ + "secretName": baseName + "-datadog-credentials", + "keyName": "api-key", + }, + "appSecret": map[string]interface{}{ + "secretName": baseName + "-datadog-credentials", + "keyName": "app-key", + }, }, } - } else if _, found = config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section]; !found { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section] = pulumi.Map{ - "env": endpointsEnvVar, + if state["spec"].(map[string]interface{})["global"] == nil { + state["spec"].(map[string]interface{})["global"] = defaultGlobal + } else { + stateGlobal := state["spec"].(map[string]interface{})["global"].(map[string]interface{}) + err := mergo.Map(&stateGlobal, defaultGlobal) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging DDA global YAML: %v", err), nil) + } + } + }, + // Configure Fake Intake + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + if d.opts.FakeIntake == nil { + return + } + fakeIntakeOverride := map[string]interface{}{ + "nodeAgent": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + "clusterAgent": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + "clusterChecksRunner": map[string]interface{}{ + "env": d.fakeIntakeEnvVars(), + }, + } + if state["spec"].(map[string]interface{})["override"] == nil { + state["spec"].(map[string]interface{})["override"] = fakeIntakeOverride + } else { + stateOverride := state["spec"].(map[string]interface{})["override"].(map[string]interface{}) + err := mergo.Map(&stateOverride, fakeIntakeOverride) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging fakeintake override YAML: %v", err), nil) + } + } + }, + // Configure Image pull secret + func(state map[string]interface{}, _ ...pulumi.ResourceOption) { + if d.imagePullSecret == nil { + return } - } else if _, found = config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"]; !found { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"] = endpointsEnvVar - } else { - config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"] = append(config["spec"].(pulumi.Map)["override"].(pulumi.Map)[section].(pulumi.Map)["env"].(pulumi.StringMapArray), endpointsEnvVar...) - } - } -} - -func configureImagePullSecret(config kubernetes.UntypedArgs, secret *corev1.Secret) { - if secret == nil { - return - } - for _, section := range []string{"nodeAgent", "clusterAgent", "clusterChecksRunner"} { - if _, found := config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section]; !found { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section] = pulumi.Map{ - "image": pulumi.Map{ - "pullSecrets": pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + imgPullSecretOverride := map[string]interface{}{ + "nodeAgent": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), }, }, }, - } - } else if _, found = config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"]; !found { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"] = pulumi.Map{ - "pullSecrets": pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + "clusterAgent": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), + }, }, }, - } - } else { - config["spec"].(map[string]interface{})["override"].(map[string]interface{})[section].(map[string]interface{})["image"].(map[string]interface{})["pullSecrets"] = pulumi.MapArray{ - pulumi.Map{ - "name": secret.Metadata.Name(), + "clusterChecksRunner": map[string]interface{}{ + "image": map[string]interface{}{ + "pullSecrets": map[string]interface{}{ + "name": d.imagePullSecret.Metadata.Name(), + }, + }, }, } - } - } -} -func mergeYamlToConfig(config kubernetes.UntypedArgs, yamlConfig string) (kubernetes.UntypedArgs, error) { - var configMap, yamlMap map[string]interface{} - configJSON, err := json.Marshal(config) - if err != nil { - fmt.Println(fmt.Sprintf("Error marshalling original DDA config: %v)", err)) - return config, err - } - - if err := json.Unmarshal(configJSON, &configMap); err != nil { - return config, fmt.Errorf("error unmarshalling original DDA config: %v", err) - } - if err := yaml.Unmarshal([]byte(yamlConfig), &yamlMap); err != nil { - return config, fmt.Errorf("error unmarshalling new DDA yaml config: %v", err) - } - - if err := mergo.Map(&configMap, yamlMap, mergo.WithOverride); err != nil { - return config, fmt.Errorf("error merging DDA configs: %v", err) - } - - merged, err := json.Marshal(configMap) - if err != nil { - return config, fmt.Errorf("error marshalling merged DDA config: %v", err) - } - - var mergedConfig kubernetes.UntypedArgs - if err = json.Unmarshal(merged, &mergedConfig); err != nil { - return config, fmt.Errorf("error ummarshalling merged DDA config: %v", err) + if state["spec"].(map[string]interface{})["override"] == nil { + state["spec"].(map[string]interface{})["override"] = imgPullSecretOverride + } else { + stateOverride := state["spec"].(map[string]interface{})["override"].(map[string]interface{}) + err := mergo.Map(&stateOverride, imgPullSecretOverride) + if err != nil { + d.ctx.Log.Debug(fmt.Sprintf("Error merging imagePullSecrets override YAML: %v", err), nil) + } + } + }, } - - return mergedConfig, nil } diff --git a/components/datadog/apps/dogstatsd/images/dogstatsd/go.mod b/components/datadog/apps/dogstatsd/images/dogstatsd/go.mod index ee48b3291..91e08f08d 100644 --- a/components/datadog/apps/dogstatsd/images/dogstatsd/go.mod +++ b/components/datadog/apps/dogstatsd/images/dogstatsd/go.mod @@ -2,12 +2,9 @@ module dogstatsd go 1.22 -require github.com/DataDog/datadog-go/v5 v5.5.0 +require github.com/DataDog/datadog-go/v5 v5.6.0 require ( github.com/Microsoft/go-winio v0.5.0 // indirect golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect ) - -// Temporary replacement of the main branch until https://github.com/DataDog/datadog-go/pull/304 is released -replace github.com/DataDog/datadog-go/v5 => github.com/DataDog/datadog-go/v5 v5.5.1-0.20240327105053-fa1f6814eaf7 diff --git a/components/datadog/apps/dogstatsd/images/dogstatsd/go.sum b/components/datadog/apps/dogstatsd/images/dogstatsd/go.sum index 8de2f7e95..4c1c48ccb 100644 --- a/components/datadog/apps/dogstatsd/images/dogstatsd/go.sum +++ b/components/datadog/apps/dogstatsd/images/dogstatsd/go.sum @@ -1,5 +1,5 @@ -github.com/DataDog/datadog-go/v5 v5.5.1-0.20240327105053-fa1f6814eaf7 h1:pOfzUVqO/rT7VTY9L7qK5oeiuBBIiuZgt0Nf7J3ywDQ= -github.com/DataDog/datadog-go/v5 v5.5.1-0.20240327105053-fa1f6814eaf7/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= +github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= +github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/components/datadog/apps/nginx/k8s.go b/components/datadog/apps/nginx/k8s.go index 50fee6c18..0d5140006 100644 --- a/components/datadog/apps/nginx/k8s.go +++ b/components/datadog/apps/nginx/k8s.go @@ -50,6 +50,7 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace Name: pulumi.String(namespace), Labels: pulumi.StringMap{ "related_team": pulumi.String("contp"), + "related_org": pulumi.String("agent-org"), }, Annotations: pulumi.StringMap{ "related_email": pulumi.String("team-container-platform@datadoghq.com"), diff --git a/components/datadog/operator/component.go b/components/datadog/operator/component.go new file mode 100644 index 000000000..6d5546ac9 --- /dev/null +++ b/components/datadog/operator/component.go @@ -0,0 +1,27 @@ +package operator + +import ( + compkubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/test-infra-definitions/components" +) + +// OperatorOutput is used to import the Operator component +type OperatorOutput struct { // nolint:revive, We want to keep the name as Output + components.JSONImporter + + Operator compkubernetes.KubernetesObjRefOutput `json:"operator"` +} + +// Operator represents an Operator installation +type Operator struct { + pulumi.ResourceState + components.Component + + Operator *compkubernetes.KubernetesObjectRef `pulumi:"operator"` +} + +func (o *Operator) Export(ctx *pulumi.Context, out *OperatorOutput) error { + return components.Export(ctx, o, out) +} diff --git a/components/datadog/operator/helm.go b/components/datadog/operator/helm.go index 15701ebe1..6a80abda8 100644 --- a/components/datadog/operator/helm.go +++ b/components/datadog/operator/helm.go @@ -39,7 +39,7 @@ type HelmComponent struct { func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi.ResourceOption) (*HelmComponent, error) { apiKey := e.AgentAPIKey() appKey := e.AgentAPPKey() - baseName := "dda" + opts = append(opts, pulumi.Providers(args.KubeProvider), e.WithProviders(config.ProviderRandom), pulumi.DeletedWith(args.KubeProvider)) helmComponent := &HelmComponent{} @@ -64,7 +64,7 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi secret, err := corev1.NewSecret(e.Ctx(), "datadog-credentials", &corev1.SecretArgs{ Metadata: metav1.ObjectMetaArgs{ Namespace: ns.Metadata.Name(), - Name: pulumi.Sprintf("%s-datadog-credentials", baseName), + Name: pulumi.String("dda-datadog-credentials"), }, StringData: pulumi.StringMap{ "api-key": apiKey, @@ -93,9 +93,9 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi operatorImagePath = args.OperatorFullImagePath } operatorImagePath, operatorImageTag := utils.ParseImageReference(operatorImagePath) - linuxInstallName := baseName + "-linux" + linuxInstallName := "datadog-operator-linux" - values := buildLinuxHelmValues(baseName, operatorImagePath, operatorImageTag) + values := buildLinuxHelmValues(operatorImagePath, operatorImageTag) values.configureImagePullSecret(imgPullSecret) defaultYAMLValues := values.toYAMLPulumiAssetOutput() @@ -132,10 +132,10 @@ func NewHelmInstallation(e config.Env, args HelmInstallationArgs, opts ...pulumi type HelmValues pulumi.Map -func buildLinuxHelmValues(baseName string, operatorImagePath string, operatorImageTag string) HelmValues { +func buildLinuxHelmValues(operatorImagePath string, operatorImageTag string) HelmValues { return HelmValues{ - "apiKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), - "appKeyExistingSecret": pulumi.String(baseName + "-datadog-credentials"), + "apiKeyExistingSecret": pulumi.String("dda-datadog-credentials"), + "appKeyExistingSecret": pulumi.String("dda-datadog-credentials"), "image": pulumi.Map{ "repository": pulumi.String(operatorImagePath), "tag": pulumi.String(operatorImageTag), diff --git a/components/datadog/operator/operator.go b/components/datadog/operator/operator.go index 69c90043d..cf8325594 100644 --- a/components/datadog/operator/operator.go +++ b/components/datadog/operator/operator.go @@ -1,6 +1,7 @@ package operator import ( + compkubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -9,21 +10,6 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" ) -// OperatorOutput is used to import the Operator component -type Output struct { - components.JSONImporter -} - -// Operator represents an Operator installation -type Operator struct { - pulumi.ResourceState - components.Component -} - -func (h *Operator) Export(ctx *pulumi.Context, out *Output) error { - return components.Export(ctx, h, out) -} - func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Provider, options ...operatorparams.Option) (*Operator, error) { return components.NewComponent(e, resourceName, func(comp *Operator) error { params, err := operatorparams.NewParams(e, options...) @@ -32,7 +18,7 @@ func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Pro } pulumiResourceOptions := append(params.PulumiResourceOptions, pulumi.Parent(comp)) - _, err = NewHelmInstallation(e, HelmInstallationArgs{ + release, err := NewHelmInstallation(e, HelmInstallationArgs{ KubeProvider: kubeProvider, Namespace: params.Namespace, ValuesYAML: params.HelmValues, @@ -42,6 +28,12 @@ func NewOperator(e config.Env, resourceName string, kubeProvider *kubernetes.Pro return err } + comp.Operator, err = compkubernetes.NewKubernetesObjRef(e, "datadog-operator", params.Namespace, "Pod", release.LinuxHelmReleaseStatus.AppVersion().Elem(), release.LinuxHelmReleaseStatus.Version().Elem(), map[string]string{"app.kubernetes.io/name": "datadog-operator"}) + + if err != nil { + return err + } + return nil }) } diff --git a/components/datadog/operatorparams/params.go b/components/datadog/operatorparams/params.go index 188fb01b3..5a54cd7d4 100644 --- a/components/datadog/operatorparams/params.go +++ b/components/datadog/operatorparams/params.go @@ -11,11 +11,11 @@ import ( ) type Params struct { - // OperatorFullImagePath is the full path of the docker agent image to use. + // OperatorFullImagePath is the full path of the operator image to use. OperatorFullImagePath string - // Namespace is the namespace to deploy the agent to. + // Namespace is the namespace to deploy the operator to. Namespace string - // HelmValues is the Helm values to use for the agent installation. + // HelmValues is the Helm values to use for the operator installation. HelmValues pulumi.AssetOrArchiveArray // PulumiResourceOptions is a list of resources to depend on. PulumiResourceOptions []pulumi.ResourceOption @@ -25,8 +25,7 @@ type Option = func(*Params) error func NewParams(e config.Env, options ...Option) (*Params, error) { version := &Params{ - Namespace: "datadog", - OperatorFullImagePath: "gcr.io/datadoghq/operator:latest", + Namespace: "datadog", } if e.PipelineID() != "" && e.CommitSHA() != "" { @@ -52,11 +51,21 @@ func WithOperatorFullImagePath(path string) func(*Params) error { } } -// WithHelmValues adds helm values to the agent installation. If used several times, the helm values are merged together +// WithHelmValues adds helm values to the operator installation. If used several times, the helm values are merged together // If the same values is defined several times the latter call will override the previous one. +// Accepts a string for single-line values (e.g. installCRDs: true) or a string literal in yaml format +// for multi-line values func WithHelmValues(values string) func(*Params) error { return func(p *Params) error { p.HelmValues = append(p.HelmValues, pulumi.NewStringAsset(values)) return nil } } + +// WithPulumiResourceOptions sets the resources to depend on. +func WithPulumiResourceOptions(resources ...pulumi.ResourceOption) func(*Params) error { + return func(p *Params) error { + p.PulumiResourceOptions = append(p.PulumiResourceOptions, resources...) + return nil + } +} diff --git a/components/datadog/agent/kubernetes_object_ref.go b/components/kubernetes/kubernetes_object_ref.go similarity index 88% rename from components/datadog/agent/kubernetes_object_ref.go rename to components/kubernetes/kubernetes_object_ref.go index 7bdcf8b2e..f3ff3e1f2 100644 --- a/components/datadog/agent/kubernetes_object_ref.go +++ b/components/kubernetes/kubernetes_object_ref.go @@ -1,4 +1,4 @@ -package agent +package kubernetes import ( "github.com/pulumi/pulumi/sdk/v3/go/pulumi" @@ -7,7 +7,7 @@ import ( "github.com/DataDog/test-infra-definitions/components" ) -type KubernetesObjRefOutput struct { +type KubernetesObjRefOutput struct { // nolint:revive, We want to keep the name as ObjRefOutput components.JSONImporter Namespace string `json:"namespace"` @@ -18,7 +18,7 @@ type KubernetesObjRefOutput struct { LabelSelectors map[string]string `json:"labelSelectors"` } -type KubernetesObjectRef struct { +type KubernetesObjectRef struct { // nolint:revive, We want to keep the name as ObjectRef pulumi.ResourceState components.Component diff --git a/go.mod b/go.mod index 1e4a6e71f..3df87dc63 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,11 @@ require ( github.com/DataDog/datadog-agent/pkg/util/optional v0.59.1 github.com/Masterminds/semver v1.5.0 github.com/alessio/shellescape v1.4.2 - github.com/aws/aws-sdk-go-v2 v1.32.2 - github.com/aws/aws-sdk-go-v2/config v1.27.40 - github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 - github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 + github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/google/gofuzz v1.2.0 github.com/pulumi/pulumi-aws/sdk/v6 v6.56.1 @@ -23,8 +23,8 @@ require ( github.com/pulumi/pulumi-azure-native-sdk/network/v2 v2.73.1 github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 github.com/pulumi/pulumi-command/sdk v1.0.1 - github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 - github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 + github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 + github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 github.com/pulumi/pulumi-libvirt/sdk v0.5.3 @@ -46,21 +46,21 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.38 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 // indirect - github.com/aws/smithy-go v1.22.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/charmbracelet/bubbles v0.18.0 // indirect diff --git a/go.sum b/go.sum index f4c6ab658..e5e64d8ef 100644 --- a/go.sum +++ b/go.sum @@ -27,46 +27,46 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= -github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= -github.com/aws/aws-sdk-go-v2/config v1.27.40 h1:sie4mPBGFOO+Z27+yHzvyN31G20h/bf2xb5mCbpLv2Q= -github.com/aws/aws-sdk-go-v2/config v1.27.40/go.mod h1:4KW7Aa5tNo+0VHnuLnnE1vPHtwMurlNZNS65IdcewHA= -github.com/aws/aws-sdk-go-v2/credentials v1.17.38 h1:iM90eRhCeZtlkzCNCG1JysOzJXGYf5rx80aD1lUgNDU= -github.com/aws/aws-sdk-go-v2/credentials v1.17.38/go.mod h1:TCVYPZeQuLaYNEkf/TVn6k5k/zdVZZ7xH9po548VNNg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 h1:FKdiFzTxlTRO71p0C7VrLbkkdW8qfMKF5+ej6bTmkT0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19/go.mod h1:abO3pCj7WLQPTllnSeYImqFfkGrmJV0JovWo/gqT5N0= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2 h1:VDQaVwGOokbd3VUbHF+wupiffdrbAZPdQnr5XZMJqrs= -github.com/aws/aws-sdk-go-v2/service/ecr v1.36.2/go.mod h1:lvUlMghKYmSxSfv0vU7pdU/8jSY+s0zpG8xXhaGKCw0= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4 h1:CTkPGE8fiElvLtYWl/U+Eu5+1fVXiZbJUjyVCRSRgxk= -github.com/aws/aws-sdk-go-v2/service/ecs v1.47.4/go.mod h1:sMFLFhL27cKYa/eQYZp4asvIwHsnJWrAzTUpy9AQdnU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 h1:FQNWhRuSq8QwW74GtU0MrveNhZbqvHsA4dkA9w8fTDQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0/go.mod h1:j/zZ3zmWfGCK91K73YsfHP53BSTLSjL/y6YN39XbBLM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0 h1:AdbiDUgQZmM28rDIZbiSwFxz8+3B94aOXxzs6oH+EA0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.0/go.mod h1:uV476Bd80tiDTX4X2redMtagQUg65aU/gzPojSJ4kSI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 h1:1NKXS8XfhMM0bg5wVYa/eOH8AM2f6JijugbKEyQFTIg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0/go.mod h1:ph931DUfVfgrhZR7py9olSvHCiRpvaGxNvlWBcXxFds= -github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 h1:2dSm7frMrw2tdJ0QvyccQNJyPGaP24dyDgZ6h1QJMGU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0/go.mod h1:4XSVpw66upN8wND3JZA29eXl2NOZvfFVq7DIP6xvfuQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.23.4 h1:ck/Y8XWNR1gHa4BFkwE3oSu7XDJGwl+8TI7E/RB2EcQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.23.4/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4 h1:4f2/JKYZHAZbQ7koBpZ012bKi32NHPY0m7TDuJgsbug= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.4/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E= -github.com/aws/aws-sdk-go-v2/service/sts v1.31.4 h1:uK6dUUdJtqutK1XO/tmNaQMJiPLCJY/eAeOOmqQ6ygY= -github.com/aws/aws-sdk-go-v2/service/sts v1.31.4/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI= -github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= -github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 h1:r67ps7oHCYnflpgDy2LZU0MAQtQbYIOqNNnqGO6xQkE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25/go.mod h1:GrGY+Q4fIokYLtjCVB/aFfCVL6hhGUFl8inD18fDalE= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7 h1:R+5XKIJga2K9Dkj0/iQ6fD/MBGo02oxGGFTc512lK/Q= +github.com/aws/aws-sdk-go-v2/service/ecr v1.36.7/go.mod h1:fDPQV/6ONOQOjvtKhtypIy1wcGLcKYtoK/lvZ9fyDGQ= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2 h1:LRM6z+wmXqAgCvuH36RR+Wf8SZZhvOVjt6f5r38V2II= +github.com/aws/aws-sdk-go-v2/service/ecs v1.52.2/go.mod h1:Ghi1OWUv4+VMEULWiHsKH2gNA3KAcMoLWsvU0eRXvIA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 h1:HCpPsWqmYQieU7SS6E9HXfdAMSud0pteVXieJmcpIRI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6/go.mod h1:ngUiVRCco++u+soRRVBIvBZxSMMvOVMXA4PJ36JLfSw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 h1:BbGDtTi0T1DYlmjBiCr/le3wzhA37O8QTC5/Ab8+EXk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6/go.mod h1:hLMJt7Q8ePgViKupeymbqI0la+t9/iYFBjxQCFwuAwI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 h1:nyuzXooUNJexRT0Oy0UQY6AhOzxPxhtt4DcBIHyCnmw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0/go.mod h1:sT/iQz8JK3u/5gZkT+Hmr7GzVZehUMkRZpOaAwYXeGY= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -224,10 +224,10 @@ github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1 h1:yzXxwwq3tHdtSOi5vjKmKXq7 github.com/pulumi/pulumi-azure-native-sdk/v2 v2.73.1/go.mod h1:ChjIUNDNeN6jI33ZOivHUFqM6purDiLP01mghMGe1Fs= github.com/pulumi/pulumi-command/sdk v1.0.1 h1:ZuBSFT57nxg/fs8yBymUhKLkjJ6qmyN3gNvlY/idiN0= github.com/pulumi/pulumi-command/sdk v1.0.1/go.mod h1:C7sfdFbUIoXKoIASfXUbP/U9xnwPfxvz8dBpFodohlA= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5 h1:7OjAfgLz5PAy95ynbgPAlWls5WBe4I/QW/61TdPWRlQ= -github.com/pulumi/pulumi-docker/sdk/v4 v4.5.5/go.mod h1:XZKLFXbw13olxuztlWnmVUPYZp2a+BqzqhuMl0j/Ow8= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0 h1:F3xAOBZ/In4PqydTsKeg3tou/c5FZ+JTp5dQO0oMjqE= -github.com/pulumi/pulumi-eks/sdk/v3 v3.3.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7 h1:cuIl5YyIghqtnFMGsdtPOeaNSix5S2CrqO0/UZ1Yjsc= +github.com/pulumi/pulumi-docker/sdk/v4 v4.5.7/go.mod h1:f2ek887nKRSwNtqTqCFENJSOH0PXm1b3FhzSXYL0IyM= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0 h1:s2Cpu6E2lmADNUbutbJGm6O+O9j0mBLlrhQmc40ukt0= +github.com/pulumi/pulumi-eks/sdk/v3 v3.4.0/go.mod h1:QbAamxfUpDJC81BGtyEuV0P88RrdbOjQEhbgY+OOPpg= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0 h1:21oSj+TKlKTzQcxN9Hik7iSNNHPUQXN4s3itOnahy/w= github.com/pulumi/pulumi-gcp/sdk/v7 v7.38.0/go.mod h1:YaEZms1NgXFqGhObKVofcAeWXu2V+3t/BAXdHQZq7fU= github.com/pulumi/pulumi-kubernetes/sdk/v4 v4.18.3 h1:quqoGsLbF7lpGpGU4mi5WfVLIAo4gfvoQeYYmemx1Dg= diff --git a/requirements.txt b/requirements.txt index 64ae653e7..07666b15e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ lxml==5.3.0 python-gitlab==5.1.0 PyGithub==2.5.0 boto3>=1.28.0 +pyright==1.1.390 diff --git a/resources/local/podman/vm.go b/resources/local/podman/vm.go index 6461e513c..6ec2bf8e2 100644 --- a/resources/local/podman/vm.go +++ b/resources/local/podman/vm.go @@ -20,6 +20,7 @@ type VMArgs struct { //go:embed data/Dockerfile var dockerfileContent string +var customDockerConfig = "{}" func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.ResourceOption) (address pulumi.StringOutput, user string, port int, err error) { interpreter := []string{"/bin/bash", "-c"} @@ -49,14 +50,22 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou return pulumi.StringOutput{}, "", -1, err } + // Use a config to avoid docker hooks that can call vault or other services (credHelpers) + err = os.WriteFile(path.Join(dataPath, "config.json"), []byte(customDockerConfig), 0600) + if err != nil { + return pulumi.StringOutput{}, "", -1, err + } + + podmanCommand := "podman --config " + dataPath + opts = utils.MergeOptions(opts, e.WithProviders(config.ProviderCommand)) // TODO use NewLocalRunner // requires a refactor to pass interpreter buildPodman, err := local.NewCommand(e.Ctx(), e.CommonNamer().ResourceName("podman-build", args.Name), &local.CommandArgs{ Interpreter: pulumi.ToStringArray(interpreter), Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, - Create: pulumi.Sprintf("podman build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", args.Name), - Delete: pulumi.Sprintf("podman rmi %s", args.Name), + Create: pulumi.Sprintf("%s build --format=docker --build-arg DOCKER_HOST_SSH_PUBLIC_KEY=\"$DOCKER_HOST_SSH_PUBLIC_KEY\" -t %s .", podmanCommand, args.Name), + Delete: pulumi.Sprintf("%s rmi %s", podmanCommand, args.Name), Triggers: pulumi.Array{}, AssetPaths: pulumi.StringArray{}, Dir: pulumi.String(dataPath), @@ -68,8 +77,8 @@ func NewInstance(e resourceslocal.Environment, args VMArgs, opts ...pulumi.Resou runPodman, err := local.NewCommand(e.Ctx(), e.CommonNamer().ResourceName("podman-run", args.Name), &local.CommandArgs{ Interpreter: pulumi.ToStringArray(interpreter), Environment: pulumi.StringMap{"DOCKER_HOST_SSH_PUBLIC_KEY": pulumi.String(string(publicKey))}, - Create: pulumi.Sprintf("podman run -d --name=%[1]s_run -p 50022:22 %[1]s", args.Name), - Delete: pulumi.Sprintf("podman stop %[1]s_run && podman rm %[1]s_run", args.Name), + Create: pulumi.Sprintf("%s run -d --name=%[2]s_run -p 50022:22 %[2]s", podmanCommand, args.Name), + Delete: pulumi.Sprintf("%s stop %[2]s_run && podman rm %[2]s_run", podmanCommand, args.Name), Triggers: pulumi.Array{}, AssetPaths: pulumi.StringArray{}, Dir: pulumi.String(dataPath), diff --git a/scenarios/aws/eks/run.go b/scenarios/aws/eks/run.go index fb63d1812..dccb999bd 100644 --- a/scenarios/aws/eks/run.go +++ b/scenarios/aws/eks/run.go @@ -38,9 +38,11 @@ func Run(ctx *pulumi.Context) error { return err } - if _, err := vpa.DeployCRD(&awsEnv, cluster.KubeProvider); err != nil { + vpaCrd, err := vpa.DeployCRD(&awsEnv, cluster.KubeProvider) + if err != nil { return err } + dependsOnVPA := utils.PulumiDependsOn(vpaCrd) if awsEnv.InitOnly() { return nil @@ -111,11 +113,11 @@ func Run(ctx *pulumi.Context) error { // Deploy testing workload if awsEnv.TestingWorkloadDeploy() { - if _, err := nginx.K8sAppDefinition(&awsEnv, cluster.KubeProvider, "workload-nginx", "", true, dependsOnDDAgent /* for DDM */); err != nil { + if _, err := nginx.K8sAppDefinition(&awsEnv, cluster.KubeProvider, "workload-nginx", "", true, dependsOnDDAgent /* for DDM */, dependsOnVPA); err != nil { return err } - if _, err := redis.K8sAppDefinition(&awsEnv, cluster.KubeProvider, "workload-redis", true, dependsOnDDAgent /* for DDM */); err != nil { + if _, err := redis.K8sAppDefinition(&awsEnv, cluster.KubeProvider, "workload-redis", true, dependsOnDDAgent /* for DDM */, dependsOnVPA); err != nil { return err } diff --git a/scenarios/aws/kindvm/run.go b/scenarios/aws/kindvm/run.go index 5bc48b12f..cb1cd0e0a 100644 --- a/scenarios/aws/kindvm/run.go +++ b/scenarios/aws/kindvm/run.go @@ -15,6 +15,7 @@ import ( dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + "github.com/DataDog/test-infra-definitions/components/datadog/operator" "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" @@ -68,9 +69,11 @@ func Run(ctx *pulumi.Context) error { return err } - if _, err := vpa.DeployCRD(&awsEnv, kindKubeProvider); err != nil { + vpaCrd, err := vpa.DeployCRD(&awsEnv, kindKubeProvider) + if err != nil { return err } + dependsOnVPA := utils.PulumiDependsOn(vpaCrd) var fakeIntake *fakeintakeComp.Fakeintake if awsEnv.AgentUseFakeintake() { @@ -145,11 +148,31 @@ agents: operatorparams.WithNamespace("datadog"), ) + operatorComp, err := operator.NewOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator"), kindKubeProvider, operatorOpts...) + if err != nil { + return err + } + + if err := operatorComp.Export(awsEnv.Ctx(), nil); err != nil { + return err + } + + ddaConfig := agentwithoperatorparams.DDAConfig{ + Name: "dda-with-operator", + YamlConfig: ` +apiVersion: datadoghq.com/v2alpha1 +kind: DatadogAgent +spec: + global: + kubelet: + tlsVerify: false +`} + ddaOptions := make([]agentwithoperatorparams.Option, 0) ddaOptions = append( ddaOptions, agentwithoperatorparams.WithNamespace("datadog"), - agentwithoperatorparams.WithTLSKubeletVerify(false), + agentwithoperatorparams.WithDDAConfig(ddaConfig), ) if fakeIntake != nil { @@ -159,7 +182,7 @@ agents: ) } - operatorAgentComponent, err := agent.NewDDAWithOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("dd-operator-agent"), kindKubeProvider, operatorOpts, ddaOptions...) + k8sAgentWithOperatorComp, err := agent.NewDDAWithOperator(&awsEnv, awsEnv.CommonNamer().ResourceName("datadog-agent-with-operator"), kindKubeProvider, ddaOptions...) if err != nil { return err @@ -167,7 +190,7 @@ agents: dependsOnDDAgent = utils.PulumiDependsOn(operatorAgentComponent) - if err := operatorAgentComponent.Export(awsEnv.Ctx(), nil); err != nil { + if err := k8sAgentWithOperatorComp.Export(awsEnv.Ctx(), nil); err != nil { return err } @@ -182,11 +205,11 @@ agents: // Deploy testing workload if awsEnv.TestingWorkloadDeploy() { - if _, err := nginx.K8sAppDefinition(&awsEnv, kindKubeProvider, "workload-nginx", "", true, dependsOnDDAgent /* for DDM */); err != nil { + if _, err := nginx.K8sAppDefinition(&awsEnv, kindKubeProvider, "workload-nginx", "", true, dependsOnDDAgent /* for DDM */, dependsOnVPA); err != nil { return err } - if _, err := redis.K8sAppDefinition(&awsEnv, kindKubeProvider, "workload-redis", true, dependsOnDDAgent /* for DDM */); err != nil { + if _, err := redis.K8sAppDefinition(&awsEnv, kindKubeProvider, "workload-redis", true, dependsOnDDAgent /* for DDM */, dependsOnVPA); err != nil { return err } diff --git a/tasks/ci.py b/tasks/ci.py index d1f33cfb6..200dd8a11 100644 --- a/tasks/ci.py +++ b/tasks/ci.py @@ -9,12 +9,16 @@ @task def create_bump_pr_and_close_stale_ones_on_datadog_agent(ctx, branch: str, new_commit_sha: str, old_commit_sha: str): + is_dev_branch = False if os.getenv("CI") != "true": print("This task should only be run in CI") return if os.getenv("GITHUB_TOKEN") is None: print("GITHUB_TOKEN is not set") return + if os.getenv("CI_COMMIT_BRANCH") != "main": + print("Running on a dev branch") + is_dev_branch = True repo = Github(auth=Auth.Token(os.environ["GITHUB_TOKEN"])).get_repo("DataDog/datadog-agent") pr_body = f""" @@ -36,6 +40,10 @@ def create_bump_pr_and_close_stale_ones_on_datadog_agent(ctx, branch: str, new_c print(f"PR created: {new_pr.html_url}") + if is_dev_branch: + print("Skipping stale PRs check since this is a dev branch") + return + print("Looking for stale auto bump PRs...") issues = repo.get_issues(state="open", labels=["automatic/test-infra-bump"]) diff --git a/tasks/localpodman/vm.py b/tasks/localpodman/vm.py index 0c7b9e5dc..3da90b893 100644 --- a/tasks/localpodman/vm.py +++ b/tasks/localpodman/vm.py @@ -50,7 +50,7 @@ def create_vm( raise Exit("The field `local.publicKeyPath` is required in the config file") extra_flags = { - "ddinfra:local/defaultPublicKeyPath": cfg.get_gcp().publicKeyPath, + "ddinfra:local/defaultPublicKeyPath": cfg.get_local().publicKeyPath, } full_stack_name = deploy(