From 715e8b31c5508aab4d87ea548d539f80d2e2a371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9na=C3=AFc=20Huard?= Date: Thu, 12 Dec 2024 12:09:45 +0100 Subject: [PATCH] Add `VerticalPodAutoscaler`s resources (#1282) --- common/utils/yaml.go | 8 ++ components/datadog/agent/kubernetes_helm.go | 37 +++++++ components/datadog/apps/nginx/k8s.go | 57 ++++++---- components/datadog/apps/redis/k8s.go | 57 ++++++---- components/kubernetes/vpa/vpa.go | 114 ++++++++++++++++++++ scenarios/aws/eks/run.go | 5 + scenarios/aws/kindvm/run.go | 5 + 7 files changed, 247 insertions(+), 36 deletions(-) create mode 100644 components/kubernetes/vpa/vpa.go diff --git a/common/utils/yaml.go b/common/utils/yaml.go index 639549461..b3a07a5d2 100644 --- a/common/utils/yaml.go +++ b/common/utils/yaml.go @@ -5,6 +5,14 @@ import ( "gopkg.in/yaml.v3" ) +func YAMLMustMarshal(v any) string { + b, err := yaml.Marshal(v) + if err != nil { + panic(err) + } + return string(b) +} + func MergeYAML(oldValuesYamlContent string, newValuesYamlContent string) (string, error) { return mergeYAML(oldValuesYamlContent, newValuesYamlContent, false) } diff --git a/components/datadog/agent/kubernetes_helm.go b/components/datadog/agent/kubernetes_helm.go index ff3bfd0db..94c6ea7c7 100644 --- a/components/datadog/agent/kubernetes_helm.go +++ b/components/datadog/agent/kubernetes_helm.go @@ -412,6 +412,43 @@ func buildLinuxHelmValues(baseName, agentImagePath, agentImageTag, clusterAgentI "value": pulumi.String("true"), }, }, + "confd": pulumi.StringMap{ + "kubernetes_state_core.yaml": pulumi.String(utils.YAMLMustMarshal(map[string]interface{}{ + "init_config": nil, + "instances": []map[string]interface{}{ + { + "collectors": []string{ + "secrets", + "configmaps", + "nodes", + "pods", + "services", + "resourcequotas", + "replicationcontrollers", + "limitranges", + "persistentvolumeclaims", + "persistentvolumes", + "namespaces", + "endpoints", + "daemonsets", + "deployments", + "replicasets", + "statefulsets", + "cronjobs", + "jobs", + "horizontalpodautoscalers", + "poddisruptionbudgets", + "storageclasses", + "volumeattachments", + "ingresses", + "verticalpodautoscalers", + }, + "labels_as_tags": map[string]interface{}{}, + "annotations_as_tags": map[string]interface{}{}, + }, + }, + })), + }, }, "clusterChecksRunner": pulumi.Map{ "enabled": pulumi.Bool(true), diff --git a/components/datadog/apps/nginx/k8s.go b/components/datadog/apps/nginx/k8s.go index c0ffb2775..50fee6c18 100644 --- a/components/datadog/apps/nginx/k8s.go +++ b/components/datadog/apps/nginx/k8s.go @@ -1,14 +1,13 @@ package nginx import ( - "fmt" - "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/utils" componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/Masterminds/semver" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1" autoscalingv2 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/autoscaling/v2" autoscalingv2beta2 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/autoscaling/v2beta2" @@ -16,7 +15,6 @@ import ( metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" policyv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1" policyv1beta1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1beta1" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -214,21 +212,19 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } if withDatadogAutoscaling { - ddm, err := yaml.NewConfigGroup(e.Ctx(), namespace+"/nginx", &yaml.ConfigGroupArgs{ - Objs: []map[string]any{ - { - "apiVersion": "datadoghq.com/v1alpha1", - "kind": "DatadogMetric", - "metadata": map[string]any{ - "name": "nginx", - "namespace": namespace, - "labels": map[string]string{ - "app": "nginx", - }, - }, - "spec": map[string]any{ - "query": fmt.Sprintf("avg:nginx.net.request_per_s{kube_cluster_name:%%%%tag_kube_cluster_name%%%%,kube_namespace:%s,kube_deployment:nginx}.rollup(60)", namespace), - }, + ddm, err := apiextensions.NewCustomResource(e.Ctx(), namespace+"/nginx", &apiextensions.CustomResourceArgs{ + ApiVersion: pulumi.String("datadoghq.com/v1alpha1"), + Kind: pulumi.String("DatadogMetric"), + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String("nginx"), + Namespace: pulumi.String(namespace), + Labels: pulumi.StringMap{ + "app": pulumi.String("nginx"), + }, + }, + OtherFields: map[string]interface{}{ + "spec": pulumi.Map{ + "query": pulumi.Sprintf("avg:nginx.net.request_per_s{kube_cluster_name:%%%%tag_kube_cluster_name%%%%,kube_namespace:%s,kube_deployment:nginx}.rollup(60)", namespace), }, }, }, opts...) @@ -321,6 +317,31 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } } + if _, err := apiextensions.NewCustomResource(e.Ctx(), namespace+"/nginx", &apiextensions.CustomResourceArgs{ + ApiVersion: pulumi.String("autoscaling.k8s.io/v1beta2"), + Kind: pulumi.String("VerticalPodAutoscaler"), + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String("nginx"), + Namespace: pulumi.String(namespace), + Labels: pulumi.StringMap{ + "app": pulumi.String("nginx"), + }, + }, + OtherFields: map[string]interface{}{ + "spec": pulumi.Map{ + "targetRef": pulumi.Map{ + "apiVersion": pulumi.String("apps/v1"), + "kind": pulumi.String("Deployment"), + "name": pulumi.String("nginx"), + }, + "updatePolicy": pulumi.Map{ + "updateMode": pulumi.String("Auto"), + }, + }, + }, + }, opts...); err != nil { + return nil, err + } } if _, err := corev1.NewService(e.Ctx(), namespace+"/nginx", &corev1.ServiceArgs{ diff --git a/components/datadog/apps/redis/k8s.go b/components/datadog/apps/redis/k8s.go index 5c1a2acfd..de27740d5 100644 --- a/components/datadog/apps/redis/k8s.go +++ b/components/datadog/apps/redis/k8s.go @@ -1,14 +1,13 @@ package redis import ( - "fmt" - "github.com/DataDog/test-infra-definitions/common/config" "github.com/DataDog/test-infra-definitions/common/utils" componentskube "github.com/DataDog/test-infra-definitions/components/kubernetes" "github.com/Masterminds/semver" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions" appsv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apps/v1" autoscalingv2 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/autoscaling/v2" autoscalingv2beta2 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/autoscaling/v2beta2" @@ -16,7 +15,6 @@ import ( metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" policyv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1" policyv1beta1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1beta1" - "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/yaml" "github.com/pulumi/pulumi/sdk/v3/go/pulumi" ) @@ -157,21 +155,19 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } if withDatadogAutoscaling { - ddm, err := yaml.NewConfigGroup(e.Ctx(), "redis", &yaml.ConfigGroupArgs{ - Objs: []map[string]any{ - { - "apiVersion": "datadoghq.com/v1alpha1", - "kind": "DatadogMetric", - "metadata": map[string]any{ - "name": "redis", - "namespace": namespace, - "labels": map[string]string{ - "app": "redis", - }, - }, - "spec": map[string]any{ - "query": fmt.Sprintf("avg:redis.net.instantaneous_ops_per_sec{kube_cluster_name:%%%%tag_kube_cluster_name%%%%,kube_namespace:%s,kube_deployment:redis}.rollup(60)", namespace), - }, + ddm, err := apiextensions.NewCustomResource(e.Ctx(), "redis", &apiextensions.CustomResourceArgs{ + ApiVersion: pulumi.String("datadoghq.com/v1alpha1"), + Kind: pulumi.String("DatadogMetric"), + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String("redis"), + Namespace: pulumi.String(namespace), + Labels: pulumi.StringMap{ + "app": pulumi.String("redis"), + }, + }, + OtherFields: map[string]interface{}{ + "spec": pulumi.Map{ + "query": pulumi.Sprintf("avg:redis.net.instantaneous_ops_per_sec{kube_cluster_name:%%%%tag_kube_cluster_name%%%%,kube_namespace:%s,kube_deployment:redis}.rollup(60)", namespace), }, }, }, opts...) @@ -254,6 +250,31 @@ func K8sAppDefinition(e config.Env, kubeProvider *kubernetes.Provider, namespace } } + if _, err := apiextensions.NewCustomResource(e.Ctx(), "redis", &apiextensions.CustomResourceArgs{ + ApiVersion: pulumi.String("autoscaling.k8s.io/v1beta2"), + Kind: pulumi.String("VerticalPodAutoscaler"), + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String("redis"), + Namespace: pulumi.String(namespace), + Labels: pulumi.StringMap{ + "app": pulumi.String("redis"), + }, + }, + OtherFields: map[string]interface{}{ + "spec": pulumi.Map{ + "targetRef": pulumi.Map{ + "apiVersion": pulumi.String("apps/v1"), + "kind": pulumi.String("Deployment"), + "name": pulumi.String("redis"), + }, + "updatePolicy": pulumi.Map{ + "updateMode": pulumi.String("Auto"), + }, + }, + }, + }, opts...); err != nil { + return nil, err + } } if _, err := corev1.NewService(e.Ctx(), "redis", &corev1.ServiceArgs{ diff --git a/components/kubernetes/vpa/vpa.go b/components/kubernetes/vpa/vpa.go new file mode 100644 index 000000000..e3363749f --- /dev/null +++ b/components/kubernetes/vpa/vpa.go @@ -0,0 +1,114 @@ +package vpa + +import ( + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + apiextensions "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/apiextensions/v1" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + + "github.com/DataDog/test-infra-definitions/common/config" +) + +func DeployCRD(e config.Env, kubeProvider *kubernetes.Provider, opts ...pulumi.ResourceOption) (*apiextensions.CustomResourceDefinition, error) { + opts = append(opts, pulumi.Providers(kubeProvider), pulumi.DeletedWith(kubeProvider)) + + // This is the definition from https://github.com/kubernetes/autoscaler/blob/4d092e5f0afd519b082972c4656b3d52ae512b64/vertical-pod-autoscaler/deploy/vpa-v1-crd.yaml + return apiextensions.NewCustomResourceDefinition(e.Ctx(), "vertical-pod-autoscaler", &apiextensions.CustomResourceDefinitionArgs{ + Metadata: &metav1.ObjectMetaArgs{ + Name: pulumi.String("verticalpodautoscalers.autoscaling.k8s.io"), + Annotations: pulumi.StringMap{ + "api-approved.kubernetes.io": pulumi.String("https://github.com/kubernetes/kubernetes/pull/63797"), + }, + }, + Spec: &apiextensions.CustomResourceDefinitionSpecArgs{ + Group: pulumi.String("autoscaling.k8s.io"), + Scope: pulumi.String("Namespaced"), + Names: &apiextensions.CustomResourceDefinitionNamesArgs{ + Kind: pulumi.String("VerticalPodAutoscaler"), + ListKind: pulumi.String("VerticalPodAutoscalerList"), + Singular: pulumi.String("verticalpodautoscaler"), + Plural: pulumi.String("verticalpodautoscalers"), + ShortNames: pulumi.StringArray{pulumi.String("vpa")}, + }, + Versions: &apiextensions.CustomResourceDefinitionVersionArray{ + &apiextensions.CustomResourceDefinitionVersionArgs{ + Name: pulumi.String("v1beta2"), + Served: pulumi.Bool(true), + Storage: pulumi.Bool(true), + Schema: &apiextensions.CustomResourceValidationArgs{ + OpenAPIV3Schema: &apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + Properties: apiextensions.JSONSchemaPropsMap{ + "spec": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + Properties: apiextensions.JSONSchemaPropsMap{ + "targetRef": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + }, + "updatePolicy": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + Properties: apiextensions.JSONSchemaPropsMap{ + "minReplicas": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("integer"), + }, + "updateMode": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("string"), + }, + }, + }, + "resourcePolicy": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + Properties: apiextensions.JSONSchemaPropsMap{ + "containerPolicies": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("array"), + Items: &apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + Properties: apiextensions.JSONSchemaPropsMap{ + "containerName": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("string"), + }, + "controlledValue": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("string"), + Enum: pulumi.Array{ + pulumi.String("RequestsAndLimits"), + pulumi.String("RequestsOnly"), + }, + }, + "mode": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("string"), + Enum: pulumi.Array{ + pulumi.String("Auto"), + pulumi.String("Off"), + }, + }, + "minAllowed": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + }, + "maxAllowed": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("object"), + }, + "controlledResources": apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("array"), + Items: &apiextensions.JSONSchemaPropsArgs{ + Type: pulumi.String("string"), + Enum: pulumi.Array{ + pulumi.String("cpu"), + pulumi.String("memory"), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, opts...) +} diff --git a/scenarios/aws/eks/run.go b/scenarios/aws/eks/run.go index f7b13fec0..0b2aeb287 100644 --- a/scenarios/aws/eks/run.go +++ b/scenarios/aws/eks/run.go @@ -13,6 +13,7 @@ import ( dogstatsdstandalone "github.com/DataDog/test-infra-definitions/components/datadog/dogstatsd-standalone" fakeintakeComp "github.com/DataDog/test-infra-definitions/components/datadog/fakeintake" "github.com/DataDog/test-infra-definitions/components/datadog/kubernetesagentparams" + "github.com/DataDog/test-infra-definitions/components/kubernetes/vpa" resourcesAws "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/fakeintake" @@ -37,6 +38,10 @@ func Run(ctx *pulumi.Context) error { return err } + if _, err := vpa.DeployCRD(&awsEnv, cluster.KubeProvider); err != nil { + return err + } + if awsEnv.InitOnly() { return nil } diff --git a/scenarios/aws/kindvm/run.go b/scenarios/aws/kindvm/run.go index 843b2c0b2..9a89e67f6 100644 --- a/scenarios/aws/kindvm/run.go +++ b/scenarios/aws/kindvm/run.go @@ -18,6 +18,7 @@ import ( "github.com/DataDog/test-infra-definitions/components/datadog/operatorparams" localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/DataDog/test-infra-definitions/components/kubernetes/vpa" "github.com/DataDog/test-infra-definitions/components/os" resAws "github.com/DataDog/test-infra-definitions/resources/aws" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" @@ -67,6 +68,10 @@ func Run(ctx *pulumi.Context) error { return err } + if _, err := vpa.DeployCRD(&awsEnv, kindKubeProvider); err != nil { + return err + } + var dependsOnCrd pulumi.ResourceOption var fakeIntake *fakeintakeComp.Fakeintake