Allow ReplicaSet and Job metadata generators to use partial meta obje…

…cts (#109) We only use metadata from Jobs and ReplicaSets, but require that full resources are supplied. This change relaxes this requirement, allowing PartialObjectMetadata resources to be used. This allows callers to use metadata informers and avoid having to receive and deserialize non-metadata updates from the API Server. See elastic/elastic-agent#5580 for an example of how this could be used. I'm planning to add the metadata informer from that PR to this library as well. Together, these will allow us to greatly reduce memory used for processing and storing ReplicaSets and Jobs in beats and elastic-agent. This is will help elastic/elastic-agent#5580 and elastic/elastic-agent#4729 specifically, and elastic/elastic-agent#3801 in general.
elastic · Oct 3, 2024 · 70c0872 · 70c0872
1 parent 3f3a65d
commit 70c0872
Show file tree

Hide file tree

Showing 5 changed files with 87 additions and 10 deletions.
diff --git a/kubernetes/metadata/job.go b/kubernetes/metadata/job.go
@@ -18,6 +18,7 @@
 package metadata
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8s "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 
@@ -65,7 +66,7 @@ func (jb *job) GenerateECS(obj kubernetes.Resource) mapstr.M {
 
 // GenerateK8s generates job metadata from a resource object
 func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M {
-	_, ok := obj.(*kubernetes.Job)
+	_, ok := obj.(metav1.Object)
 	if !ok {
 		return nil
 	}
@@ -81,12 +82,12 @@ func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M {
 	}
 
 	if obj, ok, _ := jb.store.GetByKey(name); ok {
-		jobObj, ok := obj.(*kubernetes.Job)
+		res, ok := obj.(kubernetes.Resource)
 		if !ok {
 			return nil
 		}
 
-		return jb.GenerateK8s(jobObj, opts...)
+		return jb.GenerateK8s(res, opts...)
 	}
 
 	return nil

diff --git a/kubernetes/metadata/job_test.go b/kubernetes/metadata/job_test.go
@@ -39,7 +39,7 @@ func TestJob_Generate(t *testing.T) {
 	client := k8sfake.NewSimpleClientset()
 	boolean := true
 	tests := []struct {
-		input  kubernetes.Resource
+		input  *batchv1.Job
 		output mapstr.M
 		name   string
 	}{
@@ -92,6 +92,10 @@ func TestJob_Generate(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
+			objMeta := &metav1.PartialObjectMetadata{
+				ObjectMeta: test.input.ObjectMeta,
+			}
+			assert.Equal(t, test.output, metagen.Generate(objMeta))
 		})
 	}
 }
@@ -144,6 +148,42 @@ func TestJob_GenerateFromName(t *testing.T) {
 				"namespace": defaultNs,
 			},
 		},
+		{
+			name: "test simple object with owner",
+			input: &metav1.PartialObjectMetadata{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      name,
+					UID:       types.UID(uid),
+					Namespace: defaultNs,
+					Labels: map[string]string{
+						"foo": "bar",
+					},
+					Annotations: map[string]string{},
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: "apps",
+							Kind:       "CronJob",
+							Name:       "nginx-job",
+							UID:        "005f3b90-4b9d-12f8-acf0-31020a840144",
+							Controller: &boolean,
+						},
+					},
+				},
+			},
+			output: mapstr.M{
+				"job": mapstr.M{
+					"name": name,
+					"uid":  uid,
+				},
+				"labels": mapstr.M{
+					"foo": "bar",
+				},
+				"cronjob": mapstr.M{
+					"name": "nginx-job",
+				},
+				"namespace": defaultNs,
+			},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/kubernetes/metadata/replicaset.go b/kubernetes/metadata/replicaset.go
@@ -18,6 +18,7 @@
 package metadata
 
 import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8s "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 
@@ -67,7 +68,7 @@ func (rs *replicaset) GenerateECS(obj kubernetes.Resource) mapstr.M {
 
 // GenerateK8s generates replicaset metadata from a resource object
 func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M {
-	_, ok := obj.(*kubernetes.ReplicaSet)
+	_, ok := obj.(metav1.Object)
 	if !ok {
 		return nil
 	}
@@ -83,12 +84,12 @@ func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr
 	}
 
 	if obj, ok, _ := rs.store.GetByKey(name); ok {
-		replicaSet, ok := obj.(*kubernetes.ReplicaSet)
+		res, ok := obj.(kubernetes.Resource)
 		if !ok {
 			return nil
 		}
 
-		return rs.GenerateK8s(replicaSet, opts...)
+		return rs.GenerateK8s(res, opts...)
 	}
 
 	return nil

diff --git a/kubernetes/metadata/replicaset_test.go b/kubernetes/metadata/replicaset_test.go
@@ -40,7 +40,7 @@ func TestReplicaset_Generate(t *testing.T) {
 	client := k8sfake.NewSimpleClientset()
 	boolean := true
 	tests := []struct {
-		input  kubernetes.Resource
+		input  *appsv1.ReplicaSet
 		output mapstr.M
 		name   string
 	}{
@@ -111,6 +111,11 @@ func TestReplicaset_Generate(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			assert.Equal(t, test.output, metagen.Generate(test.input))
+			// verify this works with just the metadata
+			objMeta := &metav1.PartialObjectMetadata{
+				ObjectMeta: test.input.ObjectMeta,
+			}
+			assert.Equal(t, test.output, metagen.Generate(objMeta))
 		})
 	}
 }
@@ -181,6 +186,35 @@ func TestReplicase_GenerateFromName(t *testing.T) {
 				"namespace": defaultNs,
 			},
 		},
+		{
+			name: "test simple object with owner",
+			input: &metav1.PartialObjectMetadata{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "nginx-rs",
+					Namespace: defaultNs,
+					UID:       uid,
+					OwnerReferences: []metav1.OwnerReference{
+						{
+							APIVersion: "apps",
+							Kind:       "Deployment",
+							Name:       "nginx-deployment",
+							UID:        "005f3b90-4b9d-12f8-acf0-31020a840144",
+							Controller: &boolean,
+						},
+					},
+				},
+			},
+			output: mapstr.M{
+				"replicaset": mapstr.M{
+					"name": "nginx-rs",
+					"uid":  uid,
+				},
+				"deployment": mapstr.M{
+					"name": "nginx-deployment",
+				},
+				"namespace": defaultNs,
+			},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/kubernetes/metadata/resource.go b/kubernetes/metadata/resource.go
@@ -21,10 +21,11 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/elastic/elastic-agent-autodiscover/kubernetes"
 	"k8s.io/apimachinery/pkg/api/meta"
+
 	k8s "k8s.io/client-go/kubernetes"
 
-	"github.com/elastic/elastic-agent-autodiscover/kubernetes"
 	"github.com/elastic/elastic-agent-autodiscover/utils"
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/mapstr"
@@ -85,7 +86,7 @@ func (r *Resource) Generate(kind string, obj kubernetes.Resource, opts ...FieldO
 }
 
 // GenerateECS generates ECS metadata from a resource object
-func (r *Resource) GenerateECS(obj kubernetes.Resource) mapstr.M {
+func (r *Resource) GenerateECS(_ kubernetes.Resource) mapstr.M {
 	ecsMeta := mapstr.M{}
 	if r.clusterInfo.URL != "" {
 		_, _ = ecsMeta.Put("orchestrator.cluster.url", r.clusterInfo.URL)