From cb9c404967d0e026ef17ed54b884f05faae6b74b Mon Sep 17 00:00:00 2001 From: kooomix Date: Wed, 3 Jan 2024 09:19:44 +0200 Subject: [PATCH 1/3] extract pod selector labels --- armometadata/k8sutils.go | 46 +++++++++++++------- armometadata/k8sutils_test.go | 79 ++++++++++++++++++++++++----------- 2 files changed, 85 insertions(+), 40 deletions(-) diff --git a/armometadata/k8sutils.go b/armometadata/k8sutils.go index 15b9476..89b561d 100644 --- a/armometadata/k8sutils.go +++ b/armometadata/k8sutils.go @@ -106,7 +106,7 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { } // ExtractMetadataFromBytes extracts metadata from the JSON bytes of a Kubernetes object -func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[string]string, map[string]string, string, string, string, string) { +func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[string]string, map[string]string, string, string, string, string, map[string]string) { // output values annotations := map[string]string{} labels := map[string]string{} @@ -115,8 +115,9 @@ func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[s resourceVersion := "" kind := "" apiVersion := "" + podSelectorMatchLabels := map[string]string{} // ujson parsing - var parent string + var parent, subParent, subParent2 string err := ujson.Walk(input, func(level int, key, value []byte) bool { switch level { case 1: @@ -129,39 +130,54 @@ func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[s } // skip everything except metadata - if !bytes.EqualFold(key, []byte(`"metadata"`)) { + if !bytes.EqualFold(key, []byte(`"metadata"`)) && !bytes.EqualFold(key, []byte(`"spec"`)) { return false } + + parent = unquote(key) case 2: - // read creationTimestamp - if bytes.EqualFold(key, []byte(`"creationTimestamp"`)) { - creationTs = unquote(value) - } - // read resourceVersion - if bytes.EqualFold(key, []byte(`"resourceVersion"`)) { - resourceVersion = unquote(value) + if parent == "metadata" { + // read creationTimestamp + if bytes.EqualFold(key, []byte(`"creationTimestamp"`)) { + creationTs = unquote(value) + } + // read resourceVersion + if bytes.EqualFold(key, []byte(`"resourceVersion"`)) { + resourceVersion = unquote(value) + } + } + // record parent for level 3 - parent = unquote(key) + subParent = unquote(key) + case 3: // read annotations - if parent == "annotations" { + if subParent == "annotations" { annotations[unquote(key)] = unquote(value) } // read labels - if parent == "labels" { + if subParent == "labels" { labels[unquote(key)] = unquote(value) } + + subParent2 = unquote(key) + case 4: // read ownerReferences - if parent == "ownerReferences" { + if subParent == "ownerReferences" { ownerReferences[unquote(key)] = unquote(value) } + if subParent2 == "matchLabels" { + podSelectorMatchLabels[unquote(key)] = unquote(value) + + } + } return true }) - return err, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion + return err, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion, podSelectorMatchLabels } func unquote(value []byte) string { diff --git a/armometadata/k8sutils_test.go b/armometadata/k8sutils_test.go index 0e4cab3..08f646b 100644 --- a/armometadata/k8sutils_test.go +++ b/armometadata/k8sutils_test.go @@ -124,16 +124,41 @@ func BoolPtr(b bool) *bool { func TestExtractMetadataFromJsonBytes(t *testing.T) { tests := []struct { - name string - want error - annotations map[string]string - labels map[string]string - ownerReferences map[string]string - creationTs string - resourceVersion string - kind string - apiVersion string + name string + want error + annotations map[string]string + labels map[string]string + ownerReferences map[string]string + creationTs string + resourceVersion string + kind string + apiVersion string + podSelectorMatchLabels map[string]string }{ + { + name: "networkpolicy_withoutmatching_labels", + annotations: map[string]string{}, + labels: map[string]string{}, + ownerReferences: map[string]string{}, + creationTs: "2023-11-16T10:12:35Z", + resourceVersion: "", + kind: "NetworkPolicy", + apiVersion: "networking.k8s.io/v1", + podSelectorMatchLabels: map[string]string{}, + }, + { + name: "networkpolicy_withmatching_labels", + annotations: map[string]string{}, + labels: map[string]string{}, + ownerReferences: map[string]string{}, + creationTs: "2023-11-16T10:12:35Z", + resourceVersion: "", + kind: "NetworkPolicy", + apiVersion: "networking.k8s.io/v1", + podSelectorMatchLabels: map[string]string{ + "role": "frontend", + }, + }, { name: "applicationactivity", annotations: map[string]string{ @@ -147,11 +172,12 @@ func TestExtractMetadataFromJsonBytes(t *testing.T) { "kubescape.io/workload-name": "storage", "kubescape.io/workload-namespace": "kubescape", }, - ownerReferences: map[string]string{}, - creationTs: "2023-11-16T10:15:05Z", - resourceVersion: "1", - kind: "ApplicationActivity", - apiVersion: "spdx.softwarecomposition.kubescape.io/v1beta1", + ownerReferences: map[string]string{}, + creationTs: "2023-11-16T10:15:05Z", + resourceVersion: "1", + kind: "ApplicationActivity", + apiVersion: "spdx.softwarecomposition.kubescape.io/v1beta1", + podSelectorMatchLabels: map[string]string{}, }, { name: "pod", @@ -178,10 +204,11 @@ func TestExtractMetadataFromJsonBytes(t *testing.T) { "name": "kubescape-549f95c69", "uid": "c0ff7d3b-4183-482c-81c5-998faf0b6150", }, - creationTs: "2023-11-16T10:12:35Z", - resourceVersion: "59348379", - kind: "Pod", - apiVersion: "v1", + creationTs: "2023-11-16T10:12:35Z", + resourceVersion: "59348379", + kind: "Pod", + apiVersion: "v1", + podSelectorMatchLabels: map[string]string{}, }, { name: "sbom", @@ -193,18 +220,19 @@ func TestExtractMetadataFromJsonBytes(t *testing.T) { "kubescape.io/image-id": "quay-io-kubescape-kubescape-sha256-608b85d3de51caad84a2bfe089ec", "kubescape.io/image-name": "quay-io-kubescape-kubescape", }, - ownerReferences: map[string]string{}, - creationTs: "2023-11-16T10:13:40Z", - resourceVersion: "1", - kind: "SBOMSPDXv2p3", - apiVersion: "spdx.softwarecomposition.kubescape.io/v1beta1", + ownerReferences: map[string]string{}, + creationTs: "2023-11-16T10:13:40Z", + resourceVersion: "1", + kind: "SBOMSPDXv2p3", + apiVersion: "spdx.softwarecomposition.kubescape.io/v1beta1", + podSelectorMatchLabels: map[string]string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { input, err := os.ReadFile(fmt.Sprintf("testdata/%s.json", tt.name)) assert.NoError(t, err) - got, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion := ExtractMetadataFromJsonBytes(input) + got, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion, podSelectorMatchLabels := ExtractMetadataFromJsonBytes(input) assert.Equal(t, tt.want, got) assert.Equal(t, tt.annotations, annotations) assert.Equal(t, tt.labels, labels) @@ -213,6 +241,7 @@ func TestExtractMetadataFromJsonBytes(t *testing.T) { assert.Equal(t, tt.resourceVersion, resourceVersion) assert.Equal(t, tt.kind, kind) assert.Equal(t, tt.apiVersion, apiVersion) + assert.Equal(t, tt.podSelectorMatchLabels, podSelectorMatchLabels) }) } } @@ -221,6 +250,6 @@ func BenchmarkExtractMetadataFromJsonBytes(b *testing.B) { input, err := os.ReadFile("testdata/applicationactivity.json") assert.NoError(b, err) for i := 0; i < b.N; i++ { - _, _, _, _, _, _, _, _ = ExtractMetadataFromJsonBytes(input) + _, _, _, _, _, _, _, _, _ = ExtractMetadataFromJsonBytes(input) } } From 3a908c5130e17e8fe77ccf895fca8d96d477ce0f Mon Sep 17 00:00:00 2001 From: kooomix Date: Wed, 3 Jan 2024 09:23:44 +0200 Subject: [PATCH 2/3] push files --- .../networkpolicy_withmatching_labels.json | 31 +++++++++++++++++++ .../networkpolicy_withoutmatching_labels.json | 23 ++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 armometadata/testdata/networkpolicy_withmatching_labels.json create mode 100644 armometadata/testdata/networkpolicy_withoutmatching_labels.json diff --git a/armometadata/testdata/networkpolicy_withmatching_labels.json b/armometadata/testdata/networkpolicy_withmatching_labels.json new file mode 100644 index 0000000..caabc92 --- /dev/null +++ b/armometadata/testdata/networkpolicy_withmatching_labels.json @@ -0,0 +1,31 @@ +{ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "creationTimestamp": "2023-11-16T10:12:35Z", + "name": "allow-frontend-backend", + "namespace": "default" + }, + "spec": { + "podSelector": { + "matchLabels": { + "role": "frontend" + } + }, + "policyTypes": ["Ingress"], + "ingress": [ + { + "from": [ + { + "podSelector": { + "matchLabels": { + "role": "backend" + } + } + } + ] + } + ] + } + } + \ No newline at end of file diff --git a/armometadata/testdata/networkpolicy_withoutmatching_labels.json b/armometadata/testdata/networkpolicy_withoutmatching_labels.json new file mode 100644 index 0000000..640dba3 --- /dev/null +++ b/armometadata/testdata/networkpolicy_withoutmatching_labels.json @@ -0,0 +1,23 @@ +{ + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "creationTimestamp": "2023-11-16T10:12:35Z", + "name": "allow-all-in-namespace", + "namespace": "default" + }, + "spec": { + "podSelector": {}, + "policyTypes": ["Ingress"], + "ingress": [ + { + "from": [ + { + "podSelector": {} + } + ] + } + ] + } + } + \ No newline at end of file From 5b9c1abb8bfb77ed8845bc2d1ce21e3a54fa2ba4 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Wed, 3 Jan 2024 08:48:36 +0100 Subject: [PATCH 3/3] add struct for metadata Signed-off-by: Matthias Bertschy --- armometadata/k8sutils.go | 48 ++++++++++++++++++++--------------- armometadata/k8sutils_test.go | 24 +++++++++--------- 2 files changed, 40 insertions(+), 32 deletions(-) diff --git a/armometadata/k8sutils.go b/armometadata/k8sutils.go index 89b561d..21b4211 100644 --- a/armometadata/k8sutils.go +++ b/armometadata/k8sutils.go @@ -105,31 +105,40 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { return config, err } +type Metadata struct { + Annotations map[string]string + Labels map[string]string + OwnerReferences map[string]string + CreationTimestamp string + ResourceVersion string + Kind string + ApiVersion string + PodSelectorMatchLabels map[string]string +} + // ExtractMetadataFromBytes extracts metadata from the JSON bytes of a Kubernetes object -func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[string]string, map[string]string, string, string, string, string, map[string]string) { +func ExtractMetadataFromJsonBytes(input []byte) (Metadata, error) { // output values - annotations := map[string]string{} - labels := map[string]string{} - ownerReferences := map[string]string{} - creationTs := "" - resourceVersion := "" - kind := "" - apiVersion := "" - podSelectorMatchLabels := map[string]string{} + m := Metadata{ + Annotations: map[string]string{}, + Labels: map[string]string{}, + OwnerReferences: map[string]string{}, + PodSelectorMatchLabels: map[string]string{}, + } // ujson parsing var parent, subParent, subParent2 string err := ujson.Walk(input, func(level int, key, value []byte) bool { switch level { case 1: if bytes.EqualFold(key, []byte(`"kind"`)) { - kind = unquote(value) + m.Kind = unquote(value) } if bytes.EqualFold(key, []byte(`"apiVersion"`)) { - apiVersion = unquote(value) + m.ApiVersion = unquote(value) } - // skip everything except metadata + // skip everything except metadata and spec if !bytes.EqualFold(key, []byte(`"metadata"`)) && !bytes.EqualFold(key, []byte(`"spec"`)) { return false } @@ -139,11 +148,11 @@ func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[s if parent == "metadata" { // read creationTimestamp if bytes.EqualFold(key, []byte(`"creationTimestamp"`)) { - creationTs = unquote(value) + m.CreationTimestamp = unquote(value) } // read resourceVersion if bytes.EqualFold(key, []byte(`"resourceVersion"`)) { - resourceVersion = unquote(value) + m.ResourceVersion = unquote(value) } } @@ -154,11 +163,11 @@ func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[s case 3: // read annotations if subParent == "annotations" { - annotations[unquote(key)] = unquote(value) + m.Annotations[unquote(key)] = unquote(value) } // read labels if subParent == "labels" { - labels[unquote(key)] = unquote(value) + m.Labels[unquote(key)] = unquote(value) } subParent2 = unquote(key) @@ -166,18 +175,17 @@ func ExtractMetadataFromJsonBytes(input []byte) (error, map[string]string, map[s case 4: // read ownerReferences if subParent == "ownerReferences" { - ownerReferences[unquote(key)] = unquote(value) + m.OwnerReferences[unquote(key)] = unquote(value) } if subParent2 == "matchLabels" { - podSelectorMatchLabels[unquote(key)] = unquote(value) - + m.PodSelectorMatchLabels[unquote(key)] = unquote(value) } } return true }) - return err, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion, podSelectorMatchLabels + return m, err } func unquote(value []byte) string { diff --git a/armometadata/k8sutils_test.go b/armometadata/k8sutils_test.go index 08f646b..0b3c758 100644 --- a/armometadata/k8sutils_test.go +++ b/armometadata/k8sutils_test.go @@ -125,7 +125,7 @@ func BoolPtr(b bool) *bool { func TestExtractMetadataFromJsonBytes(t *testing.T) { tests := []struct { name string - want error + wantErr error annotations map[string]string labels map[string]string ownerReferences map[string]string @@ -232,16 +232,16 @@ func TestExtractMetadataFromJsonBytes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { input, err := os.ReadFile(fmt.Sprintf("testdata/%s.json", tt.name)) assert.NoError(t, err) - got, annotations, labels, ownerReferences, creationTs, resourceVersion, kind, apiVersion, podSelectorMatchLabels := ExtractMetadataFromJsonBytes(input) - assert.Equal(t, tt.want, got) - assert.Equal(t, tt.annotations, annotations) - assert.Equal(t, tt.labels, labels) - assert.Equal(t, tt.ownerReferences, ownerReferences) - assert.Equal(t, tt.creationTs, creationTs) - assert.Equal(t, tt.resourceVersion, resourceVersion) - assert.Equal(t, tt.kind, kind) - assert.Equal(t, tt.apiVersion, apiVersion) - assert.Equal(t, tt.podSelectorMatchLabels, podSelectorMatchLabels) + m, err := ExtractMetadataFromJsonBytes(input) + assert.Equal(t, tt.wantErr, err) + assert.Equal(t, tt.annotations, m.Annotations) + assert.Equal(t, tt.labels, m.Labels) + assert.Equal(t, tt.ownerReferences, m.OwnerReferences) + assert.Equal(t, tt.creationTs, m.CreationTimestamp) + assert.Equal(t, tt.resourceVersion, m.ResourceVersion) + assert.Equal(t, tt.kind, m.Kind) + assert.Equal(t, tt.apiVersion, m.ApiVersion) + assert.Equal(t, tt.podSelectorMatchLabels, m.PodSelectorMatchLabels) }) } } @@ -250,6 +250,6 @@ func BenchmarkExtractMetadataFromJsonBytes(b *testing.B) { input, err := os.ReadFile("testdata/applicationactivity.json") assert.NoError(b, err) for i := 0; i < b.N; i++ { - _, _, _, _, _, _, _, _, _ = ExtractMetadataFromJsonBytes(input) + _, _ = ExtractMetadataFromJsonBytes(input) } }