diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/diff.go b/pkg/app/pipedv1/plugin/kubernetes/provider/diff.go index 640720c353..89407d36f8 100644 --- a/pkg/app/pipedv1/plugin/kubernetes/provider/diff.go +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/diff.go @@ -15,6 +15,10 @@ package provider import ( + "fmt" + "sort" + "strings" + "go.uber.org/zap" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -81,3 +85,143 @@ func normalizeNewSecret(old, new *unstructured.Unstructured) (*unstructured.Unst return &unstructured.Unstructured{Object: newO}, nil } + +type DiffListResult struct { + Adds []Manifest + Deletes []Manifest + Changes []DiffListChange +} + +type DiffListChange struct { + Old Manifest + New Manifest + Diff *diff.Result +} + +func (r *DiffListResult) NoChanges() bool { + return len(r.Adds)+len(r.Deletes)+len(r.Changes) == 0 +} + +func (r *DiffListResult) TotalOutOfSync() int { + return len(r.Adds) + len(r.Deletes) + len(r.Changes) +} + +func DiffList(liveManifests, desiredManifests []Manifest, logger *zap.Logger, opts ...diff.Option) (*DiffListResult, error) { + adds, deletes, newChanges, oldChanges := groupManifests(liveManifests, desiredManifests) + result := &DiffListResult{ + Adds: adds, + Deletes: deletes, + Changes: make([]DiffListChange, 0, len(newChanges)), + } + + for i := 0; i < len(newChanges); i++ { + diffResult, err := Diff(oldChanges[i], newChanges[i], logger, opts...) + if err != nil { + logger.Error("Failed to diff manifests", zap.Error(err)) + continue + } + if !diffResult.HasDiff() { + continue + } + result.Changes = append(result.Changes, DiffListChange{ + Old: oldChanges[i], + New: newChanges[i], + Diff: diffResult, + }) + } + + return result, nil +} + +func groupManifests(olds, news []Manifest) (adds, deletes, newChanges, oldChanges []Manifest) { + // Sort the manifests before comparing. + sort.Slice(news, func(i, j int) bool { + return news[i].Key().String() < news[j].Key().String() + }) + sort.Slice(olds, func(i, j int) bool { + return olds[i].Key().String() < olds[j].Key().String() + }) + + var n, o int + for { + if n >= len(news) || o >= len(olds) { + break + } + if news[n].Key().String() == olds[o].Key().String() { + newChanges = append(newChanges, news[n]) + oldChanges = append(oldChanges, olds[o]) + n++ + o++ + continue + } + // Has in news but not in olds so this should be a added one. + if news[n].Key().String() < olds[o].Key().String() { + adds = append(adds, news[n]) + n++ + continue + } + // Has in olds but not in news so this should be an deleted one. + deletes = append(deletes, olds[o]) + o++ + } + + if len(news) > n { + adds = append(adds, news[n:]...) + } + if len(olds) > o { + deletes = append(deletes, olds[o:]...) + } + return adds, deletes, newChanges, oldChanges +} + +type DiffRenderOptions struct { + MaskSecret bool + MaskConfigMap bool + // Maximum number of changed manifests should be shown. + // Zero means rendering all. + MaxChangedManifests int +} + +func (r *DiffListResult) Render(opt DiffRenderOptions) string { + var b strings.Builder + index := 0 + for _, delete := range r.Deletes { + index++ + b.WriteString(fmt.Sprintf("- %d. %s\n\n", index, delete.Key().ReadableString())) + } + for _, add := range r.Adds { + index++ + b.WriteString(fmt.Sprintf("+ %d. %s\n\n", index, add.Key().ReadableString())) + } + + maxPrintDiffs := len(r.Changes) + if opt.MaxChangedManifests != 0 && opt.MaxChangedManifests < maxPrintDiffs { + maxPrintDiffs = opt.MaxChangedManifests + } + + for _, change := range r.Changes[:maxPrintDiffs] { + key := change.Old.Key() + opts := []diff.RenderOption{ + diff.WithLeftPadding(1), + } + + if opt.MaskSecret && change.Old.IsSecret() { + opts = append(opts, diff.WithMaskPath("data")) + } else if opt.MaskConfigMap && change.Old.IsConfigMap() { + opts = append(opts, diff.WithMaskPath("data")) + } + renderer := diff.NewRenderer(opts...) + + index++ + b.WriteString(fmt.Sprintf("# %d. %s\n\n", index, key.ReadableString())) + + b.WriteString(renderer.Render(change.Diff.Nodes())) + b.WriteString("\n") + } + + if maxPrintDiffs < len(r.Changes) { + b.WriteString(fmt.Sprintf("... (omitted %d other changed manifests)\n", len(r.Changes)-maxPrintDiffs)) + } + + return b.String() +} diff --git a/pkg/app/pipedv1/plugin/kubernetes/provider/diff_test.go b/pkg/app/pipedv1/plugin/kubernetes/provider/diff_test.go index fd25a9070d..2dd8485788 100644 --- a/pkg/app/pipedv1/plugin/kubernetes/provider/diff_test.go +++ b/pkg/app/pipedv1/plugin/kubernetes/provider/diff_test.go @@ -15,6 +15,7 @@ package provider import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -22,6 +23,7 @@ import ( "go.uber.org/zap" "github.com/pipe-cd/pipecd/pkg/plugin/diff" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) func TestDiff(t *testing.T) { @@ -237,3 +239,556 @@ spec: }) } } + +func TestDiffList(t *testing.T) { + t.Parallel() + + testcases := []struct { + name string + live string + desired string + wantAdds int + wantDels int + wantMods int + }{ + { + name: "no changes", + live: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment +spec: + replicas: 3`, + desired: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment +spec: + replicas: 3`, + wantAdds: 0, + wantDels: 0, + wantMods: 0, + }, + { + name: "one addition", + live: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 3`, + desired: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2 +spec: + replicas: 3`, + wantAdds: 1, + wantDels: 0, + wantMods: 0, + }, + { + name: "one deletion", + live: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2 +spec: + replicas: 3`, + desired: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 3`, + wantAdds: 0, + wantDels: 1, + wantMods: 0, + }, + { + name: "one modification", + live: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment +spec: + replicas: 3`, + desired: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment +spec: + replicas: 5`, + wantAdds: 0, + wantDels: 0, + wantMods: 1, + }, + { + name: "mixed changes", + live: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 3 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2 +spec: + replicas: 3`, + desired: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +spec: + replicas: 5 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-3 +spec: + replicas: 3`, + wantAdds: 1, + wantDels: 1, + wantMods: 1, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + liveManifests, err := ParseManifests(tc.live) + require.NoError(t, err) + desiredManifests, err := ParseManifests(tc.desired) + require.NoError(t, err) + + result, err := DiffList(liveManifests, desiredManifests, zap.NewNop(), diff.WithEquateEmpty(), diff.WithIgnoreAddingMapKeys(), diff.WithCompareNumberAndNumericString()) + require.NoError(t, err) + + assert.Equal(t, tc.wantAdds, len(result.Adds)) + assert.Equal(t, tc.wantDels, len(result.Deletes)) + assert.Equal(t, tc.wantMods, len(result.Changes)) + assert.Equal(t, tc.wantAdds+tc.wantDels+tc.wantMods == 0, result.NoChanges()) + assert.Equal(t, tc.wantAdds+tc.wantDels+tc.wantMods, result.TotalOutOfSync()) + }) + } +} + +func TestGroupManifests(t *testing.T) { + t.Parallel() + + testcases := []struct { + name string + olds string + news string + wantAdds int + wantDeletes int + wantChanges int + }{ + { + name: "empty lists", + olds: "", + news: "", + wantAdds: 0, + wantDeletes: 0, + wantChanges: 0, + }, + { + name: "only additions", + olds: "", + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2`, + wantAdds: 2, + wantDeletes: 0, + wantChanges: 0, + }, + { + name: "only deletions", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2`, + news: "", + wantAdds: 0, + wantDeletes: 2, + wantChanges: 0, + }, + { + name: "only changes", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2`, + wantAdds: 0, + wantDeletes: 0, + wantChanges: 2, + }, + { + name: "mixed changes", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment-3`, + wantAdds: 1, + wantDeletes: 1, + wantChanges: 1, + }, + { + name: "different resource types with same name", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-1`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 +--- +apiVersion: v1 +kind: Service +metadata: + name: test-1`, + wantAdds: 0, + wantDeletes: 0, + wantChanges: 2, + }, + { + name: "different namespaces with same name", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 + namespace: ns1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 + namespace: ns2`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 + namespace: ns1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 + namespace: ns2`, + wantAdds: 0, + wantDeletes: 0, + wantChanges: 2, + }, + { + name: "old list larger than new list", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-3`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-2`, + wantAdds: 0, + wantDeletes: 2, + wantChanges: 1, + }, + { + name: "new list larger than old list", + olds: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-2`, + news: `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-3`, + wantAdds: 2, + wantDeletes: 0, + wantChanges: 1, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + var olds, news []Manifest + var err error + + if tc.olds != "" { + olds, err = ParseManifests(tc.olds) + require.NoError(t, err) + } + + if tc.news != "" { + news, err = ParseManifests(tc.news) + require.NoError(t, err) + } + + adds, deletes, newChanges, oldChanges := groupManifests(olds, news) + assert.Equal(t, tc.wantAdds, len(adds)) + assert.Equal(t, tc.wantDeletes, len(deletes)) + assert.Equal(t, tc.wantChanges, len(newChanges)) + assert.Equal(t, len(newChanges), len(oldChanges)) + }) + } +} + +func TestDiffListResult_Render(t *testing.T) { + t.Parallel() + + testcases := []struct { + name string + result *DiffListResult + opt DiffRenderOptions + expected string + }{ + { + name: "empty result", + result: &DiffListResult{ + Adds: []Manifest{}, + Deletes: []Manifest{}, + Changes: []DiffListChange{}, + }, + opt: DiffRenderOptions{}, + expected: "", + }, + { + name: "only deletes", + result: &DiffListResult{ + Deletes: []Manifest{ + { + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "test-deployment", + "namespace": "default", + }, + }, + }, + }, + }, + }, + opt: DiffRenderOptions{}, + expected: `- 1. name="test-deployment", kind="Deployment", namespace="default", apiGroup="apps"` + "\n\n", + }, + { + name: "only adds", + result: &DiffListResult{ + Adds: []Manifest{ + { + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]interface{}{ + "name": "test-deployment", + "namespace": "default", + }, + }, + }, + }, + }, + }, + opt: DiffRenderOptions{}, + expected: `+ 1. name="test-deployment", kind="Deployment", namespace="default", apiGroup="apps"` + "\n\n", + }, + { + name: "with changes", + result: func(t *testing.T) *DiffListResult { + old := Manifest{ + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": "test-secret", + "namespace": "default", + }, + "data": map[string]interface{}{ + "password": "old-password", + }, + }, + }, + } + new := Manifest{ + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]interface{}{ + "name": "test-secret", + "namespace": "default", + }, + "data": map[string]interface{}{ + "password": "new-password", + }, + }, + }, + } + diffResult, err := Diff(old, new, zap.NewNop(), diff.WithEquateEmpty(), diff.WithIgnoreAddingMapKeys()) + if err != nil { + t.Fatal(err) + } + return &DiffListResult{ + Changes: []DiffListChange{ + { + Old: old, + New: new, + Diff: diffResult, + }, + }, + } + }(t), + opt: DiffRenderOptions{ + MaskSecret: true, + }, + expected: `# 1. name="test-secret", kind="Secret", namespace="default", apiGroup=""` + "\n\n #data\n- data: *****\n+ data: *****\n\n\n", + }, + { + name: "with max changed manifests", + result: func(t *testing.T) *DiffListResult { + changes := make([]DiffListChange, 0, 2) + for i := 1; i <= 2; i++ { + old := Manifest{ + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": fmt.Sprintf("test-cm-%d", i), + "namespace": "default", + }, + "data": map[string]interface{}{ + "key": "value1", + }, + }, + }, + } + new := Manifest{ + body: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": map[string]interface{}{ + "name": fmt.Sprintf("test-cm-%d", i), + "namespace": "default", + }, + "data": map[string]interface{}{ + "key": "value2", + }, + }, + }, + } + diffResult, err := Diff(old, new, zap.NewNop(), diff.WithEquateEmpty(), diff.WithIgnoreAddingMapKeys()) + if err != nil { + t.Fatal(err) + } + changes = append(changes, DiffListChange{ + Old: old, + New: new, + Diff: diffResult, + }) + } + return &DiffListResult{ + Changes: changes, + } + }(t), + opt: DiffRenderOptions{ + MaxChangedManifests: 1, + MaskConfigMap: true, + }, + expected: `# 1. name="test-cm-1", kind="ConfigMap", namespace="default", apiGroup=""` + "\n\n data:\n #data.key\n- key: *****\n+ key: *****\n\n\n... (omitted 1 other changed manifests)\n", + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got := tc.result.Render(tc.opt) + assert.Equal(t, tc.expected, got) + }) + } +}