Skip to content

Commit 14e817e

Browse files
rafal-slominskiipavithra-kavach
rafal-slominskii
and
pavithra-kavach
authored
NPT-689: Nexus DM Upgrade (#90)
NPT-689: Nexus DM Upgrade Modify the current datamodel installer script to incorporate the CRD upgrade We want to invoke the comparison of datamodels in nexus installer. This change adds new repository - the app takes the directory and compares crds to the ones it takes with kubernetes api. It panics when it finds any incompatibilities. Co-authored-by: pavithras <[email protected]>
1 parent d2ebde1 commit 14e817e

File tree

21 files changed

+1761
-17
lines changed

21 files changed

+1761
-17
lines changed

Diff for: ‎common-library/pkg/nexus-compare/compare.go

+14-11
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,19 @@ import (
1717
)
1818

1919
const (
20-
SpecMatch = ".*properties\\/spec.*"
21-
StatusMatch = ".*properties\\/status.*"
22-
SingletonPath = "/is_singleton"
23-
ApiGenPath = "/nexus-rest-api-gen"
20+
SpecMatch = ".*properties\\/spec.*"
21+
StatusMatch = ".*properties\\/status.*"
22+
SingletonPath = "/is_singleton"
23+
ApiGenPath = "/nexus-rest-api-gen"
24+
RequiredString = "required"
2425
)
2526

2627
func CompareFiles(data1, data2 []byte) (bool, *bytes.Buffer, error) {
2728
buffer := new(bytes.Buffer)
2829
headerColor, _ := colorful.Hex("#B9311B")
2930
fileColor, _ := colorful.Hex("#088F8F")
3031

31-
name, err := getSpecName(data1)
32+
name, err := GetSpecName(data1)
3233
if err != nil {
3334
return true, nil, err
3435
}
@@ -163,8 +164,10 @@ func filterReport(r *dyff.Report) *dyff.Report {
163164
for _, di := range r.Diffs {
164165
var ds []dyff.Detail
165166
for _, d := range di.Details {
166-
if d.From == nil && d.To != nil { // allow to add stuff
167-
continue
167+
if d.From == nil && d.To != nil && !strings.HasSuffix(di.Path.String(), fmt.Sprintf("/%s", RequiredString)) { // allow to add stuff, but not in required list
168+
if len(d.To.Content) == 0 || d.To.Content[0].Value != RequiredString { //allow to add required map field
169+
continue
170+
}
168171
}
169172
if di.Path.String() == SingletonPath && d.From.Value == strconv.FormatBool(true) && d.To.Value == strconv.FormatBool(false) { // allow singleton change to false
170173
continue
@@ -186,11 +189,11 @@ func filterReport(r *dyff.Report) *dyff.Report {
186189
}
187190

188191
func getAnnotationReport(data1, data2 []byte) (dyff.Report, error) {
189-
aNexus, err := getMapNode(data1, []string{"metadata", "annotations", "nexus"})
192+
aNexus, err := GetMapNode(data1, []string{"metadata", "annotations", "nexus"})
190193
if err != nil {
191194
return dyff.Report{}, err
192195
}
193-
bNexus, err := getMapNode(data2, []string{"metadata", "annotations", "nexus"})
196+
bNexus, err := GetMapNode(data2, []string{"metadata", "annotations", "nexus"})
194197
if err != nil {
195198
return dyff.Report{}, err
196199
}
@@ -246,7 +249,7 @@ func writeTempFile(data []byte) (*os.File, error) {
246249
return aFile, err
247250
}
248251

249-
func getSpecName(data []byte) (string, error) {
252+
func GetSpecName(data []byte) (string, error) {
250253
t := make(map[string]interface{})
251254
err := yaml.Unmarshal(data, &t)
252255
if err != nil {
@@ -255,7 +258,7 @@ func getSpecName(data []byte) (string, error) {
255258
return t["metadata"].(map[string]interface{})["name"].(string), nil
256259
}
257260

258-
func getMapNode(data []byte, path []string) (interface{}, error) {
261+
func GetMapNode(data []byte, path []string) (interface{}, error) {
259262
var t interface{}
260263
var ok bool
261264
errPath := "root"

Diff for: ‎common-library/pkg/nexus-compare/compare_test.go

+60-3
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,18 @@ var _ = Describe("Compare lib tests", func() {
3030
Expect(text.String()).Should(ContainSubstring(v))
3131
}
3232
})
33-
It("should return false when adding field", func() {
33+
It("should return false and only notice change in required", func() {
3434
ans, text, err := CompareFiles([]byte(baseSpec), []byte(addedField))
3535
Expect(err).NotTo(HaveOccurred())
36-
Expect(ans).To(BeFalse())
37-
Expect(text.String()).To(BeEmpty())
36+
Expect(ans).To(BeTrue())
37+
changeCheck := []string{"/spec/versions/name=v1/schema/openAPIV3Schema/properties/status/properties/nexus/required", "one required field added", "- testaddrequire"}
38+
changeCheckNot := []string{"/spec/versions/name=v1/schema/openAPIV3Schema/properties/status/properties/nexus/properties/addedField"}
39+
for _, v := range changeCheck {
40+
Expect(text.String()).Should(ContainSubstring(v))
41+
}
42+
for _, v := range changeCheckNot {
43+
Expect(text.String()).ShouldNot(ContainSubstring(v))
44+
}
3845
})
3946
It("should report change in nexus annotation", func() {
4047
ans, text, err := CompareFiles([]byte(baseSpec), []byte(changeAnnotation))
@@ -59,6 +66,15 @@ var _ = Describe("Compare lib tests", func() {
5966
Expect(text.String()).Should(ContainSubstring(v))
6067
}
6168
})
69+
It("it shouldn't filter out added required field", func() {
70+
ans, text, err := CompareFiles([]byte(removeRequired), []byte(baseSpec))
71+
Expect(err).ToNot(HaveOccurred())
72+
Expect(ans).To(BeTrue())
73+
changeCheck := []string{"spec changes", "/spec/versions/name=v1/schema/openAPIV3Schema/properties/spec", "one field added", "required", "name"}
74+
for _, v := range changeCheck {
75+
Expect(text.String()).Should(ContainSubstring(v))
76+
}
77+
})
6278
})
6379

6480
var baseSpec = `
@@ -228,6 +244,7 @@ spec:
228244
required:
229245
- sourceGeneration
230246
- remoteGeneration
247+
- testaddrequire
231248
type: object
232249
type: object
233250
type: object
@@ -919,3 +936,43 @@ status:
919936
- v1
920937
921938
`
939+
940+
var removeRequired = `
941+
apiVersion: apiextensions.k8s.io/v1
942+
kind: CustomResourceDefinition
943+
metadata:
944+
annotations:
945+
nexus: |
946+
{"name":"gns.IgnoreChild","hierarchy":["roots.root.tsm.tanzu.vmware.com","configs.config.tsm.tanzu.vmware.com","gnses.gns.tsm.tanzu.vmware.com"],"is_singleton":false,"nexus-rest-api-gen":{"uris":null}}
947+
creationTimestamp: null
948+
name: ignorechilds.gns.tsm.tanzu.vmware.com
949+
spec:
950+
conversion:
951+
strategy: None
952+
group: gns.tsm.tanzu.vmware.com
953+
versions:
954+
- name: v1
955+
schema:
956+
openAPIV3Schema:
957+
properties:
958+
spec:
959+
properties:
960+
name:
961+
type: string
962+
type: object
963+
status:
964+
properties:
965+
nexus:
966+
properties:
967+
remoteGeneration:
968+
format: int64
969+
type: integer
970+
sourceGeneration:
971+
format: int64
972+
type: integer
973+
required:
974+
- sourceGeneration
975+
- remoteGeneration
976+
type: object
977+
type: object
978+
type: object`

Diff for: ‎compiler/Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ ADD cmd/nexus-openapi-gen /go/bin/nexus-openapi-gen
66
ADD cmd/gqlgen /go/bin/gqlgen
77

88
WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler
9-
RUN mv .git .gitmodules kube-openapi gqlgen nexus common-library ..
9+
RUN mv .git .gitmodules kube-openapi gqlgen nexus install-validator common-library ..
1010
RUN git config --global --add safe.directory '*' && git config --global --add url."https://github.com/".insteadOf "[email protected]:"
1111
RUN make init_submodules
1212

@@ -19,6 +19,8 @@ RUN mkdir nexustmp && cd nexustmp && go mod init && \
1919
WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler/_deps/github.com/kubernetes/code-generator
2020
RUN go mod download
2121

22+
WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/install-validator
23+
RUN CGO_ENABLED=0 go build --trimpath -o install.so ./cmd/validate
2224

2325
WORKDIR /go/src/github.com/vmware-tanzu/graph-framework-for-microservices/compiler
2426
RUN go mod download

Diff for: ‎compiler/Makefile

+10-2
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ docker.builder:
5858
docker build --no-cache -t ${BUILDER_NAME}:${BUILDER_TAG} builder/
5959

6060
.PHONY: docker
61-
docker: init_submodules ${BUILDER_NAME}\:${BUILDER_TAG}.image.exists build_openapigen_in_container build_gqlgen_in_container
61+
docker: init_submodules ${BUILDER_NAME}\:${BUILDER_TAG}.image.exists build_openapigen_in_container build_gqlgen_in_container build_install_validator_in_container
6262
git archive -o compiler.tar --format=tar HEAD
63-
tar -rf compiler.tar ../.git ../.gitmodules ../gqlgen ../kube-openapi ../nexus ../common-library
63+
tar -rf compiler.tar ../.git ../.gitmodules ../gqlgen ../kube-openapi ../nexus ../install-validator ../common-library
6464
docker build --no-cache \
6565
--build-arg BUILDER_TAG=${BUILDER_TAG} \
6666
-t ${IMAGE_NAME}:${TAG} .
@@ -98,6 +98,12 @@ build_gqlgen_in_container:
9898
build_gqlgen:
9999
GOBIN=${PKG_NAME}/cmd go install ../gqlgen
100100

101+
build_install_validator_in_container:
102+
$(call run_in_container,make build_install_validator)
103+
104+
build_install_validator:
105+
cd ../install-validator && GOBIN=${PKG_NAME}/cmd go install ./cmd/validate
106+
101107
.PHONY: unit-test
102108
unit-test:
103109
ginkgo -cover ./pkg/...
@@ -178,6 +184,8 @@ generate_code:
178184
@echo "Nexus Compiler: Moving files to output directory"
179185
cp -r _generated/{client,apis,crds,common,nexus-client,helper,nexus-gql} ${GENERATED_OUTPUT_DIRECTORY}
180186
cp -r ${GOPATH}/src/nexustempmodule/nexus-gql/graphql.so ${GENERATED_OUTPUT_DIRECTORY}/nexus-gql
187+
mkdir -p ${GENERATED_OUTPUT_DIRECTORY}/install-validator
188+
cp -r ${GOPATH}/src/github.com/vmware-tanzu/graph-framework-for-microservices/install-validator/install.so ${GENERATED_OUTPUT_DIRECTORY}/install-validator/install.so || echo "Could not find install.so file, skipping"
181189
@echo "Nexus Compiler: Compiler code generation completed"
182190

183191
.PHONY: test_generate_code_in_container

Diff for: ‎install-validator/Makefile

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
.PHONY: race-unit-test
2+
race-unit-test:
3+
ginkgo -race -cover ./pkg/...
4+
5+
.PHONY: test-fmt
6+
test-fmt:
7+
test -z $$(goimports -w -l cmd pkg)
8+
9+
.PHONY: vet
10+
vet:
11+
go vet ./cmd/... ./pkg/...
12+
13+
.PHONY: lint
14+
lint:
15+
golangci-lint run ./cmd/... ./pkg/...
16+
17+
.PHONY: test
18+
test: test-fmt vet lint race-unit-test
19+
20+
.PHONY: build
21+
build:
22+
cd cmd/validate && \
23+
CGO_ENABLED=0 go build -ldflags="-w -s" .

Diff for: ‎install-validator/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# common-library

Diff for: ‎install-validator/cmd/validate/main.go

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package main
2+
3+
import (
4+
"os"
5+
"strings"
6+
7+
"github.com/sirupsen/logrus"
8+
ext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
9+
"k8s.io/client-go/dynamic"
10+
11+
nexuscompare "github.com/vmware-tanzu/graph-framework-for-microservices/common-library/pkg/nexus-compare"
12+
"github.com/vmware-tanzu/graph-framework-for-microservices/install-validator/pkg/dir"
13+
kubewrapper "github.com/vmware-tanzu/graph-framework-for-microservices/install-validator/pkg/k8s-utils"
14+
)
15+
16+
const (
17+
dirEnv = "CRD_SPEC_DIR"
18+
forceEnv = "FORCE"
19+
groupNamePath = "/NAME"
20+
)
21+
22+
func main() {
23+
directory := ""
24+
if directory = os.Getenv(dirEnv); directory == "" {
25+
directory = "/crds"
26+
}
27+
force := false
28+
if crdForceStr := os.Getenv(forceEnv); strings.ToLower(crdForceStr) == "true" {
29+
force = true
30+
}
31+
32+
// setup k8s client
33+
config, err := kubewrapper.GetRestConfig()
34+
if err != nil {
35+
logrus.Fatal(err)
36+
}
37+
clientset, err := ext.NewForConfig(config)
38+
if err != nil {
39+
logrus.Fatal(err)
40+
}
41+
dynamicClient, err := dynamic.NewForConfig(config)
42+
if err != nil {
43+
logrus.Fatal(err)
44+
}
45+
c := kubewrapper.Client{Clientset: clientset, DynamicClient: dynamicClient}
46+
err = c.FetchGroup(groupNamePath)
47+
if err != nil {
48+
logrus.Fatal(err)
49+
}
50+
51+
err = dir.ApplyDir(directory, force, &c, nexuscompare.CompareFiles)
52+
if err != nil {
53+
logrus.Fatal(err)
54+
}
55+
}

Diff for: ‎install-validator/go.mod

+79
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
module github.com/vmware-tanzu/graph-framework-for-microservices/install-validator
2+
3+
go 1.19
4+
5+
require (
6+
github.com/golang/mock v1.4.4
7+
github.com/onsi/ginkgo v1.16.5
8+
github.com/onsi/gomega v1.20.2
9+
github.com/sirupsen/logrus v1.8.1
10+
github.com/vmware-tanzu/graph-framework-for-microservices/common-library v0.0.0-20221130051552-34fc418458a3
11+
k8s.io/apiextensions-apiserver v0.25.4
12+
k8s.io/apimachinery v0.25.4
13+
k8s.io/client-go v0.25.4
14+
sigs.k8s.io/yaml v1.2.0
15+
)
16+
17+
require (
18+
github.com/BurntSushi/toml v1.2.1 // indirect
19+
github.com/davecgh/go-spew v1.1.1 // indirect
20+
github.com/emicklei/go-restful/v3 v3.8.0 // indirect
21+
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
22+
github.com/fsnotify/fsnotify v1.5.4 // indirect
23+
github.com/go-logr/logr v1.2.3 // indirect
24+
github.com/go-openapi/jsonpointer v0.19.5 // indirect
25+
github.com/go-openapi/jsonreference v0.20.0 // indirect
26+
github.com/go-openapi/swag v0.21.1 // indirect
27+
github.com/gogo/protobuf v1.3.2 // indirect
28+
github.com/golang/protobuf v1.5.2 // indirect
29+
github.com/gonvenience/bunt v1.3.4 // indirect
30+
github.com/gonvenience/neat v1.3.11 // indirect
31+
github.com/gonvenience/term v1.0.2 // indirect
32+
github.com/gonvenience/text v1.0.7 // indirect
33+
github.com/gonvenience/wrap v1.1.2 // indirect
34+
github.com/gonvenience/ytbx v1.4.4 // indirect
35+
github.com/google/gnostic v0.6.9 // indirect
36+
github.com/google/go-cmp v0.5.9 // indirect
37+
github.com/google/gofuzz v1.2.0 // indirect
38+
github.com/homeport/dyff v1.5.6 // indirect
39+
github.com/imdario/mergo v0.3.6 // indirect
40+
github.com/josharian/intern v1.0.0 // indirect
41+
github.com/json-iterator/go v1.1.12 // indirect
42+
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
43+
github.com/mailru/easyjson v0.7.6 // indirect
44+
github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
45+
github.com/mattn/go-isatty v0.0.16 // indirect
46+
github.com/mitchellh/go-ps v1.0.0 // indirect
47+
github.com/mitchellh/hashstructure v1.1.0 // indirect
48+
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
49+
github.com/modern-go/reflect2 v1.0.2 // indirect
50+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
51+
github.com/nxadm/tail v1.4.8 // indirect
52+
github.com/pkg/errors v0.9.1 // indirect
53+
github.com/sergi/go-diff v1.2.0 // indirect
54+
github.com/spf13/pflag v1.0.5 // indirect
55+
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
56+
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
57+
golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
58+
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
59+
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
60+
golang.org/x/sys v0.1.0 // indirect
61+
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
62+
golang.org/x/text v0.3.8 // indirect
63+
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
64+
google.golang.org/appengine v1.6.7 // indirect
65+
google.golang.org/protobuf v1.28.0 // indirect
66+
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect
67+
gopkg.in/inf.v0 v0.9.1 // indirect
68+
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
69+
gopkg.in/yaml.v2 v2.4.0 // indirect
70+
gopkg.in/yaml.v3 v3.0.1 // indirect
71+
k8s.io/api v0.25.4 // indirect
72+
k8s.io/klog/v2 v2.70.1 // indirect
73+
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
74+
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
75+
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
76+
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
77+
)
78+
79+
replace github.com/vmware-tanzu/graph-framework-for-microservices/common-library => ../common-library

0 commit comments

Comments
 (0)