diff --git a/cmd/dataset/command.go b/cmd/dataset/command.go index 62e4f7a..8231760 100644 --- a/cmd/dataset/command.go +++ b/cmd/dataset/command.go @@ -27,24 +27,26 @@ import ( func NewDatasetCmd() *cobra.Command { exec := &cobra.Command{ - Use: "dataset $pncBaseUrl $indyBaseUrl $groupBuildId", - Short: "To generate test dataset from any PNC successful group build", + Use: "dataset $pncBaseUrl $indyBaseUrl $buildId", + Short: "To generate test dataset from any PNC successful group build.", Example: "dataset https://orch-stage.xyz.com http://indy-admin-stage.xyz.com 2836", Run: func(cmd *cobra.Command, args []string) { if !validate(args) { cmd.Help() os.Exit(1) } - dataset.Run(args[0], args[1], args[2]) + groupBuild, _ := cmd.Flags().GetBool("groupBuild") + dataset.Run(args[0], args[1], args[2], groupBuild) }, } + exec.Flags().BoolP("groupBuild", "g", false, "Is group build.") return exec } func validate(args []string) bool { if len(args) < 3 { - fmt.Printf("there are 3 mandatory arguments: pncBaseUrl, indyBaseUrl, groupBuildId!\n\n") + fmt.Printf("there are 3 mandatory arguments: pncBaseUrl, indyBaseUrl, buildId!\n\n") return false } return true diff --git a/go.mod b/go.mod index 14c0f70..ace6209 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/commonjava/indy-tests go 1.14 require ( + github.com/go-git/go-git/v5 v5.4.2 github.com/go-openapi/strfmt v0.20.1 // indirect github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/mattn/go-runewidth v0.0.13 // indirect github.com/sirupsen/logrus v1.4.2 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v0.0.3 - github.com/go-git/go-git/v5 v5.4.2 + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 7be736e..ad9748b 100644 --- a/go.sum +++ b/go.sum @@ -228,5 +228,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/dataset/dep_graph.go b/pkg/dataset/dep_graph.go new file mode 100644 index 0000000..49b1df1 --- /dev/null +++ b/pkg/dataset/dep_graph.go @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2011-2021 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dataset + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +type DepGraph struct { + Vertices map[string]interface{} `json:"vertices"` + Edges []Edge `json:"edges"` +} +type Edge struct { + Source string + Target string +} + +type BuildQueue struct { + Builds []Build +} + +func (bq *BuildQueue) append(b Build) { + bq.Builds = append(bq.Builds, b) +} + +type Build struct { + Id string + Items []string `yaml:"build"` +} + +func (b *Build) append(item string) { + b.Items = append(b.Items, item) +} + +func getBuildQueueAsYaml(edges []Edge) string { + fmt.Printf("Make build queue, edges.len: %d\n", len(edges)) + bQueue := getBuildQueue(edges) + d, _ := yaml.Marshal(&bQueue.Builds) + var s = string(d) + fmt.Println(s) + return s +} + +func getBuildQueue(edges []Edge) BuildQueue { + var bQueue BuildQueue + level := 1 + roots := make(map[string]bool) + for ok := true; ok; ok = (len(edges) > 0) { + keys, values := getKeysAndValues(edges) + s := make(map[string]bool) + //fmt.Printf("Keys: %s\n", strings.Join(keys, " ")) + //fmt.Printf("level %d:\n", level) + for index, v := range edges { + if !contains(keys, v.Target) { + s[v.Target] = true + if isRoot(v.Source, values) { + roots[v.Source] = true + } + edges[index].Target = "" // mark it + } + } + bQueue.append(makeBuild(level, s)) + edges = cleanUp(edges) + //fmt.Printf("After clean up, len: %d\n", len(edges)) + level += 1 + } + //fmt.Printf("level %d:\n", level) + bQueue.append(makeBuild(level, roots)) + return bQueue +} + +func makeBuild(level int, m map[string]bool) Build { + var rb Build + rb.Id = "id" + fmt.Sprint(level) + for k := range m { + rb.append(k) + } + return rb +} + +// A source is root if it is not in targets +func isRoot(source string, targets []string) bool { + return !contains(targets, source) +} + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +func getKeysAndValues(edges []Edge) ([]string, []string) { + var keys []string + var values []string + for _, v := range edges { + keys = append(keys, v.Source) + values = append(values, v.Target) + } + return keys, values +} + +// Remove edges with empty target +func cleanUp(edges []Edge) []Edge { + var ret []Edge + for _, v := range edges { + if v.Target != "" { + ret = append(ret, v) + } + } + return ret +} diff --git a/pkg/dataset/dep_graph_test.go b/pkg/dataset/dep_graph_test.go new file mode 100644 index 0000000..2dd58c0 --- /dev/null +++ b/pkg/dataset/dep_graph_test.go @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2011-2021 Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dataset + +import ( + "testing" +) + +func Test_getBuildQueue(t *testing.T) { + type args struct { + edges []Edge + } + + tests := []struct { + name string + args args + want string + }{ + // TODO: Add test cases. + {name: "test", args: args{edges}, want: "ok"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getBuildQueue(tt.args.edges) + size := len(got.Builds) + if len(got.Builds) != 5 { + t.Errorf("getBuildQueue(), size: %d, want: %d", size, 5) + } + var total int + for _, b := range got.Builds { + total += len(b.Items) + } + if total != 23 { + t.Errorf("getBuildQueue(), total: %d, want: %d", total, 23) + } + }) + } +} + +var edges = []Edge{ + { + Source: "90440", + Target: "90447", + }, + { + Source: "90439", + Target: "90446", + }, + { + Source: "90440", + Target: "90439", + }, + { + Source: "90450", + Target: "90440", + }, + { + Source: "90452", + Target: "90446", + }, + { + Source: "90451", + Target: "90452", + }, + { + Source: "90455", + Target: "90451", + }, + { + Source: "90444", + Target: "90446", + }, + { + Source: "90441", + Target: "90444", + }, + { + Source: "90441", + Target: "90439", + }, + { + Source: "90453", + Target: "90441", + }, + { + Source: "90434", + Target: "90444", + }, + { + Source: "90453", + Target: "90434", + }, + { + Source: "90453", + Target: "90439", + }, + { + Source: "90450", + Target: "90453", + }, + { + Source: "90445", + Target: "90448", + }, + { + Source: "90450", + Target: "90445", + }, + { + Source: "90442", + Target: "90443", + }, + { + Source: "90442", + Target: "90437", + }, + { + Source: "90436", + Target: "90442", + }, + { + Source: "90436", + Target: "90452", + }, + { + Source: "90436", + Target: "90438", + }, + { + Source: "90438", + Target: "90452", + }, + { + Source: "90454", + Target: "90433", + }, + { + Source: "90455", + Target: "90439", + }, + { + Source: "90435", + Target: "90446", + }, + { + Source: "90454", + Target: "90435", + }, + { + Source: "90454", + Target: "90439", + }, + { + Source: "90436", + Target: "90439", + }, + { + Source: "90436", + Target: "90449", + }, +} diff --git a/pkg/dataset/run.go b/pkg/dataset/run.go index e210ad5..b269601 100644 --- a/pkg/dataset/run.go +++ b/pkg/dataset/run.go @@ -67,22 +67,18 @@ const ( * |-- da.json => same as above * |-- tracking.json => same as above */ -func Run(pncBaseUrl, indyBaseUrl, buildId string) { +func Run(pncBaseUrl, indyBaseUrl, buildId string, isGroupBuild bool) { //Create folder, e.g, 'dataset/2836' dirLoc := path.Join(DATASET_DIR, buildId) err := os.MkdirAll(dirLoc, 0755) common.RePanic(err) - //Check if this is a group build - isGroupBuild := false - buildURL := pncBaseUrl + "/pnc-rest/v2/group-builds/" + buildId - if common.HttpExists(buildURL) { - isGroupBuild = true - } + buildURL := "" //Download group-build.json or build.json var buildJsonFileLoc string if isGroupBuild { + buildURL = pncBaseUrl + "/pnc-rest/v2/group-builds/" + buildId buildJsonFileLoc = path.Join(dirLoc, "group-build.json") } else { buildURL = pncBaseUrl + "/pnc-rest/v2/builds/" + buildId @@ -121,12 +117,29 @@ func Run(pncBaseUrl, indyBaseUrl, buildId string) { os.MkdirAll(buildsDir, 0755) //Parse dependency-graph.json to generate data for each bc - parseDependency(pncBaseUrl, indyBaseUrl, buildsDir, dependencyGraphFileLoc) + result := parseDependency(pncBaseUrl, indyBaseUrl, buildsDir, dependencyGraphFileLoc) + + // Iterate through builds and generate files + for k := range result.Vertices { + buildId := k + buildDir := path.Join(buildsDir, buildId) + generateFile(pncBaseUrl, indyBaseUrl, buildDir, buildId) + } + + buildQueueFileLoc := path.Join(dirLoc, "build-queue.yaml") + generateBuildQueueFile(buildQueueFileLoc, result.Edges) } else { generateFile(pncBaseUrl, indyBaseUrl, dirLoc, buildId) } } +func generateBuildQueueFile(fileLoc string, edges []Edge) { + if !common.FileOrDirExists(fileLoc) { + err := ioutil.WriteFile(fileLoc, []byte(getBuildQueueAsYaml(edges)), 0644) + common.RePanic(err) + } +} + func downloadFileIfNotExist(url, fileLoc string) bool { if !common.FileOrDirExists(fileLoc) { success := common.DownloadFile(url, fileLoc) @@ -190,25 +203,13 @@ func parseBuildJson(fileLoc string) (bool, string) { return temporaryBuild, buildType } -func parseDependency(pncBaseUrl, indyBaseUrl, buildsDir, fileLoc string) { +func parseDependency(pncBaseUrl, indyBaseUrl, buildsDir, fileLoc string) DepGraph { // Read jsonFile byteValue := common.ReadByteFromFile(fileLoc) - // Parse it - var result map[string]interface{} + var result DepGraph json.Unmarshal([]byte(byteValue), &result) - - // Iterate through builds and generate files - vertices := result["vertices"] - v := reflect.ValueOf(vertices) - if v.Kind() == reflect.Map { - for _, key := range v.MapKeys() { - //val := v.MapIndex(key) - buildId := key.String() - buildDir := path.Join(buildsDir, buildId) - generateFile(pncBaseUrl, indyBaseUrl, buildDir, buildId) - } - } + return result } func generateFile(pncBaseUrl, indyBaseUrl, buildDir, buildId string) {