Merge pull request #65 from liggitt/prune

Make depstat work properly with go1.17+ go mod graphs

.github/workflows/go.yml

@@ -30,5 +30,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Run golint
-        run: make lint
+        run: |
+          export PATH=$PATH:$(go env GOPATH)/bin
+          make lint
 

Makefile

@@ -7,6 +7,6 @@ build:
 	go build 
 
 .PHONY: lint
-lint: 
-	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.39.0
-	./bin/golangci-lint run --verbose --enable gofmt
+lint:
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.39.0
+	golangci-lint run --verbose --enable gofmt

cmd/stats.go

@@ -23,6 +23,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
+var dir string
 var jsonOutput bool
 var verbose bool
 var mainModules []string
@@ -46,10 +47,8 @@ var statsCmd = &cobra.Command{
 		}
 
 		// get the longest chain
-		var longestChain Chain
 		var temp Chain
-		getLongestChain(depGraph.MainModules[0], depGraph.Graph, temp, &longestChain)
-
+		longestChain := getLongestChain(depGraph.MainModules[0], depGraph.Graph, temp, map[string]Chain{})
 		// get values
 		maxDepth := len(longestChain)
 		directDeps := len(depGraph.DirectDepList)
@@ -98,30 +97,44 @@ var statsCmd = &cobra.Command{
 }
 
 // get the longest chain starting from currentDep
-func getLongestChain(currentDep string, graph map[string][]string, currentChain Chain, longestChain *Chain) {
+func getLongestChain(currentDep string, graph map[string][]string, currentChain Chain, longestChains map[string]Chain) Chain {
+	// fmt.Println(strings.Repeat("  ", len(currentChain)), currentDep)
+
+	// already computed
+	if longestChain, ok := longestChains[currentDep]; ok {
+		return longestChain
+	}
+
+	deps := graph[currentDep]
+
+	if len(deps) == 0 {
+		// we have no dependencies, our longest chain is just us
+		longestChains[currentDep] = Chain{currentDep}
+		return longestChains[currentDep]
+	}
+
+	if contains(currentChain, currentDep) {
+		// we've already been visited in the current chain, avoid cycles but also don't record a longest chain for currentDep
+		return nil
+	}
+
 	currentChain = append(currentChain, currentDep)
-	_, ok := graph[currentDep]
-	if ok {
-		for _, dep := range graph[currentDep] {
-			if !contains(currentChain, dep) {
-				cpy := make(Chain, len(currentChain))
-				copy(cpy, currentChain)
-				getLongestChain(dep, graph, cpy, longestChain)
-			} else {
-				if len(currentChain) > len(*longestChain) {
-					*longestChain = currentChain
-				}
-			}
-		}
-	} else {
-		if len(currentChain) > len(*longestChain) {
-			*longestChain = currentChain
+	// find the longest dependency chain
+	var longestDepChain Chain
+	for _, dep := range deps {
+		depChain := getLongestChain(dep, graph, currentChain, longestChains)
+		if len(depChain) > len(longestDepChain) {
+			longestDepChain = depChain
 		}
 	}
+	// prepend ourselves to the longest of our dependencies' chains and persist
+	longestChains[currentDep] = append(Chain{currentDep}, longestDepChain...)
+	return longestChains[currentDep]
 }
 
 func init() {
 	rootCmd.AddCommand(statsCmd)
+	statsCmd.Flags().StringVarP(&dir, "dir", "d", "", "Directory containing the module to evaluate. Defaults to the current directory.")
 	statsCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Get additional details")
 	statsCmd.Flags().BoolVarP(&jsonOutput, "json", "j", false, "Get the output in JSON format")
 	statsCmd.Flags().StringSliceVarP(&mainModules, "mainModules", "m", []string{}, "Enter modules whose dependencies should be considered direct dependencies; defaults to the first module encountered in `go mod graph` output")

cmd/utils.go

@@ -45,6 +45,9 @@ type DependencyOverview struct {
 func getDepInfo(mainModules []string) *DependencyOverview {
 	// get output of "go mod graph" in a string
 	goModGraph := exec.Command("go", "mod", "graph")
+	if dir != "" {
+		goModGraph.Dir = dir
+	}
 	goModGraphOutput, err := goModGraph.Output()
 	if err != nil {
 		log.Fatal(err)
@@ -112,38 +115,131 @@ func sliceContains(val []Chain, key Chain) bool {
 	return false
 }
 
+type module struct {
+	name    string
+	version string
+}
+
+func parseModule(s string) module {
+	if strings.Contains(s, "@") {
+		parts := strings.SplitN(s, "@", 2)
+		return module{name: parts[0], version: parts[1]}
+	}
+	return module{name: s}
+}
+
 func generateGraph(goModGraphOutputString string, mainModules []string) DependencyOverview {
 	depGraph := DependencyOverview{MainModules: mainModules}
+	versionedGraph := make(map[module][]module)
+	var lhss []module
 	graph := make(map[string][]string)
 	scanner := bufio.NewScanner(strings.NewReader(goModGraphOutputString))
 
+	var versionedMainModules []module
+	var seenVersionedMainModules = map[module]bool{}
 	for scanner.Scan() {
 		line := scanner.Text()
 		words := strings.Fields(line)
-		// remove versions
-		words[0] = (strings.Split(words[0], "@"))[0]
-		words[1] = (strings.Split(words[1], "@"))[0]
 
-		// we don't want to add the same dep again
-		if !contains(graph[words[0]], words[1]) {
-			graph[words[0]] = append(graph[words[0]], words[1])
+		lhs := parseModule(words[0])
+		if len(versionedMainModules) == 0 || contains(mainModules, lhs.name) {
+			if !seenVersionedMainModules[lhs] {
+				// remember our root module and listed main modules
+				versionedMainModules = append(versionedMainModules, lhs)
+				seenVersionedMainModules[lhs] = true
+			}
 		}
-
 		if len(depGraph.MainModules) == 0 {
-			depGraph.MainModules = append(depGraph.MainModules, words[0])
+			// record the first module we see as the main module by default
+			depGraph.MainModules = append(depGraph.MainModules, lhs.name)
 		}
+		rhs := parseModule(words[1])
+
+		// remember the order we observed lhs modules in
+		if len(versionedGraph[lhs]) == 0 {
+			lhss = append(lhss, lhs)
+		}
+		// record this lhs -> rhs relationship
+		versionedGraph[lhs] = append(versionedGraph[lhs], rhs)
+	}
+
+	// record effective versions of modules required by our main modules
+	// in go1.17+, the main module records effective versions of all dependencies, even indirect ones
+	effectiveVersions := map[string]string{}
+	for _, mm := range versionedMainModules {
+		for _, m := range versionedGraph[mm] {
+			if effectiveVersions[m.name] < m.version {
+				effectiveVersions[m.name] = m.version
+			}
+		}
+	}
 
-		// if the LHS is a mainModule
-		// then RHS is a direct dep else transitive dep
-		if contains(depGraph.MainModules, words[0]) && contains(depGraph.MainModules, words[1]) {
+	type edge struct {
+		from module
+		to   module
+	}
+
+	// figure out which modules in the graph are reachable from the effective versions required by our main modules
+	reachableModules := map[string]module{}
+	// start with our main modules
+	var toVisit []edge
+	for _, m := range versionedMainModules {
+		toVisit = append(toVisit, edge{to: m})
+	}
+	for len(toVisit) > 0 {
+		from := toVisit[0].from
+		v := toVisit[0].to
+		toVisit = toVisit[1:]
+		if _, reachable := reachableModules[v.name]; reachable {
+			// already flagged as reachable
+			continue
+		}
+		// mark as reachable
+		reachableModules[v.name] = from
+		if effectiveVersion, ok := effectiveVersions[v.name]; ok && effectiveVersion > v.version {
+			// replace with the effective version if applicable
+			v.version = effectiveVersion
+		} else {
+			// set the effective version
+			effectiveVersions[v.name] = v.version
+		}
+		// queue dependants of this to check for reachability
+		for _, m := range versionedGraph[v] {
+			toVisit = append(toVisit, edge{from: v, to: m})
+		}
+	}
+
+	for _, lhs := range lhss {
+		if _, reachable := reachableModules[lhs.name]; !reachable {
+			// this is not reachable via required versions, skip it
+			continue
+		}
+		if effectiveVersion, ok := effectiveVersions[lhs.name]; ok && effectiveVersion != lhs.version {
+			// this is not the effective version in our graph, skip it
 			continue
-		} else if contains(depGraph.MainModules, words[0]) {
-			if !contains(depGraph.DirectDepList, words[1]) {
-				depGraph.DirectDepList = append(depGraph.DirectDepList, words[1])
+		}
+		// fmt.Println(lhs.name, "via", reachableModules[lhs.name])
+
+		for _, rhs := range versionedGraph[lhs] {
+			// we don't want to add the same dep again
+			if !contains(graph[lhs.name], rhs.name) {
+				graph[lhs.name] = append(graph[lhs.name], rhs.name)
 			}
-		} else if !contains(depGraph.MainModules, words[0]) {
-			if !contains(depGraph.TransDepList, words[1]) {
-				depGraph.TransDepList = append(depGraph.TransDepList, words[1])
+
+			// if the LHS is a mainModule
+			// then RHS is a direct dep else transitive dep
+			if contains(depGraph.MainModules, lhs.name) && contains(depGraph.MainModules, rhs.name) {
+				continue
+			} else if contains(depGraph.MainModules, lhs.name) {
+				if !contains(depGraph.DirectDepList, rhs.name) {
+					// fmt.Println(rhs.name, "via", lhs)
+					depGraph.DirectDepList = append(depGraph.DirectDepList, rhs.name)
+				}
+			} else if !contains(depGraph.MainModules, lhs.name) {
+				if !contains(depGraph.TransDepList, rhs.name) {
+					// fmt.Println(rhs.name, "via", lhs)
+					depGraph.TransDepList = append(depGraph.TransDepList, rhs.name)
+				}
 			}
 		}
 	}

cmd/utils_test.go

@@ -17,7 +17,6 @@ limitations under the License.
 package cmd
 
 import (
-	"fmt"
 	"testing"
 )
 
@@ -55,10 +54,9 @@ func Test_getChains_simple(t *testing.T) {
 	}
 
 	var cycleChains []Chain
-	var longestChain Chain
 	var chains []Chain
 	var temp Chain
-	getLongestChain("A", graph, temp, &longestChain)
+	longestChain := getLongestChain("A", graph, temp, map[string]Chain{})
 	maxDepth := len(longestChain)
 	getCycleChains("A", graph, temp, &cycleChains)
 	getAllChains("A", graph, temp, &chains)
@@ -80,7 +78,6 @@ func Test_getChains_simple(t *testing.T) {
 "E" -> "F"
 "F" -> "H"
 `
-	fmt.Println(getFileContentsForAllDeps(overview))
 	if correctFileContentsForAllDeps != getFileContentsForAllDeps(overview) {
 		t.Errorf("File contents for graph of all dependencies are wrong")
 	}
@@ -150,10 +147,9 @@ func Test_getChains_cycle(t *testing.T) {
 	}
 
 	var cycleChains []Chain
-	var longestChain Chain
 	var chains []Chain
 	var temp Chain
-	getLongestChain("A", graph, temp, &longestChain)
+	longestChain := getLongestChain("A", graph, temp, map[string]Chain{})
 	maxDepth := len(longestChain)
 	getCycleChains("A", graph, temp, &cycleChains)
 	getAllChains("A", graph, temp, &chains)
@@ -244,10 +240,9 @@ func Test_getChains_cycle_2(t *testing.T) {
 	}
 
 	var cycleChains []Chain
-	var longestChain Chain
 	var chains []Chain
 	var temp Chain
-	getLongestChain("A", graph, temp, &longestChain)
+	longestChain := getLongestChain("A", graph, temp, map[string]Chain{})
 	maxDepth := len(longestChain)
 	getCycleChains("A", graph, temp, &cycleChains)
 	getAllChains("A", graph, temp, &chains)
@@ -364,7 +359,7 @@ func getGoModGraphTestData() string {
 		     | \ |  / \
 		     F   C     E
 	*/
-	goModGraphOutputString := `[email protected] G@1.2
+	goModGraphOutputString := `[email protected] G@1.5
 [email protected] [email protected]
 [email protected] [email protected]
 [email protected] [email protected]
@@ -413,3 +408,35 @@ func Test_generateGraph_custom_mainModule(t *testing.T) {
 		t.Errorf("Expected transitive dependencies are %s but got %s", transitiveDependencyList, depGraph.TransDepList)
 	}
 }
+
+func Test_generateGraph_overridden_versions(t *testing.T) {
+	mainModules := []string{"A", "D"}
+	// obsolete C@v1 has a cycle with D@v1 and a transitive ref to unwanted dependency E@v1
+	// effective version C@v2 updates to D@v2, which still has a cycle back to C@v2, but no dependency on E
+	depGraph := generateGraph(`A B@v2
+A C@v2
+A D@v2
+B@v2 C@v1
+C@v1 D@v1
+D@v1 C@v1
+D@v1 E@v1
+C@v2 D@v2
+C@v2 F@v2
+D@v2 C@v2
+D@v2 G@v2`, mainModules)
+
+	transitiveDependencyList := []string{"C", "D", "F"}
+	directDependencyList := []string{"B", "C", "G"}
+
+	if !isSliceSame(depGraph.MainModules, mainModules) {
+		t.Errorf("Expected mainModules are %s but got %s", mainModules, depGraph.MainModules)
+	}
+
+	if !isSliceSame(depGraph.DirectDepList, directDependencyList) {
+		t.Errorf("Expected direct dependecies are %s but got %s", directDependencyList, depGraph.DirectDepList)
+	}
+
+	if !isSliceSame(depGraph.TransDepList, transitiveDependencyList) {
+		t.Errorf("Expected transitive dependencies are %s but got %s", transitiveDependencyList, depGraph.TransDepList)
+	}
+}