Skip to content

Commit cce94ef

Browse files
HollandDMHollandDM
committed
update 2
1 parent d5e5433 commit cce94ef

File tree

5 files changed

+212
-138
lines changed

5 files changed

+212
-138
lines changed

‎core/codesig/src/mill/codesig/CodeSig.scala

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,63 @@ package mill.codesig
33
import mill.codesig.JvmModel.*
44

55
object CodeSig {
6-
def compute(
7-
classFiles: Seq[os.Path],
8-
upstreamClasspath: Seq[os.Path],
9-
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
10-
logger: Logger,
11-
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
12-
): CallGraphAnalysis = {
13-
implicit val st: SymbolTable = new SymbolTable()
146

7+
private def callGraphAnalysis(
8+
classFiles: Seq[os.Path],
9+
upstreamClasspath: Seq[os.Path],
10+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
11+
)(implicit st: SymbolTable): CallGraphAnalysis = {
1512
val localSummary = LocalSummary.apply(classFiles.iterator.map(os.read.inputStream(_)))
16-
logger.log(localSummary)
1713

1814
val externalSummary = ExternalSummary.apply(localSummary, upstreamClasspath)
19-
logger.log(externalSummary)
2015

2116
val resolvedMethodCalls = ResolvedCalls.apply(localSummary, externalSummary)
22-
logger.log(resolvedMethodCalls)
2317

2418
new CallGraphAnalysis(
2519
localSummary,
2620
resolvedMethodCalls,
2721
externalSummary,
28-
ignoreCall,
29-
logger,
30-
prevTransitiveCallGraphHashesOpt
22+
ignoreCall
3123
)
3224
}
25+
26+
def getCallGraphAnalysis(
27+
classFiles: Seq[os.Path],
28+
upstreamClasspath: Seq[os.Path],
29+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
30+
): CallGraphAnalysis = {
31+
implicit val st: SymbolTable = new SymbolTable()
32+
33+
callGraphAnalysis(classFiles, upstreamClasspath, ignoreCall)
34+
}
35+
36+
def compute(
37+
classFiles: Seq[os.Path],
38+
upstreamClasspath: Seq[os.Path],
39+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
40+
logger: Logger,
41+
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
42+
): CallGraphAnalysis = {
43+
implicit val st: SymbolTable = new SymbolTable()
44+
45+
val callAnalysis = callGraphAnalysis(classFiles, upstreamClasspath, ignoreCall)
46+
47+
logger.log(callAnalysis.localSummary)
48+
logger.log(callAnalysis.externalSummary)
49+
logger.log(callAnalysis.resolved)
50+
51+
logger.mandatoryLog(callAnalysis.methodCodeHashes)
52+
logger.mandatoryLog(callAnalysis.prettyCallGraph)
53+
logger.mandatoryLog(callAnalysis.transitiveCallGraphHashes0)
54+
55+
logger.log(callAnalysis.transitiveCallGraphHashes)
56+
57+
val spanningInvalidationTree = callAnalysis.calculateSpanningInvalidationTree {
58+
prevTransitiveCallGraphHashesOpt()
59+
}
60+
61+
logger.mandatoryLog(spanningInvalidationTree)
62+
63+
callAnalysis
64+
}
3365
}

‎core/codesig/src/mill/codesig/ExternalSummary.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import mill.codesig.JvmModel.*
55
import org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor, Opcodes}
66

77
import java.net.URLClassLoader
8+
import scala.util.Try
89

910
case class ExternalSummary(
1011
directMethods: Map[JCls, Map[MethodSig, Boolean]],
@@ -47,7 +48,8 @@ object ExternalSummary {
4748

4849
def load(cls: JCls): Unit = methodsPerCls.getOrElse(cls, load0(cls))
4950

50-
def load0(cls: JCls): Unit = {
51+
// Some macros implementations will fail the ClassReader, we can skip them
52+
def load0(cls: JCls): Unit = Try {
5153
val visitor = new MyClassVisitor()
5254
val resourcePath =
5355
os.resource(upstreamClassloader) / os.SubPath(cls.name.replace('.', '/') + ".class")
@@ -61,7 +63,7 @@ object ExternalSummary {
6163
methodsPerCls(cls) = visitor.methods
6264
ancestorsPerCls(cls) = visitor.ancestors
6365
ancestorsPerCls(cls).foreach(load)
64-
}
66+
}.getOrElse(())
6567

6668
(allDirectAncestors ++ allMethodCallParamClasses)
6769
.filter(!localSummary.contains(_))

‎core/codesig/src/mill/codesig/ReachabilityAnalysis.scala

Lines changed: 88 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,17 @@ package mill.codesig
22

33
import mill.codesig.JvmModel.*
44
import mill.internal.{SpanningForest, Tarjans}
5-
import ujson.Obj
5+
import ujson.{Obj, Arr}
66
import upickle.default.{Writer, writer}
77

88
import scala.collection.immutable.SortedMap
9+
import scala.collection.mutable
910

1011
class CallGraphAnalysis(
11-
localSummary: LocalSummary,
12-
resolved: ResolvedCalls,
13-
externalSummary: ExternalSummary,
14-
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
15-
logger: Logger,
16-
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
12+
val localSummary: LocalSummary,
13+
val resolved: ResolvedCalls,
14+
val externalSummary: ExternalSummary,
15+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
1716
)(implicit st: SymbolTable) {
1817

1918
val methods: Map[MethodDef, LocalSummary.MethodInfo] = for {
@@ -40,17 +39,13 @@ class CallGraphAnalysis(
4039
lazy val methodCodeHashes: SortedMap[String, Int] =
4140
methods.map { case (k, vs) => (k.toString, vs.codeHash) }.to(SortedMap)
4241

43-
logger.mandatoryLog(methodCodeHashes)
44-
4542
lazy val prettyCallGraph: SortedMap[String, Array[CallGraphAnalysis.Node]] = {
4643
indexGraphEdges.zip(indexToNodes).map { case (vs, k) =>
4744
(k.toString, vs.map(indexToNodes))
4845
}
4946
.to(SortedMap)
5047
}
5148

52-
logger.mandatoryLog(prettyCallGraph)
53-
5449
def transitiveCallGraphValues[V: scala.reflect.ClassTag](
5550
nodeValues: Array[V],
5651
reduce: (V, V) => V,
@@ -78,44 +73,45 @@ class CallGraphAnalysis(
7873
.collect { case (CallGraphAnalysis.LocalDef(d), v) => (d.toString, v) }
7974
.to(SortedMap)
8075

81-
logger.mandatoryLog(transitiveCallGraphHashes0)
82-
logger.log(transitiveCallGraphHashes)
83-
84-
lazy val spanningInvalidationTree: Obj = prevTransitiveCallGraphHashesOpt() match {
85-
case Some(prevTransitiveCallGraphHashes) =>
86-
CallGraphAnalysis.spanningInvalidationTree(
87-
prevTransitiveCallGraphHashes,
88-
transitiveCallGraphHashes0,
89-
indexToNodes,
90-
indexGraphEdges
91-
)
92-
case None => ujson.Obj()
76+
def calculateSpanningInvalidationTree(
77+
prevTransitiveCallGraphHashesOpt: => Option[Map[String, Int]]
78+
): Obj = {
79+
prevTransitiveCallGraphHashesOpt match {
80+
case Some(prevTransitiveCallGraphHashes) =>
81+
CallGraphAnalysis.spanningInvalidationTree(
82+
prevTransitiveCallGraphHashes,
83+
transitiveCallGraphHashes0,
84+
indexToNodes,
85+
indexGraphEdges
86+
)
87+
case None => ujson.Obj()
88+
}
9389
}
9490

95-
logger.mandatoryLog(spanningInvalidationTree)
91+
def calculateInvalidClassName(
92+
prevTransitiveCallGraphHashesOpt: => Option[Map[String, Int]]
93+
): Set[String] = {
94+
prevTransitiveCallGraphHashesOpt match {
95+
case Some(prevTransitiveCallGraphHashes) =>
96+
CallGraphAnalysis.invalidClassNames(
97+
prevTransitiveCallGraphHashes,
98+
transitiveCallGraphHashes0,
99+
indexToNodes,
100+
indexGraphEdges
101+
)
102+
case None => Set.empty
103+
}
104+
}
96105
}
97106

98107
object CallGraphAnalysis {
99108

100-
/**
101-
* Computes the minimal spanning forest of the that covers the nodes in the
102-
* call graph whose transitive call graph hashes has changed since the last
103-
* run, rendered as a JSON dictionary tree. This provides a great "debug
104-
* view" that lets you easily Cmd-F to find a particular node and then trace
105-
* it up the JSON hierarchy to figure out what upstream node was the root
106-
* cause of the change in the callgraph.
107-
*
108-
* There are typically multiple possible spanning forests for a given graph;
109-
* one is chosen arbitrarily. This is usually fine, since when debugging you
110-
* typically are investigating why there's a path to a node at all where none
111-
* should exist, rather than trying to fully analyse all possible paths
112-
*/
113-
def spanningInvalidationTree(
109+
private def getSpanningForest(
114110
prevTransitiveCallGraphHashes: Map[String, Int],
115111
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
116112
indexToNodes: Array[Node],
117113
indexGraphEdges: Array[Array[Int]]
118-
): ujson.Obj = {
114+
) = {
119115
val transitiveCallGraphHashes0Map = transitiveCallGraphHashes0.toMap
120116

121117
val nodesWithChangedHashes = indexGraphEdges
@@ -135,12 +131,64 @@ object CallGraphAnalysis {
135131
val reverseGraphEdges =
136132
indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array[Int]())).toArray
137133

134+
SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false)
135+
}
136+
137+
/**
138+
* Computes the minimal spanning forest of the that covers the nodes in the
139+
* call graph whose transitive call graph hashes has changed since the last
140+
* run, rendered as a JSON dictionary tree. This provides a great "debug
141+
* view" that lets you easily Cmd-F to find a particular node and then trace
142+
* it up the JSON hierarchy to figure out what upstream node was the root
143+
* cause of the change in the callgraph.
144+
*
145+
* There are typically multiple possible spanning forests for a given graph;
146+
* one is chosen arbitrarily. This is usually fine, since when debugging you
147+
* typically are investigating why there's a path to a node at all where none
148+
* should exist, rather than trying to fully analyse all possible paths
149+
*/
150+
def spanningInvalidationTree(
151+
prevTransitiveCallGraphHashes: Map[String, Int],
152+
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
153+
indexToNodes: Array[Node],
154+
indexGraphEdges: Array[Array[Int]]
155+
): ujson.Obj = {
138156
SpanningForest.spanningTreeToJsonTree(
139-
SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false),
157+
getSpanningForest(prevTransitiveCallGraphHashes, transitiveCallGraphHashes0, indexToNodes, indexGraphEdges),
140158
k => indexToNodes(k).toString
141159
)
142160
}
143161

162+
/**
163+
* Get all class names that have their hashcode changed compared to prevTransitiveCallGraphHashes
164+
*/
165+
def invalidClassNames(
166+
prevTransitiveCallGraphHashes: Map[String, Int],
167+
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
168+
indexToNodes: Array[Node],
169+
indexGraphEdges: Array[Array[Int]]
170+
): Set[String] = {
171+
val rootNode = getSpanningForest(prevTransitiveCallGraphHashes, transitiveCallGraphHashes0, indexToNodes, indexGraphEdges)
172+
173+
val jsonValueQueue = mutable.ArrayDeque[(Int, SpanningForest.Node)]()
174+
jsonValueQueue.appendAll(rootNode.values.toSeq)
175+
val invalidClassNames = Set.newBuilder[String]
176+
177+
while (jsonValueQueue.nonEmpty) {
178+
val (nodeIndex, node) = jsonValueQueue.removeHead()
179+
node.values.foreach { case (childIndex, childNode) =>
180+
jsonValueQueue.append((childIndex, childNode))
181+
}
182+
indexToNodes(nodeIndex) match {
183+
case CallGraphAnalysis.LocalDef(methodDef) => invalidClassNames.addOne(methodDef.cls.name)
184+
case CallGraphAnalysis.Call(methodCall) => invalidClassNames.addOne(methodCall.cls.name)
185+
case CallGraphAnalysis.ExternalClsCall(externalCls) => invalidClassNames.addOne(externalCls.name)
186+
}
187+
}
188+
189+
invalidClassNames.result()
190+
}
191+
144192
def indexGraphEdges(
145193
indexToNodes: Array[Node],
146194
methods: Map[MethodDef, LocalSummary.MethodInfo],

‎runner/src/mill/runner/MillBuildRootModule.scala

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,30 @@ abstract class MillBuildRootModule()(implicit
163163
super.callGraphAnalysisIgnoreCalls(callSiteOpt, calledSig) || isCommand || isMillDiscover
164164
}
165165

166-
def codeSignatures: T[Map[String, Int]] = Task {
167-
val (analysisFolder, _) = callGraphAnalysis()
168-
val transitiveCallGraphHashes0 = upickle.default.read[Map[String, Int]](
169-
os.read.stream(analysisFolder / "transitiveCallGraphHashes0.json")
170-
)
171-
transitiveCallGraphHashes0
166+
def codeSignatures: T[Map[String, Int]] = Task(persistent = true) {
167+
os.remove.all(Task.dest / "previous")
168+
if (os.exists(Task.dest / "current")) os.move.over(Task.dest / "current", Task.dest / "previous")
169+
170+
val debugEnabled = Task.log.debugEnabled
171+
172+
val callAnalysis = mill.codesig.CodeSig
173+
.compute(
174+
classFiles = os.walk(compile().classes.path).filter(_.ext == "class"),
175+
upstreamClasspath = compileClasspath().toSeq.map(_.path),
176+
ignoreCall = (callSiteOpt, calledSig) => callGraphAnalysisIgnoreCalls(callSiteOpt, calledSig),
177+
logger = new mill.codesig.Logger(
178+
Task.dest / "current",
179+
Option.when(debugEnabled)(Task.dest / "current")
180+
),
181+
prevTransitiveCallGraphHashesOpt = () =>
182+
Option.when(os.exists(Task.dest / "previous/transitiveCallGraphHashes0.json"))(
183+
upickle.default.read[Map[String, Int]](
184+
os.read.stream(Task.dest / "previous/transitiveCallGraphHashes0.json")
185+
)
186+
)
187+
)
188+
189+
callAnalysis.transitiveCallGraphHashes
172190
}
173191

174192
override def sources: T[Seq[PathRef]] = Task {

0 commit comments

Comments
 (0)