Skip to content

Commit e8e8541

Browse files
HollandDMHollandDM
committed
update
update 2 update 3
1 parent 49bf4fa commit e8e8541

File tree

6 files changed

+347
-256
lines changed

6 files changed

+347
-256
lines changed

‎core/codesig/src/mill/codesig/CodeSig.scala

Lines changed: 46 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,31 +3,63 @@ package mill.codesig
33
import mill.codesig.JvmModel.*
44

55
object CodeSig {
6-
def compute(
7-
classFiles: Seq[os.Path],
8-
upstreamClasspath: Seq[os.Path],
9-
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
10-
logger: Logger,
11-
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
12-
): CallGraphAnalysis = {
13-
implicit val st: SymbolTable = new SymbolTable()
146

7+
private def callGraphAnalysis(
8+
classFiles: Seq[os.Path],
9+
upstreamClasspath: Seq[os.Path],
10+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
11+
)(implicit st: SymbolTable): CallGraphAnalysis = {
1512
val localSummary = LocalSummary.apply(classFiles.iterator.map(os.read.inputStream(_)))
16-
logger.log(localSummary)
1713

1814
val externalSummary = ExternalSummary.apply(localSummary, upstreamClasspath)
19-
logger.log(externalSummary)
2015

2116
val resolvedMethodCalls = ResolvedCalls.apply(localSummary, externalSummary)
22-
logger.log(resolvedMethodCalls)
2317

2418
new CallGraphAnalysis(
2519
localSummary,
2620
resolvedMethodCalls,
2721
externalSummary,
28-
ignoreCall,
29-
logger,
30-
prevTransitiveCallGraphHashesOpt
22+
ignoreCall
3123
)
3224
}
25+
26+
def getCallGraphAnalysis(
27+
classFiles: Seq[os.Path],
28+
upstreamClasspath: Seq[os.Path],
29+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
30+
): CallGraphAnalysis = {
31+
implicit val st: SymbolTable = new SymbolTable()
32+
33+
callGraphAnalysis(classFiles, upstreamClasspath, ignoreCall)
34+
}
35+
36+
def compute(
37+
classFiles: Seq[os.Path],
38+
upstreamClasspath: Seq[os.Path],
39+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
40+
logger: Logger,
41+
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
42+
): CallGraphAnalysis = {
43+
implicit val st: SymbolTable = new SymbolTable()
44+
45+
val callAnalysis = callGraphAnalysis(classFiles, upstreamClasspath, ignoreCall)
46+
47+
logger.log(callAnalysis.localSummary)
48+
logger.log(callAnalysis.externalSummary)
49+
logger.log(callAnalysis.resolved)
50+
51+
logger.mandatoryLog(callAnalysis.methodCodeHashes)
52+
logger.mandatoryLog(callAnalysis.prettyCallGraph)
53+
logger.mandatoryLog(callAnalysis.transitiveCallGraphHashes0)
54+
55+
logger.log(callAnalysis.transitiveCallGraphHashes)
56+
57+
val spanningInvalidationTree = callAnalysis.calculateSpanningInvalidationTree {
58+
prevTransitiveCallGraphHashesOpt()
59+
}
60+
61+
logger.mandatoryLog(spanningInvalidationTree)
62+
63+
callAnalysis
64+
}
3365
}

‎core/codesig/src/mill/codesig/ExternalSummary.scala

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import mill.codesig.JvmModel.*
55
import org.objectweb.asm.{ClassReader, ClassVisitor, MethodVisitor, Opcodes}
66

77
import java.net.URLClassLoader
8+
import scala.util.Try
89

910
case class ExternalSummary(
1011
directMethods: Map[JCls, Map[MethodSig, Boolean]],
@@ -47,7 +48,8 @@ object ExternalSummary {
4748

4849
def load(cls: JCls): Unit = methodsPerCls.getOrElse(cls, load0(cls))
4950

50-
def load0(cls: JCls): Unit = {
51+
// Some macros implementations will fail the ClassReader, we can skip them
52+
def load0(cls: JCls): Unit = Try {
5153
val visitor = new MyClassVisitor()
5254
val resourcePath =
5355
os.resource(upstreamClassloader) / os.SubPath(cls.name.replace('.', '/') + ".class")
@@ -61,7 +63,7 @@ object ExternalSummary {
6163
methodsPerCls(cls) = visitor.methods
6264
ancestorsPerCls(cls) = visitor.ancestors
6365
ancestorsPerCls(cls).foreach(load)
64-
}
66+
}.getOrElse(())
6567

6668
(allDirectAncestors ++ allMethodCallParamClasses)
6769
.filter(!localSummary.contains(_))

‎core/codesig/src/mill/codesig/ReachabilityAnalysis.scala

Lines changed: 84 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,17 @@ package mill.codesig
22

33
import mill.codesig.JvmModel.*
44
import mill.internal.{SpanningForest, Tarjans}
5-
import ujson.{Arr, Obj}
5+
import ujson.{Obj, Arr}
66
import upickle.default.{Writer, writer}
77

88
import scala.collection.immutable.SortedMap
99
import scala.collection.mutable
1010

1111
class CallGraphAnalysis(
12-
localSummary: LocalSummary,
13-
resolved: ResolvedCalls,
14-
externalSummary: ExternalSummary,
15-
ignoreCall: (Option[MethodDef], MethodSig) => Boolean,
16-
logger: Logger,
17-
prevTransitiveCallGraphHashesOpt: () => Option[Map[String, Int]]
12+
val localSummary: LocalSummary,
13+
val resolved: ResolvedCalls,
14+
val externalSummary: ExternalSummary,
15+
ignoreCall: (Option[MethodDef], MethodSig) => Boolean
1816
)(implicit st: SymbolTable) {
1917

2018
val methods: Map[MethodDef, LocalSummary.MethodInfo] = for {
@@ -41,17 +39,13 @@ class CallGraphAnalysis(
4139
lazy val methodCodeHashes: SortedMap[String, Int] =
4240
methods.map { case (k, vs) => (k.toString, vs.codeHash) }.to(SortedMap)
4341

44-
logger.mandatoryLog(methodCodeHashes)
45-
4642
lazy val prettyCallGraph: SortedMap[String, Array[CallGraphAnalysis.Node]] = {
4743
indexGraphEdges.zip(indexToNodes).map { case (vs, k) =>
4844
(k.toString, vs.map(indexToNodes))
4945
}
5046
.to(SortedMap)
5147
}
5248

53-
logger.mandatoryLog(prettyCallGraph)
54-
5549
def transitiveCallGraphValues[V: scala.reflect.ClassTag](
5650
nodeValues: Array[V],
5751
reduce: (V, V) => V,
@@ -79,45 +73,45 @@ class CallGraphAnalysis(
7973
.collect { case (CallGraphAnalysis.LocalDef(d), v) => (d.toString, v) }
8074
.to(SortedMap)
8175

82-
logger.mandatoryLog(transitiveCallGraphHashes0)
83-
logger.log(transitiveCallGraphHashes)
84-
85-
lazy val (spanningInvalidationTree: Obj, invalidClassNames: Arr) = prevTransitiveCallGraphHashesOpt() match {
86-
case Some(prevTransitiveCallGraphHashes) =>
87-
CallGraphAnalysis.spanningInvalidationTree(
88-
prevTransitiveCallGraphHashes,
89-
transitiveCallGraphHashes0,
90-
indexToNodes,
91-
indexGraphEdges
92-
)
93-
case None => ujson.Obj() -> ujson.Arr()
76+
def calculateSpanningInvalidationTree(
77+
prevTransitiveCallGraphHashesOpt: => Option[Map[String, Int]]
78+
): Obj = {
79+
prevTransitiveCallGraphHashesOpt match {
80+
case Some(prevTransitiveCallGraphHashes) =>
81+
CallGraphAnalysis.spanningInvalidationTree(
82+
prevTransitiveCallGraphHashes,
83+
transitiveCallGraphHashes0,
84+
indexToNodes,
85+
indexGraphEdges
86+
)
87+
case None => ujson.Obj()
88+
}
9489
}
9590

96-
logger.mandatoryLog(spanningInvalidationTree)
97-
logger.mandatoryLog(invalidClassNames)
91+
def calculateInvalidatedClassNames(
92+
prevTransitiveCallGraphHashesOpt: => Option[Map[String, Int]]
93+
): Set[String] = {
94+
prevTransitiveCallGraphHashesOpt match {
95+
case Some(prevTransitiveCallGraphHashes) =>
96+
CallGraphAnalysis.invalidatedClassNames(
97+
prevTransitiveCallGraphHashes,
98+
transitiveCallGraphHashes0,
99+
indexToNodes,
100+
indexGraphEdges
101+
)
102+
case None => Set.empty
103+
}
104+
}
98105
}
99106

100107
object CallGraphAnalysis {
101108

102-
/**
103-
* Computes the minimal spanning forest of the that covers the nodes in the
104-
* call graph whose transitive call graph hashes has changed since the last
105-
* run, rendered as a JSON dictionary tree. This provides a great "debug
106-
* view" that lets you easily Cmd-F to find a particular node and then trace
107-
* it up the JSON hierarchy to figure out what upstream node was the root
108-
* cause of the change in the callgraph.
109-
*
110-
* There are typically multiple possible spanning forests for a given graph;
111-
* one is chosen arbitrarily. This is usually fine, since when debugging you
112-
* typically are investigating why there's a path to a node at all where none
113-
* should exist, rather than trying to fully analyse all possible paths
114-
*/
115-
def spanningInvalidationTree(
109+
private def getSpanningForest(
116110
prevTransitiveCallGraphHashes: Map[String, Int],
117111
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
118112
indexToNodes: Array[Node],
119113
indexGraphEdges: Array[Array[Int]]
120-
): (ujson.Obj, ujson.Arr) = {
114+
) = {
121115
val transitiveCallGraphHashes0Map = transitiveCallGraphHashes0.toMap
122116

123117
val nodesWithChangedHashes = indexGraphEdges
@@ -137,23 +131,62 @@ object CallGraphAnalysis {
137131
val reverseGraphEdges =
138132
indexGraphEdges.indices.map(reverseGraphMap.getOrElse(_, Array[Int]())).toArray
139133

140-
val spanningForest = SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false)
134+
SpanningForest.apply(reverseGraphEdges, nodesWithChangedHashes, false)
135+
}
141136

142-
val spanningInvalidationTree = SpanningForest.spanningTreeToJsonTree(
143-
spanningForest,
137+
/**
138+
* Computes the minimal spanning forest of the that covers the nodes in the
139+
* call graph whose transitive call graph hashes has changed since the last
140+
* run, rendered as a JSON dictionary tree. This provides a great "debug
141+
* view" that lets you easily Cmd-F to find a particular node and then trace
142+
* it up the JSON hierarchy to figure out what upstream node was the root
143+
* cause of the change in the callgraph.
144+
*
145+
* There are typically multiple possible spanning forests for a given graph;
146+
* one is chosen arbitrarily. This is usually fine, since when debugging you
147+
* typically are investigating why there's a path to a node at all where none
148+
* should exist, rather than trying to fully analyse all possible paths
149+
*/
150+
def spanningInvalidationTree(
151+
prevTransitiveCallGraphHashes: Map[String, Int],
152+
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
153+
indexToNodes: Array[Node],
154+
indexGraphEdges: Array[Array[Int]]
155+
): ujson.Obj = {
156+
SpanningForest.spanningTreeToJsonTree(
157+
getSpanningForest(prevTransitiveCallGraphHashes, transitiveCallGraphHashes0, indexToNodes, indexGraphEdges),
144158
k => indexToNodes(k).toString
145159
)
160+
}
146161

147-
val invalidSet = invalidClassNameSet(
148-
spanningForest,
149-
indexToNodes.map {
150-
case LocalDef(call) => call.cls.name
151-
case Call(call) => call.cls.name
152-
case ExternalClsCall(cls) => cls.name
162+
/**
163+
* Get all class names that have their hashcode changed compared to prevTransitiveCallGraphHashes
164+
*/
165+
def invalidatedClassNames(
166+
prevTransitiveCallGraphHashes: Map[String, Int],
167+
transitiveCallGraphHashes0: Array[(CallGraphAnalysis.Node, Int)],
168+
indexToNodes: Array[Node],
169+
indexGraphEdges: Array[Array[Int]]
170+
): Set[String] = {
171+
val rootNode = getSpanningForest(prevTransitiveCallGraphHashes, transitiveCallGraphHashes0, indexToNodes, indexGraphEdges)
172+
173+
val jsonValueQueue = mutable.ArrayDeque[(Int, SpanningForest.Node)]()
174+
jsonValueQueue.appendAll(rootNode.values.toSeq)
175+
val builder = Set.newBuilder[String]
176+
177+
while (jsonValueQueue.nonEmpty) {
178+
val (nodeIndex, node) = jsonValueQueue.removeHead()
179+
node.values.foreach { case (childIndex, childNode) =>
180+
jsonValueQueue.append((childIndex, childNode))
153181
}
154-
)
182+
indexToNodes(nodeIndex) match {
183+
case CallGraphAnalysis.LocalDef(methodDef) => builder.addOne(methodDef.cls.name)
184+
case CallGraphAnalysis.Call(methodCall) => builder.addOne(methodCall.cls.name)
185+
case CallGraphAnalysis.ExternalClsCall(externalCls) => builder.addOne(externalCls.name)
186+
}
187+
}
155188

156-
(spanningInvalidationTree, invalidSet)
189+
builder.result()
157190
}
158191

159192
def indexGraphEdges(
@@ -278,24 +311,6 @@ object CallGraphAnalysis {
278311
}
279312
}
280313

281-
private def invalidClassNameSet(
282-
spanningForest: SpanningForest.Node,
283-
indexToClassName: Array[String]
284-
): Set[String] = {
285-
val queue = mutable.ArrayBuffer.empty[(Int, SpanningForest.Node)]
286-
val result = mutable.Set.empty[String]
287-
288-
queue.appendAll(spanningForest.values)
289-
290-
while (queue.nonEmpty) {
291-
val (index, node) = queue.remove(0)
292-
result += indexToClassName(index)
293-
queue.appendAll(node.values)
294-
}
295-
296-
result.toSet
297-
}
298-
299314
/**
300315
* Represents the three types of nodes in our call graph. These are kept heterogeneous
301316
* because flattening them out into a homogenous graph of MethodDef -> MethodDef edges

‎core/define/src/mill/define/Task.scala

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,16 @@ object Task extends TaskBase {
124124
inline def Command[T](inline t: Result[T])(implicit
125125
inline w: W[T],
126126
inline ctx: mill.define.ModuleCtx
127-
): Command[T] = ${ TaskMacros.commandImpl[T]('t)('w, 'ctx, exclusive = '{ false }) }
127+
): Command[T] = ${ TaskMacros.commandImpl[T]('t)('w, 'ctx, exclusive = '{ false }, persistent = '{ false }) }
128+
129+
130+
/**
131+
* This version allow [[Command]] to be persistent
132+
*/
133+
inline def Command[T](inline persistent: Boolean)(inline t: Result[T])(implicit
134+
inline w: W[T],
135+
inline ctx: mill.define.ModuleCtx
136+
): Command[T] = ${ TaskMacros.commandImpl[T]('t)('w, 'ctx, exclusive = '{ false }, persistent = '{ persistent }) }
128137

129138
/**
130139
* @param exclusive Exclusive commands run serially at the end of an evaluation,
@@ -142,7 +151,7 @@ object Task extends TaskBase {
142151
inline def apply[T](inline t: Result[T])(implicit
143152
inline w: W[T],
144153
inline ctx: mill.define.ModuleCtx
145-
): Command[T] = ${ TaskMacros.commandImpl[T]('t)('w, 'ctx, '{ this.exclusive }) }
154+
): Command[T] = ${ TaskMacros.commandImpl[T]('t)('w, 'ctx, '{ this.exclusive }, '{ false }) }
146155
}
147156

148157
/**
@@ -394,7 +403,8 @@ class Command[+T](
394403
val ctx0: mill.define.ModuleCtx,
395404
val writer: W[?],
396405
val isPrivate: Option[Boolean],
397-
val exclusive: Boolean
406+
val exclusive: Boolean,
407+
override val persistent: Boolean
398408
) extends NamedTask[T] {
399409

400410
override def asCommand: Some[Command[T]] = Some(this)
@@ -541,12 +551,13 @@ private object TaskMacros {
541551
)(t: Expr[Result[T]])(
542552
w: Expr[W[T]],
543553
ctx: Expr[mill.define.ModuleCtx],
544-
exclusive: Expr[Boolean]
554+
exclusive: Expr[Boolean],
555+
persistent: Expr[Boolean]
545556
): Expr[Command[T]] = {
546557
appImpl[Command, T](
547558
(in, ev) =>
548559
'{
549-
new Command[T]($in, $ev, $ctx, $w, ${ taskIsPrivate() }, exclusive = $exclusive)
560+
new Command[T]($in, $ev, $ctx, $w, ${ taskIsPrivate() }, exclusive = $exclusive, persistent = $persistent)
550561
},
551562
t
552563
)

0 commit comments

Comments
 (0)