POC Remote Caching

.github/workflows/publish-bazel-remote-apis.yml

@@ -0,0 +1,34 @@
+name: Publish Bridges
+
+# Manually-triggered github action to publish mill-scala-compiler-bridges jars,
+# since those do not change frequently enough to be worth including in the main
+# publishing workflow that runs every Mill version
+on:
+  workflow_dispatch:
+jobs:
+  publish-bridges:
+    runs-on: ubuntu-latest
+
+    concurrency: publish-sonatype-${{ github.sha }}
+
+    env:
+      SONATYPE_PGP_SECRET: ${{ secrets.SONATYPE_PGP_SECRET }}
+      SONATYPE_USERNAME: ${{ secrets.SONATYPE_DEPLOY_USER }}
+      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_DEPLOY_PASSWORD }}
+      SONATYPE_PGP_PASSWORD: ${{ secrets.SONATYPE_PGP_PASSWORD }}
+      LANG: "en_US.UTF-8"
+      LC_MESSAGES: "en_US.UTF-8"
+      LC_ALL: "en_US.UTF-8"
+      MILL_BUILD_COMPILER_BRIDGES: "true"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: temurin
+
+      - run:  ci/release-bazel-remote-apis-maven.sh

build.sc

@@ -105,6 +105,7 @@ object Deps {
   val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5"
 
   val coursier = ivy"io.get-coursier::coursier:2.1.7"
+  val coursierCache = ivy"io.get-coursier::coursier-cache:2.1.7"
   val coursierInterface = ivy"io.get-coursier:interface:1.0.18"
 
   val cask = ivy"com.lihaoyi::cask:0.9.1"
@@ -395,6 +396,49 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima {
   def skipPreviousVersions: T[Seq[String]] = T(Seq.empty[String])
 }
 
+object bazelRemoteApis extends MillPublishJavaModule{
+  def sourceZip = T{
+    os.write(
+      T.dest / "source.zip",
+      requests.get("https://github.com/bazelbuild/remote-apis/archive/refs/tags/v2.2.0.zip")
+    )
+    PathRef(T.dest / "source.zip")
+  }
+
+  def bazel = T{
+    val fileName =
+      if (System.getProperty("os.name") == "Mac OS X") "bazel-5.4.1-darwin-arm64"
+      else "bazel-5.4.1-linux-x86_64"
+
+    os.write(
+      T.dest / "bazel",
+      requests.get(s"https://github.com/bazelbuild/bazel/releases/download/5.4.1/$fileName")
+    )
+    os.perms.set(T.dest / "bazel", "rwxrwxrwx")
+    PathRef(T.dest / "bazel")
+  }
+
+  def unmanagedClasspath = T{
+    val javaBuildTarget = "build/bazel/remote/execution/v2:remote_execution_java_proto"
+    os.proc("unzip", sourceZip().path, "-d", T.dest).call(cwd = T.dest)
+    os.proc(bazel().path, "build", javaBuildTarget).call(cwd = T.dest / "remote-apis-2.2.0")
+
+    os.proc(bazel().path, "cquery", javaBuildTarget, "--output=files")
+      .call(cwd = T.dest / "remote-apis-2.2.0")
+      .out
+      .lines()
+      .map(line => PathRef(T.dest / "remote-apis-2.2.0" / os.SubPath(line)))
+  }
+
+  def jar = assembly()
+
+  def publishVersion = "0.0.1"
+
+  def artifactName = "mill-bazel-remote-cache-protos"
+
+  def pomSettings = commonPomSettings(artifactName())
+}
+
 object bridge extends Cross[BridgeModule](buildBridgeScalaVersions)
 trait BridgeModule extends MillPublishJavaModule with CrossScalaModule {
   def scalaVersion = crossScalaVersion
@@ -441,7 +485,6 @@ object main extends MillStableScalaModule with BuildInfo {
     Deps.windowsAnsi,
     Deps.mainargs,
     Deps.coursierInterface,
-    Deps.requests
   )
 
   def compileIvyDeps = Agg(Deps.scalaReflect(scalaVersion()))
@@ -483,7 +526,8 @@ object main extends MillStableScalaModule with BuildInfo {
       Deps.upickle,
       Deps.pprint,
       Deps.fansi,
-      Deps.sbtTestInterface
+      Deps.sbtTestInterface,
+      Deps.coursierCache
     )
   }
 
@@ -562,6 +606,13 @@ object main extends MillStableScalaModule with BuildInfo {
 
   object eval extends MillStableScalaModule {
     def moduleDeps = Seq(define)
+
+    def ivyDeps = Agg(
+      Deps.requests,
+      ivy"com.google.protobuf:protobuf-java:3.15.6",
+      ivy"org.apache.commons:commons-compress:1.21",
+      ivy"com.lihaoyi:mill-bazel-remote-cache-protos:0.0.1",
+    )
   }
 
   object resolve extends MillStableScalaModule {

ci/release-bazel-remote-apis-maven.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -eu
+
+echo $SONATYPE_PGP_SECRET | base64 --decode > gpg_key
+
+gpg --import  --no-tty --batch --yes gpg_key
+
+rm gpg_key
+
+# Build all artifacts
+./mill -i __.publishArtifacts
+
+export MILL_BUILD_COMPILER_BRIDGES=true
+
+# Publish all artifacts
+./mill -i \
+    mill.scalalib.PublishModule/publishAll \
+    --sonatypeCreds $SONATYPE_USERNAME:$SONATYPE_PASSWORD \
+    --gpgArgs --passphrase=$SONATYPE_PGP_PASSWORD,--no-tty,--pinentry-mode,loopback,--batch,--yes,-a,-b \
+    --publishArtifacts bazelRemoteApis.publishArtifacts \
+    --readTimeout  3600000 \
+    --awaitTimeout 3600000 \
+    --release true \
+    --signed  true

main/api/src/mill/api/JsonFormatters.scala

@@ -11,11 +11,21 @@ object JsonFormatters extends JsonFormatters
  * Defines various default JSON formatters used in mill.
  */
 trait JsonFormatters {
+  private lazy val coursierCache = os.Path(coursier.paths.CoursierPaths.cacheDirectory())
+  def pathToString(path: os.Path) =
+    if (path.startsWith(os.pwd)) path.relativeTo(os.pwd).toString()
+    else if (path.startsWith(coursierCache))
+      "$COURSIER_CACHE/" + path.relativeTo(coursierCache)
+    else path.toString()
+
+  def stringToPath(s: String) = os.FilePath(s) match {
+    case p: os.Path => p
+    case s: os.SubPath =>
+      if (s.segments(0) == "$COURSIER_CACHE") coursierCache / s.segments.drop(1)
+      else os.pwd / s
+  }
   implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String]
-    .bimap[os.Path](
-      _.toString,
-      os.Path(_)
-    )
+    .bimap[os.Path](pathToString, stringToPath)
 
   implicit val regexReadWrite: RW[Regex] = upickle.default.readwriter[String]
     .bimap[Regex](

main/api/src/mill/api/PathRef.scala

@@ -6,6 +6,8 @@ import java.util.concurrent.ConcurrentHashMap
 import scala.util.{DynamicVariable, Using}
 import upickle.default.{ReadWriter => RW}
 
+import scala.collection.mutable
+
 /**
  * A wrapper around `os.Path` that calculates it's hashcode based
  * on the contents of the filesystem underneath it. Used to ensure filesystem
@@ -40,8 +42,11 @@ case class PathRef private (
       case PathRef.Revalidate.Always => "vn:"
     }
     val sig = String.format("%08x", this.sig: Integer)
-    quick + valid + sig + ":" + path.toString()
+
+    quick + valid + sig + ":" + JsonFormatters.pathToString(path)
   }
+
+  override def hashCode(): Int = toString.hashCode
 }
 
 object PathRef {
@@ -80,6 +85,15 @@ object PathRef {
   class PathRefValidationException(val pathRef: PathRef)
       extends RuntimeException(s"Invalid path signature detected: ${pathRef}")
 
+  private val gatheredPathRefs: DynamicVariable[mutable.Set[PathRef]] =
+    new DynamicVariable(null)
+
+  def gatherSerializedPathRefs[T](t: => T): (T, Set[PathRef]) = {
+    val refs = mutable.Set[PathRef]()
+    val res = gatheredPathRefs.withValue(refs)(t)
+    (res, refs.toSet)
+  }
+
   sealed trait Revalidate
   object Revalidate {
     case object Never extends Revalidate
@@ -154,19 +168,21 @@ object PathRef {
 
       java.util.Arrays.hashCode(digest.digest())
     }
-
     new PathRef(path, quick, sig, revalidate)
   }
 
   /**
    * Default JSON formatter for [[PathRef]].
    */
   implicit def jsonFormatter: RW[PathRef] = upickle.default.readwriter[String].bimap[PathRef](
-    p => p.toString(),
+    p => {
+      Option(gatheredPathRefs.value).foreach(_.add(p))
+      p.toString()
+    },
     s => {
       val Array(prefix, valid0, hex, pathString) = s.split(":", 4)
 
-      val path = os.Path(pathString)
+      val path = JsonFormatters.stringToPath(pathString)
       val quick = prefix match {
         case "qref" => true
         case "ref" => false
@@ -181,6 +197,7 @@ object PathRef {
       val sig = java.lang.Long.parseLong(hex, 16).toInt
       val pr = PathRef(path, quick, sig, validOrig)
       validatedPaths.value.revalidateIfNeededOrThrow(pr)
+      Option(gatheredPathRefs.value).foreach(_.add(pr))
       pr
     }
   )

main/define/src/mill/define/BaseModule.scala

@@ -11,12 +11,12 @@ abstract class BaseModule(
 )(implicit
     millModuleEnclosing0: sourcecode.Enclosing,
     millModuleLine0: sourcecode.Line,
-    millFile0: sourcecode.File,
+    millFile0: sourcecode.FileName,
     caller: Caller
 ) extends Module.BaseClass()(
       mill.define.Ctx.make(
-        implicitly,
-        implicitly,
+        millModuleEnclosing0,
+        millModuleLine0,
         Ctx.BasePath(millSourcePath0),
         Segments(),
         Ctx.External(external0),
@@ -25,6 +25,27 @@ abstract class BaseModule(
         caller
       )
     ) with Module {
+
+  @deprecated("bincompat stub")
+  def this(
+      millSourcePath0: os.Path,
+      external0: Boolean,
+      foreign0: Option[Segments],
+      millModuleEnclosing0: sourcecode.Enclosing,
+      millModuleLine0: sourcecode.Line,
+      millFile0: sourcecode.File,
+      caller: Caller
+  ) = this(
+    millSourcePath0,
+    external0,
+    foreign0
+  )(
+    millModuleEnclosing0,
+    millModuleLine0,
+    sourcecode.FileName(millFile0.value),
+    caller
+  )
+
   // A BaseModule should provide an empty Segments list to it's children, since
   // it is the root of the module tree, and thus must not include it's own
   // sourcecode.Name as part of the list,

main/define/src/mill/define/Ctx.scala

@@ -84,7 +84,7 @@ object Ctx {
       segments0: Segments,
       external0: External,
       foreign0: Foreign,
-      fileName: sourcecode.File,
+      fileName: sourcecode.FileName,
       enclosing: Caller
   ): Ctx = {
     Impl(
@@ -105,4 +105,26 @@ object Ctx {
       Seq()
     )
   }
+  @deprecated("bincompat stub")
+  def make(
+      millModuleEnclosing0: sourcecode.Enclosing,
+      millModuleLine0: sourcecode.Line,
+      millModuleBasePath0: BasePath,
+      segments0: Segments,
+      external0: External,
+      foreign0: Foreign,
+      fileName: sourcecode.File,
+      enclosing: Caller
+  ): Ctx = {
+    make(
+      millModuleEnclosing0,
+      millModuleLine0,
+      millModuleBasePath0,
+      segments0,
+      external0,
+      foreign0,
+      sourcecode.FileName(fileName.value),
+      enclosing
+    )
+  }
 }

main/define/src/mill/define/Segments.scala

@@ -49,4 +49,30 @@ object Segments {
   def apply(): Segments = new Segments(Nil)
   def apply(items: Seq[Segment]): Segments = new Segments(items)
   def labels(values: String*): Segments = Segments(values.map(Segment.Label))
+
+  def checkPatternMatch(pattern0: Segments, input0: Segments): Boolean = {
+    def rec(pattern: List[Segment], input: List[Segment]): Boolean = {
+      import Segment.{Label, Cross}
+      (pattern, input) match {
+        case (Nil, Nil) => true
+        case (Label("__") :: pRest, iRest) => iRest.tails.exists(rec(pRest, _))
+        case (Label("_") :: pRest, _ :: iRest) => rec(pRest, iRest)
+
+        case (Label(a) :: pRest, Label(b) :: iRest) if a == b => rec(pRest, iRest)
+
+        case (Cross(as) :: pRest, Cross(bs) :: iRest) =>
+          val crossMatches = as.zip(bs).forall {
+            case ("_", _) => true
+            case (a, b) => a == b
+          }
+
+          if (crossMatches) rec(pRest, iRest)
+          else false
+
+        case (_, _) => false
+      }
+    }
+
+    rec(pattern0.value.toList, input0.value.toList)
+  }
 }