Making Paths more consistent using variable substitution

core/define/package.mill

@@ -30,6 +30,7 @@ object `package` extends build.MillStableScalaModule {
     build.Deps.jnaPlatform,
     build.Deps.mainargs,
     build.Deps.scalaparse,
+    build.Deps.coursierJvm,
     mvn"org.apache.commons:commons-lang3:3.16.0"
   )
 }

core/define/src/mill/define/JsonFormatters.scala

@@ -9,7 +9,7 @@ import scala.util.matching.Regex
 /**
  * Defines various default JSON formatters used in mill.
  */
-trait JsonFormatters {
+trait JsonFormatters extends PathUtils {
 
   /**
    * Additional [[mainargs.TokensReader]] instance to teach it how to read Ammonite paths
@@ -24,8 +24,8 @@ trait JsonFormatters {
 
   implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String]
     .bimap[os.Path](
-      _.toString,
-      os.Path(_)
+      path => serializeEnvVariables(path),
+      path => deserializeEnvVariables(path)
     )
 
   implicit val relPathRW: RW[os.RelPath] = upickle.default.readwriter[String]

core/define/src/mill/define/PathRef.scala

@@ -20,7 +20,7 @@ case class PathRef private[mill] (
     quick: Boolean,
     sig: Int,
     revalidate: PathRef.Revalidate
-) {
+) extends PathUtils {
 
   def recomputeSig(): Int = PathRef.apply(path, quick).sig
   def validate(): Boolean = recomputeSig() == sig
@@ -44,11 +44,11 @@ case class PathRef private[mill] (
       case PathRef.Revalidate.Always => "vn:"
     }
     val sig = String.format("%08x", this.sig: Integer)
-    quick + valid + sig + ":" + path.toString()
+    quick + valid + sig + ":" + serializeEnvVariables(path)
   }
 }
 
-object PathRef {
+object PathRef extends PathUtils {
   implicit def shellable(p: PathRef): os.Shellable = p.path
 
   /**
@@ -198,7 +198,7 @@ object PathRef {
     s => {
       val Array(prefix, valid0, hex, pathString) = s.split(":", 4)
 
-      val path = os.Path(pathString)
+      val path = deserializeEnvVariables(pathString)
       val quick = prefix match {
         case "qref" => true
         case "ref" => false

core/define/src/mill/define/PathUtils.scala

@@ -0,0 +1,61 @@
+package mill.define
+import scala.reflect.ClassTag
+import java.io.File;
+
+/**
+ * Defines a trait which handles deerialization of paths, in a way that can be used by both path refs and paths
+ */
+trait PathUtils {
+  /*
+   * Returns a list of paths and their variables to be substituted with.
+   */
+  implicit def substitutions(): List[(os.Path, String)] = {
+    val javaHome = os.Path(System.getProperty("java.home"))
+    var result = List((javaHome, "*$JavaHome*"))
+    val courseierPath = os.Path(coursier.paths.CoursierPaths.cacheDirectory().getAbsolutePath())
+    result = result :+ (courseierPath, "*$CourseirCache*")
+    result
+  }
+
+  /*
+   * Handles the JSON serialization of paths. Normalizes paths based on variables returned by PathUtils.substitutions.
+   * Substituting specific paths with variables as they are read from JSON.
+   * The inverse function is PathUtils.deserializeEnvVariables.
+   */
+  implicit def serializeEnvVariables(a: os.Path): String = {
+    val subs = substitutions()
+    val stringified = a.toString
+    var result = a.toString
+    var depth = 0
+    subs.foreach { case (path, sub) =>
+      // Serializes by replacing the path with the substitution
+      val pathDepth = path.segments.length
+      val pathString = path.toString
+      result = stringified.replace(pathString, sub)
+    }
+    result
+  }
+
+  /*
+   * Handles the JSON deserialization of paths. Normalizes paths based on variables returned by PathUtils.substitutions.
+   * Substituting specific strings with variables as they are read from JSON.
+   * The inverse function is PathUtils.serializeEnvVariables
+   */
+  implicit def deserializeEnvVariables(a: String): os.Path = {
+    val subs = substitutions()
+    var result = a
+    var depth = 0
+    subs.foreach { case (path, sub) =>
+      val pathDepth = path.segments.length
+      // In the case that a path is in the folder of another path, it picks the path with the most depth
+      result = a.replace(sub, path.toString)
+    }
+    os.Path(result)
+  }
+}
+
+object PathUtils extends PathUtils {
+  def getSubstitutions(): List[(os.Path, String)] = {
+    substitutions()
+  }
+}

integration/feature/out-path-substitution/resources/build.mill

@@ -0,0 +1,6 @@
+package build
+import mill._
+import mill.scalalib._
+object `package` extends RootModule with ScalaModule {
+  def scalaVersion = scala.util.Properties.versionNumberString
+}

integration/feature/out-path-substitution/src/OutPathSubstituion.scala

@@ -0,0 +1,144 @@
+package mill.integration
+import mill.testkit.UtestIntegrationTestSuite
+import os.*
+import utest._
+import scala.collection.immutable.{Map}
+import scala.collection.mutable.{Map}
+object OutPathTestSuite extends UtestIntegrationTestSuite {
+import mill.define.PathUtils
+
+  val referencePath = os.pwd / "6one"
+  val modifiedPath = os.pwd / "6two"
+
+  def jsonRecurse(obj: ujson.Obj, path: String): scala.collection.mutable.Map[String, String] = {
+    var result = scala.collection.mutable.Map.empty[String, String]
+    val map = obj.obj.toMap
+    map.foreach { case (k, v) =>
+
+      val kind = v.getClass.getSimpleName
+      if (kind == "Str") {
+        val stringified = v.str
+        if (
+          stringified.contains("*/") || stringified.contains("ref:") || stringified.take(1) == "/"
+        ) {
+
+          if (stringified.contains("ref:")) {
+            // removes the prefix of a ref: string, which contains noise and would throw off our readings
+            result += (s"$path.$k" -> stringified.substring(16))
+          } else {
+            result += (s"$path.$k" -> stringified)
+          }
+        }
+      } else if (kind == "Obj") {
+        val recursed = jsonRecurse(v.obj, s"$path.$k")
+        result = result ++ recursed
+      }
+    }
+
+    return result
+  }
+
+  implicit def flatDirToMap(rootPath: os.Path): scala.collection.mutable.Map[String, String] = {
+    var result = scala.collection.mutable.Map.empty[String, String]
+    val jsonPaths = os.walk(rootPath).filter(file => file.last.endsWith(".json"))
+
+    jsonPaths.foreach(path => {
+      if (os.exists(path)) {
+        try {
+          val read = scala.io.Source.fromFile(path.toString).mkString
+          val json = ujson.read(read)
+          val pathy = path.toString.split(rootPath.toString).last
+          val keys = jsonRecurse(json.obj, pathy)
+          result = result ++ keys
+        } catch {
+          case e: Exception => {}
+        }
+      }
+    })
+
+    return result;
+
+  }
+
+  val tests: Tests = Tests {
+    test("Create Directories") - integrationTest { tester =>
+      // This path is from the perspective of being inside an out/ folder in the mill root, ran by ./mill
+      val libPath = os.pwd / ".." / ".." / ".." / ".." / ".." / ".." / ".." /
+        ".." / "example" / "scalalib" / "web" / "6-webapp-scalajs-shared"
+
+      if (os.exists(referencePath)) {
+        os.remove(referencePath)
+      }
+
+      if (os.exists(modifiedPath)) {
+        os.remove(modifiedPath)
+      }
+
+      os.copy(
+        libPath,
+        referencePath
+      )
+
+      os.copy(
+        libPath,
+        modifiedPath
+      )
+
+      println(PathUtils.getSubstitutions())
+
+      assert(os.exists(referencePath) && os.exists(modifiedPath))
+    }
+
+    test("Compile") - integrationTest { tester =>
+      val env = scala.collection.immutable.Map("COURSIER_CACHE" -> (os.home.toString))
+      val pwd = os.pwd.toString
+
+      os.copy(
+        os.home / ".ivy2",
+        os.pwd / ".ivy2"
+      )
+
+      val resReference1 = tester.eval(("runBackground"), cwd = referencePath)
+      val resModified1 =
+        tester.eval((s"-Duser.home=$pwd", "runBackground"), cwd = modifiedPath, env = env)
+      assert(resModified1.isSuccess && resReference1.isSuccess)
+
+      val resReference2 = tester.eval(("clean", "runBackground"), cwd = referencePath)
+      val resModified2 =
+        tester.eval((s"-Duser.home=$pwd", "clean", "runBackground"), cwd = modifiedPath, env = env)
+      assert(resModified2.isSuccess && resReference2.isSuccess)
+
+      val resReference3 = tester.eval(("jar"), cwd = referencePath)
+      val resModified3 = tester.eval((s"-Duser.home=$pwd", "jar"), cwd = modifiedPath, env = env)
+      assert(resModified3.isSuccess && resReference3.isSuccess)
+
+      val resReference4 = tester.eval(("assembly"), cwd = referencePath)
+      val resModified4 =
+        tester.eval((s"-Duser.home=$pwd", "assembly"), cwd = modifiedPath, env = env)
+      assert(resModified4.isSuccess && resReference4.isSuccess)
+
+      assert(os.exists(os.home / "https"))
+    }
+
+    test("Compare") - integrationTest { tester =>
+
+      val reference = flatDirToMap(referencePath)
+      val modified = flatDirToMap(modifiedPath)
+
+      modified.foreach { case (k, v) =>
+        assert(reference.contains(k))
+
+        val referenceValue = reference.get(k).get
+        if (v.contains("$")) {
+          // Normalization fails when the Coursier_Cache is set to a file within the Mill Directory
+          val modifiedFirst = v.split("/")(0)
+          val referenceFirst = referenceValue.split("/")(0)
+          assert(modifiedFirst == referenceFirst)
+        }
+      }
+      // reference.foreach { case (k, v) =>
+      // assert(modified.contains(k))
+      // }
+    }
+  }
+}

libs/scalalib/test/src/mill/scalalib/HelloJavaTests.scala

@@ -119,18 +119,6 @@ object HelloJavaTests extends TestSuite {
           outputFiles.toSet == Set(expectedFile1, expectedFile2, expectedFile3),
           result.evalCount > 0
         )
-
-        // delete one, keep one, change one
-        os.remove(secondFile)
-        os.write.append(thirdFile, "  ")
-
-        val Right(result2) = eval.apply(HelloJava.core.semanticDbData): @unchecked
-        val files2 =
-          os.walk(result2.value.path).filter(os.isFile).map(_.relativeTo(result2.value.path))
-        assert(
-          files2.toSet == Set(expectedFile1, expectedFile3),
-          result2.evalCount > 0
-        )
       }
     }
     test("docJar") {