Merge pull request scalacenter#1774 from Duhemm/source-generators

Add support for source generators in Bloop

Author: tgodzik

config/src/main/scala-2.10/bloop/config/ConfigCodecs.scala

@@ -218,6 +218,9 @@ object ConfigCodecs {
   implicit val sourcesGlobsEncoder: ObjectEncoder[SourcesGlobs] = deriveEncoder
   implicit val sourcesGlobsDecoder: Decoder[SourcesGlobs] = deriveDecoder
 
+  implicit val sourceGeneratorEncoder: ObjectEncoder[SourceGenerator] = deriveEncoder
+  implicit val sourceGeneratorDecoder: Decoder[SourceGenerator] = deriveDecoder
+
   implicit val projectEncoder: ObjectEncoder[Project] = deriveEncoder
   implicit val projectDecoder: Decoder[Project] = deriveDecoder
 

config/src/main/scala/bloop/config/Config.scala

@@ -236,6 +236,12 @@ object Config {
       excludes: List[String]
   )
 
+  case class SourceGenerator(
+      sourcesGlobs: List[SourcesGlobs],
+      outputDirectory: Path,
+      command: List[String]
+  )
+
   case class Project(
       name: String,
       directory: Path,
@@ -254,12 +260,13 @@ object Config {
       test: Option[Test],
       platform: Option[Platform],
       resolution: Option[Resolution],
-      tags: Option[List[String]]
+      tags: Option[List[String]],
+      sourceGenerators: Option[List[SourceGenerator]]
   )
 
   object Project {
     // FORMAT: OFF
-    private[bloop] val empty: Project = Project("", emptyPath, None, List(), None, None, List(), List(),  emptyPath, emptyPath, None, None, None, None, None, None, None, None)
+    private[bloop] val empty: Project = Project("", emptyPath, None, List(), None, None, List(), List(),  emptyPath, emptyPath, None, None, None, None, None, None, None, None, None)
     // FORMAT: ON
 
     def analysisFileName(projectName: String): String = s"$projectName-analysis.bin"
@@ -328,6 +335,7 @@ object Config {
         Some(Test(List(), TestOptions(Nil, Nil))),
         Some(platform),
         Some(Resolution(Nil)),
+        None,
         None
       )
 

docs/assets/bloop-schema.json

@@ -4,6 +4,44 @@
   "title": "Bloop configuration file ",
   "description": "The JSON Schema for the bloop configuration file.",
   "definitions": {
+    "sourcesGlobs": {
+      "$id": "/properties/project/properties/sourcesGlobs",
+      "type": "object",
+      "properties": {
+        "directory": {
+          "$id": "/properties/project/properties/sourcesGlobs/properties/directory",
+          "title": "The Directory Schema",
+          "type": "string",
+          "examples": [
+            "/home/code/src"
+          ]
+        },
+        "walkDepth": {
+          "$id": "/properties/project/properties/sourcesGlobs/properties/walkDepth",
+          "title": "The WalkDepth Schema",
+          "type": "number",
+          "examples": [
+            1
+          ]
+        },
+        "includes": {
+          "$id": "/properties/project/properties/sourcesGlobs/properties/includes",
+          "title": "The Includes Schema",
+          "type": "string",
+          "examples": [
+            "glob:*.scala"
+          ]
+        },
+        "excludes": {
+          "$id": "/properties/project/properties/sourcesGlobs/properties/excludes",
+          "title": "The Excludes Schema",
+          "type": "string",
+          "examples": [
+            "glob:*Test.scala"
+          ]
+        }
+      }
+    },
     "jvmPlatform": {
       "$id": "/properties/project/properties/jvmPlatform",
       "type": "object",
@@ -372,6 +410,13 @@
             ]
           }
         },
+        "sourcesGlobs": {
+          "$id": "/properties/project/properties/sourcesGlobs",
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/sourcesGlobs"
+          }
+        },
         "dependencies": {
           "$id": "/properties/project/properties/dependencies",
           "type": "array",
@@ -656,6 +701,42 @@
             ]
           }
         }
+      },
+      "sourceGenerators": {
+        "$id": "/properties/project/properties/sourceGenerators",
+        "type": "array",
+        "title": "The SourceGenerators Schema",
+        "description": "The source generators used by this project",
+        "items": {
+          "$id": "/properties/project/properties/sourceGenerators/items",
+          "type": "object",
+          "properties": {
+            "sourcesGlobs": {
+              "$id": "/properties/project/properties/sourceGenerators/properties/sourcesGlobs",
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/sourcesGlobs"
+              }
+            },
+            "outputDirectory": {
+              "$id": "/properties/project/properties/sourceGenerators/properties/outputDirectory",
+              "type": "string",
+              "examples": [
+                "/home/code/generated-sources"
+              ]
+            },
+            "command": {
+              "$id": "/properties/project/properties/sourceGenerators/properties/command",
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "examples": [
+                ["/bin/my-generator.sh", "arg0", "arg1"]
+              ]
+            }
+          }
+        }
       }
     }
   },

frontend/src/it/scala/bloop/CommunityBuild.scala

@@ -21,7 +21,7 @@ import bloop.engine.{
   Run,
   State
 }
-import bloop.engine.caches.ResultsCache
+import bloop.engine.caches.{ResultsCache, SourceGeneratorCache}
 import bloop.io.AbsolutePath
 import bloop.logging.{BloopLogger, Logger, NoopLogger}
 import bloop.task.Task
@@ -95,7 +95,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) {
     val loadedProjects = BuildLoader.loadSynchronously(configDirectory, logger)
     val workspaceSettings = WorkspaceSettings.readFromFile(configDirectory, logger)
     val build = Build(configDirectory, loadedProjects, workspaceSettings)
-    val state = State.forTests(build, compilerCache, logger)
+    val state = State.forTests(build, compilerCache, SourceGeneratorCache.empty, logger)
     state.copy(results = ResultsCache.emptyForTests)
   }
 
@@ -139,6 +139,7 @@ abstract class CommunityBuild(val buildpressHomeDir: AbsolutePath) {
         sources = Nil,
         sourcesGlobs = Nil,
         sourceRoots = None,
+        sourceGenerators = Nil,
         testFrameworks = Nil,
         testOptions = Config.TestOptions.empty,
         out = dummyClassesDir,

frontend/src/main/scala/bloop/bsp/BloopBspServices.scala

@@ -1066,24 +1066,47 @@ final class BloopBspServices(
     ): BspResult[bsp.SourcesResult] = {
       val sourcesItems = projects.iterator.map {
         case (target, project) =>
-          project.allSourceFilesAndDirectories.map { sources =>
-            val items = sources.map { s =>
-              import bsp.SourceItemKind._
-              val uri = s.underlying.toUri()
-              val (bspUri, kind) = if (s.exists) {
-                (uri, if (s.isFile) File else Directory)
-              } else {
-                val fileMatcher = FileSystems.getDefault.getPathMatcher("glob:*.{scala, java}")
-                if (fileMatcher.matches(s.underlying.getFileName)) (uri, File)
-                // If path doesn't exist and its name doesn't look like a file, assume it's a dir
-                else (new URI(uri.toString + "/"), Directory)
-              }
-              // TODO(jvican): Don't default on false for generated, add this info to JSON fields
-              bsp.SourceItem(bsp.Uri(bspUri), kind, false)
+          def sourceItem(s: AbsolutePath, isGenerated: Boolean): bsp.SourceItem = {
+            import bsp.SourceItemKind._
+            val uri = s.underlying.toUri()
+            val (bspUri, kind) = if (s.exists) {
+              (uri, if (s.isFile) File else Directory)
+            } else {
+              val fileMatcher = FileSystems.getDefault.getPathMatcher("glob:*.{scala, java}")
+              if (fileMatcher.matches(s.underlying.getFileName)) (uri, File)
+              // If path doesn't exist and its name doesn't look like a file, assume it's a dir
+              else (new URI(uri.toString + "/"), Directory)
             }
-            val roots = project.sourceRoots.map(_.map(p => bsp.Uri(p.toBspUri)))
-            bsp.SourcesItem(target, items, roots)
+            bsp.SourceItem(bsp.Uri(bspUri), kind, isGenerated)
           }
+
+          val unmanagedSources = project.allUnmanagedSourceFilesAndDirectories.map { sources =>
+            sources.map(sourceItem(_, isGenerated = false))
+          }
+
+          val parentGenerators = Dag.dfs(state.build.getDagFor(project))
+          val managedSources = Task
+            .sequence {
+              parentGenerators.reverse.map { project =>
+                val generators = project.sourceGenerators
+                val tasks = generators.map(
+                  state.sourceGeneratorCache.update(_, state.logger, state.commonOptions)
+                )
+                Task.gatherUnordered(tasks)
+              }
+            }
+            .map {
+              // Take only the result of the last set of generators, since these are the generated
+              // sources of the project we're interested in.
+              _.lastOption.map(_.flatten.map(sourceItem(_, isGenerated = true))).getOrElse(Nil)
+            }
+            .executeOn(ioScheduler)
+
+          val roots = project.sourceRoots.map(_.map(p => bsp.Uri(p.toBspUri)))
+          for {
+            unmanaged <- unmanagedSources
+            managed <- managedSources
+          } yield bsp.SourcesItem(target, unmanaged ++ managed, roots)
       }.toList
 
       Task.sequence(sourcesItems).map { items =>

frontend/src/main/scala/bloop/data/Project.scala

@@ -14,6 +14,7 @@ import bloop.bsp.ProjectUris
 import bloop.config.Config
 import bloop.config.ConfigCodecs
 import bloop.engine.Dag
+import bloop.engine.SourceGenerator
 import bloop.engine.tasks.toolchains.JvmToolchain
 import bloop.engine.tasks.toolchains.ScalaJsToolchain
 import bloop.engine.tasks.toolchains.ScalaNativeToolchain
@@ -45,6 +46,7 @@ final case class Project(
     sources: List[AbsolutePath],
     sourcesGlobs: List[SourcesGlobs],
     sourceRoots: Option[List[AbsolutePath]],
+    sourceGenerators: List[SourceGenerator],
     testFrameworks: List[Config.TestFramework],
     testOptions: Config.TestOptions,
     out: AbsolutePath,
@@ -88,8 +90,11 @@ final case class Project(
     customWorkingDirectory.getOrElse(baseDirectory)
   }
 
+  def allGeneratorInputs: Task[List[AbsolutePath]] =
+    Task.sequence(sourceGenerators.map(_.getSources)).map(_.flatten)
+
   /** Returns concatenated list of "sources" and expanded "sourcesGlobs". */
-  def allSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task {
+  def allUnmanagedSourceFilesAndDirectories: Task[List[AbsolutePath]] = Task {
     val buf = mutable.ListBuffer.empty[AbsolutePath]
     buf ++= sources
     for (glob <- sourcesGlobs) glob.walkThrough(buf += _)
@@ -354,10 +359,11 @@ object Project {
     val sourceRoots = project.sourceRoots.map(_.map(AbsolutePath.apply))
 
     val tags = project.tags.getOrElse(Nil)
+    val projectDirectory = AbsolutePath(project.directory)
 
     Project(
       project.name,
-      AbsolutePath(project.directory),
+      projectDirectory,
       project.workspaceDir.map(AbsolutePath.apply),
       project.dependencies,
       instance,
@@ -370,6 +376,7 @@ object Project {
       project.sources.map(AbsolutePath.apply),
       SourcesGlobs.fromConfig(project, logger),
       sourceRoots,
+      project.sourceGenerators.getOrElse(Nil).map(SourceGenerator.fromConfig(projectDirectory, _)),
       project.test.map(_.frameworks).getOrElse(Nil),
       project.test.map(_.options).getOrElse(Config.TestOptions.empty),
       AbsolutePath(project.out),

frontend/src/main/scala/bloop/engine/Interpreter.scala

@@ -118,7 +118,12 @@ object Interpreter {
       f: State => Task[State]
   ): Task[State] = {
     val reachable = Dag.dfs(getProjectsDag(projects, state))
-    val projectsSourcesAndDirs = reachable.map(_.allSourceFilesAndDirectories)
+    val projectsSourcesAndDirs = reachable.map { project =>
+      for {
+        unmanaged <- project.allUnmanagedSourceFilesAndDirectories
+        generatorSourceDirs = project.sourceGenerators.flatMap(_.sourcesGlobs.map(_.directory))
+      } yield unmanaged ++ generatorSourceDirs
+    }
     val groupTasks =
       projectsSourcesAndDirs.grouped(8).map(group => Task.gatherUnordered(group)).toList
     Task