Merge branch 'develop'

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Cromwell Change Log
 
+## 80 Release Notes
+
+### Direct WES support in Cromwell
+
+Cromwell 80 no longer supports the wes2cromwell project within the Cromwell repository.
+
+In the previous release, 3 Wes2Cromwell endpoints in the Cromwell project were implemented and documented in the Swagger API. Three new endpoints,
+located within the wes2cromwell project, will also be moved, implemented, and documented within Cromwell. As a result of this, we can safely remove 
+and deprecate the wes2cromwell project from the repo.
+
+Previous endpoints:
+
+| HTTP verb | Endpoint path | Description   |
+| --------- | ------------- |---------------|
+| GET | /api/ga4gh/wes/v1/service-info | Server info |
+| POST | /api/ga4gh/wes/v1/runs/{run_id}/cancel | Abort workflow |
+| GET | /api/ga4gh/wes/v1/runs/{run_id}/status | Workflow status |
+
+Newly implemented endpoints:
+
+| HTTP verb | Endpoint path | Description     |
+| --------- | ------------- |-----------------|
+| GET | /api/ga4gh/wes/v1/runs | List workflows  |
+| POST | /api/ga4gh/wes/v1/runs | Submit workflow |
+| GET | /api/ga4gh/wes/v1/runs/{run_id} | Workflow details |
+
 ## 79 Release Notes
 
 ### Last release with CWL support

engine/src/main/resources/swagger/cromwell.yaml

@@ -705,6 +705,54 @@ paths:
         '500':
           description: An unexpected error occurred.
           $ref: '#/responses/ServerError'
+  '/api/ga4gh/wes/v1/runs':
+    get:
+      tags:
+        - GA4GH Workflow Execution Service (WES) - Alpha preview
+      summary: List runs
+      description: >-
+        Runs are listed from newest to oldest. When paging through the list, the client
+        should not make assumptions about live updates, but should assume the
+        contents of the list reflect the workflow list at the moment that the
+        first page is requested. To monitor a specific workflow run, use
+        GetRunStatus or GetRunLog.
+      operationId: ListRuns
+      parameters:
+        - name: page_size
+          in: query
+          description: >-
+            OPTIONAL The preferred number of workflow runs to return in a page.
+            If not provided, the implementation should use a default page size.
+            The implementation must not return more items than `page_size`, but
+            it may return fewer.  Clients should not assume that if fewer than
+            `page_size` items are returned that all items have been returned. 
+            The availability of additional pages is indicated by the value of
+            `next_page_token` in the response.
+          type: integer
+          format: int64
+        - name: page_token
+          in: query
+          description: >-
+            OPTIONAL Token to use to indicate where to start getting results. If
+            unspecified, return the first page of results.
+          type: string
+      responses:
+        '200':
+          description: Successful Request
+          schema:
+            $ref: '#/definitions/RunListResponse'
+        '400':
+          description: The request is malformed.
+          $ref: '#/responses/BadRequest'
+        '401':
+          description: The request is unauthorized.
+          $ref: '#/responses/InvalidSubmission'
+        '403':
+          description: The requester is not authorized to perform this action.
+          $ref: '#/responses/Forbidden'
+        '500':
+          description: An unexpected error occurred.
+          $ref: '#/responses/ServerError'
   '/api/ga4gh/wes/v1/runs/{run_id}/cancel':
     post:
       tags:
@@ -1768,3 +1816,4 @@ definitions:
           are also allowed. The version should be changed whenever the service
           is updated.
         example: 1.0.0
+

engine/src/main/scala/cromwell/server/CromwellServer.scala

@@ -11,6 +11,7 @@ import cromwell.services.instrumentation.CromwellInstrumentationActor
 import cromwell.webservice.SwaggerService
 import cromwell.webservice.routes.CromwellApiService
 import cromwell.webservice.routes.wes.WesRouteSupport
+import cromwell.webservice.routes.wes.WesRunRoutes
 
 import scala.concurrent.Future
 import scala.util.{Failure, Success}
@@ -36,6 +37,7 @@ class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Bool
     with CromwellApiService
     with CromwellInstrumentationActor
     with WesRouteSupport
+    with WesRunRoutes
     with SwaggerService
     with ActorLogging {
   implicit val actorSystem = context.system
@@ -51,7 +53,7 @@ class CromwellServerActor(cromwellSystem: CromwellSystem, gracefulShutdown: Bool
     * cromwell.yaml is broken unless the swagger index.html is patched. Copy/paste the code from rawls or cromiam if
     * actual cromwell+swagger+oauth+/api support is needed.
     */
-  val apiRoutes: Route = pathPrefix("api")(concat(workflowRoutes, womtoolRoutes, wesRoutes))
+  val apiRoutes: Route = pathPrefix("api")(concat(workflowRoutes, womtoolRoutes, wesRoutes, runRoutes))
   val nonApiRoutes: Route = concat(engineRoutes, swaggerUiResourceRoute)
   val allRoutes: Route = concat(apiRoutes, nonApiRoutes)
 

engine/src/main/scala/cromwell/webservice/routes/wes/RunListResponse.scala

@@ -0,0 +1,21 @@
+package cromwell.webservice.routes.wes
+
+import akka.http.scaladsl.model.StatusCodes
+import cromwell.services.metadata.MetadataService
+import cromwell.services.metadata.MetadataService.{WorkflowQueryFailure, WorkflowQuerySuccess}
+import cromwell.webservice.routes.wes.WesState.fromStatusString
+import cromwell.webservice.WebServiceUtils._
+
+case class RunListResponse(runs: List[WesRunStatus], next_page_token: String)
+
+object RunListResponse {
+  def fromMetadataQueryResponse(response: MetadataService.MetadataQueryResponse): WesResponse = {
+
+      response match {
+      case w: WorkflowQuerySuccess =>
+        val runs = w.response.results.toList.map(x => WesRunStatus(x.id, fromStatusString(x.status)))
+        WesResponseRunList(runs)
+      case f: WorkflowQueryFailure => WesErrorResponse(f.reason.errorRequest(StatusCodes.BadRequest).toString, StatusCodes.BadRequest.intValue)
+    }
+  }
+}

engine/src/main/scala/cromwell/webservice/routes/wes/WesResponse.scala

@@ -8,6 +8,8 @@ sealed trait WesResponse extends Product with Serializable
 final case class WesErrorResponse(msg: String, status_code: Int) extends WesResponse
 final case class WesRunId(run_id: String) extends WesResponse
 final case class WesRunStatus(run_id: String, state: WesState) extends WesResponse
+final case class WesResponseRunList(runs: List[WesRunStatus]) extends WesResponse
+
 
 final case class WesStatusInfoResponse(workflow_type_version: Map[String, Iterable[String]],
                                     supported_wes_versions: Iterable[String],
@@ -27,6 +29,7 @@ object WesResponseJsonSupport extends SprayJsonSupport with DefaultJsonProtocol
   implicit val WesResponseRunIdFormat = jsonFormat1(WesRunId)
   implicit val WesResponseStatusFormat = jsonFormat2(WesRunStatus)
   implicit val WesResponseStatusInfoFormat = jsonFormat9(WesStatusInfoResponse)
+  implicit val WesResponseRunListFormat = jsonFormat1(WesResponseRunList) 
 
   implicit object WesResponseFormat extends RootJsonFormat[WesResponse] {
     import spray.json._
@@ -37,6 +40,7 @@ object WesResponseJsonSupport extends SprayJsonSupport with DefaultJsonProtocol
         case s: WesRunStatus => s.toJson
         case e: WesErrorResponse => e.toJson
         case i: WesStatusInfoResponse => i.toJson
+        case l: WesResponseRunList => l.toJson
       }
     }
 

engine/src/main/scala/cromwell/webservice/routes/wes/WesRunRoutes.scala

@@ -0,0 +1,61 @@
+package cromwell.webservice.routes.wes
+
+import akka.actor.ActorRef
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.server.Directives._
+import akka.http.scaladsl.server.Route
+import akka.http.scaladsl.server.directives.RouteDirectives.complete
+import WesRunRoutes._
+import com.typesafe.config.ConfigFactory
+
+import cromwell.webservice.routes.MetadataRouteSupport.metadataQueryRequest
+
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+import scala.util.{Failure, Success}
+
+trait WesRunRoutes {
+
+  val serviceRegistryActor: ActorRef
+
+  lazy val runRoutes: Route =
+    pathPrefix("ga4gh" / "wes" / "v1") {
+        pathPrefix("runs") {
+          pathEnd {
+            get {
+              parameters(("page_size".as[Int].?, "page_token".?)) { (pageSize, pageToken) =>
+                completeCromwellResponse(listRuns(pageSize, pageToken, serviceRegistryActor))
+              }
+            }
+          }
+        }
+    }
+}
+
+object WesRunRoutes {
+
+import akka.util.Timeout
+import scala.concurrent.duration.FiniteDuration
+import net.ceedubs.ficus.Ficus._
+
+implicit lazy val duration: FiniteDuration = ConfigFactory.load().as[FiniteDuration]("akka.http.server.request-timeout")
+implicit lazy val timeout: Timeout = duration
+
+def completeCromwellResponse(future: => Future[WesResponse]): Route = {
+
+  import WesResponseJsonSupport.WesResponseErrorFormat
+  import cromwell.webservice.routes.wes.WesResponseJsonSupport.WesResponseFormat
+  import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+
+  onComplete(future) {
+    case Success(response: WesResponse) => complete(response)
+    case Failure(e) => complete(WesErrorResponse(e.getMessage, StatusCodes.InternalServerError.intValue))
+  }
+}
+
+def listRuns(pageSize: Option[Int], pageToken: Option[String], serviceRegistryActor: ActorRef): Future[WesResponse] = {
+  // FIXME: to handle - page_size, page_token
+  // FIXME: How to handle next_page_token in response?
+  metadataQueryRequest(Seq.empty[(String, String)], serviceRegistryActor).map(RunListResponse.fromMetadataQueryResponse)
+  }
+}

engine/src/main/scala/cromwell/webservice/routes/wes/WesState.scala

@@ -17,6 +17,12 @@ object WesState {
   case object Canceled extends WesState { override val name = "CANCELED"}
   case object Canceling extends WesState { override val name = "CANCELING"}
 
+  def fromStatusString(status: Option[String]): WesState = {
+    status match {
+      case Some(status) => fromCromwellStatus(WorkflowState.withName(status))
+      case None => Unknown
+    }
+  }
   def fromCromwellStatus(cromwellStatus: WorkflowState): WesState = {
     cromwellStatus match {
       case WorkflowOnHold => Paused

engine/src/test/scala/cromwell/webservice/MetadataBuilderActorSpec.scala

@@ -227,6 +227,36 @@ class MetadataBuilderActorSpec extends TestKitSuite with AsyncFlatSpecLike with
     )
   }
 
+
+  it should "use reverse date ordering (oldest first) for event start and stop values" in {
+    val eventBuilderList = List(
+      ("start", "1990-12-20T12:30:00.000Z", OffsetDateTime.now),
+      ("start", "1990-12-20T12:30:01.000Z", OffsetDateTime.now.plusSeconds(1)),
+      ("end", "2018-06-02T12:30:00.000Z", OffsetDateTime.now.plusSeconds(2)),
+      ("end", "2018-06-02T12:30:01.000Z", OffsetDateTime.now.plusSeconds(3)),
+    )
+    val workflowId = WorkflowId.randomId()
+    val expectedRes =
+      s""""calls": {
+         |    "fqn": [{
+         |      "attempt": 1,
+         |      "end": "2018-06-02T12:30:00.000Z",
+         |      "start": "1990-12-20T12:30:00.000Z",
+         |      "shardIndex": -1
+         |    }]
+         |  },
+         |  "id": "$workflowId"""".stripMargin
+
+    assertMetadataKeyStructure(
+      eventList = eventBuilderList,
+      expectedJson = expectedRes,
+      workflow = workflowId,
+      eventMaker = makeCallEvent,
+      metadataBuilderActorName = "mba-start-end-values",
+    )
+  }
+
+
   it should "build JSON object structure from dotted key syntax" in {
     val eventBuilderList = List(
       ("a:b:c", "abc", OffsetDateTime.now),

project/Dependencies.scala

@@ -63,7 +63,7 @@ object Dependencies {
   private val heterodonV = "1.0.0-beta3"
   private val hsqldbV = "2.6.1"
   private val http4sV = "0.21.31" // this release is EOL. We need to upgrade further for cats3. https://http4s.org/versions/
-  private val jacksonV = "2.13.2"
+  private val jacksonV = "2.13.3"
   private val janinoV = "3.1.6"
   private val jsr305V = "3.0.2"
   private val junitV = "4.13.2"
@@ -619,6 +619,14 @@ object Dependencies {
   Any dependencies that are removed may be also removed from this list.
   However, be careful about downgrading any of these dependencies.
   Older versions have known vulnerabilities, ex: CVE-2017-7525
+
+  === SECURITY UPGRADES ===
+
+  When upgrading dependencies to fix security issues, it is preferable to start with upgrading the
+  library that brings it in. Only fall back to overriding here when the latest library version still
+  has a vulnerable version of the dependency, or a major version upgrade is required and infeasible.
+  This algorithm makes it simpler to upgrade libraries in the future, because we don't have to
+  remember to remove the override.
    */
 
   val googleHttpClientDependencies = List(
@@ -703,6 +711,15 @@ object Dependencies {
     "org.asynchttpclient" % "async-http-client" % "2.10.5",
   )
 
+
+  private val nimbusdsOverrides = List(
+    "com.nimbusds" % "nimbus-jose-jwt" % "9.23",
+  )
+
+  private val bouncyCastleOverrides = List(
+    "org.bouncycastle" % "bcprov-jdk15on" % "1.70",
+  )
+
   /*
   If we use a version in one of our projects, that's the one we want all the libraries to use
   ...plus other groups of transitive dependencies shared across multiple projects
@@ -714,5 +731,7 @@ object Dependencies {
       rdf4jDependencyOverrides ++
       grpcDependencyOverrides ++
       scalaCollectionCompatOverrides ++
-      asyncHttpClientOverrides
+      asyncHttpClientOverrides ++
+      nimbusdsOverrides ++
+      bouncyCastleOverrides
 }

project/Version.scala

@@ -5,7 +5,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "79"
+  val cromwellVersion = "80"
 
   /**
     * Returns true if this project should be considered a snapshot.

project/plugins.sbt

@@ -2,3 +2,4 @@ addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.9.0")
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
 addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.2")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.3")
+addDependencyTreePlugin