Merge branch 'develop'

.github/issue_template.md

@@ -0,0 +1,20 @@
+<!--
+Hi!  Thanks for taking the time to report feedback.
+
+Before posting an issue here, please check whether your question is already answered in our:
+  forum  https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
+  documentation http://cromwell.readthedocs.io/en/develop/
+
+Other forums:
+FireCloud https://gatkforums.broadinstitute.org/firecloud/categories/ask-the-firecloud-team
+WDL https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
+CWL https://www.biostars.org/
+-->
+
+<!-- Are you seeing something that looks like a bug?  Then great!  You're in the right place.  -->
+
+<!-- Which backend are you running?  -->
+
+<!-- Paste/Attach your workflow if possible: -->
+
+<!-- Paste your configuration if possible, MAKE SURE TO OMIT PASSWORDS, TOKENS AND OTHER SENSITIVE MATERIAL: -->

.sbtopts

@@ -1,4 +1,4 @@
 -J-Xms2g
 -J-Xmx4g
 -J-XX:MaxMetaspaceSize=2g
-
+-J-Xss8m

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Cromwell Change Log
 
+## 33 Release Notes
+
+### Query endpoint
+
+#### Exclude workflows based on Labels
+
+This gives the ability to **filter out** workflows based on labels. Two new parameters called `excludeLabelAnd` and `excludeLabelOr` can be used for this purpose.
+More details on how to use them can be found [here](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/).
+
+#### Include/Exclude subworkflows
+
+Cromwell now supports excluding subworkflows from workflow query results using the `includeSubworkflows` parameter. By default they are included in the results.
+More information can be found at [REST API](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/).
+
+#### Query workflows by Submission time
+
+Cromwell now supports querying workflows by submission time. This will help find workflows that are submitted but not started yet (i.e. workflows which are
+in On Hold state). More information can be found [here](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/).
+
+#### Submission time in Workflow Query Response
+
+Submission time of a workflow is now included in WorkflowQueryResult, which is part of the response for workflow query.
+
+### File Localization (NIO) Hint
+
+Cromwell now allows tasks in WDL 1.0 can now specify an optimization in their `parameter_meta` that some `File` inputs do not need to be localized for the task to run successfully.
+Full details are available in the [documentation page for this optimization](http://cromwell.readthedocs.io/en/develop/optimizations/FileLocalization).
+
+### Bug Fixes
+
+Workflows which are in 'On Hold' state can now be fetched using the query endpoint.
+
 ## 32 Release Notes
 
 ### Backends

CromIAM/src/main/resources/swagger/cromiam.yaml

@@ -60,9 +60,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}':
     post:
       summary: Submit a new workflow for execution
@@ -143,9 +143,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/batch':
     post:
       summary: Submit a batch of new workflows for execution
@@ -203,9 +203,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/outputs':
     get:
       summary: Query for workflow outputs based on workflow id
@@ -234,9 +234,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/labels':
     patch:
       summary: Add new labels or update values for existing label keys by workflow id.
@@ -275,9 +275,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/query':
     get:
       summary: Query workflows by start dates, end dates, names, ids, or statuses.
@@ -356,9 +356,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
     post:
       summary: Query workflows by start dates, end dates, names, ids, or statuses.
       parameters:
@@ -391,9 +391,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/status':
     get:
       summary: Query for workflow status based on workflow id
@@ -424,43 +424,45 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/releaseHold':
-        post:
-          summary: Switch a workflow from 'On Hold' to 'Submitted' status
-          description: Request Cromwell to release the hold on a workflow. It will switch the status of a workflow from 'On Hold' to 'Submitted' so it can be picked for running. For instance this might be necessary in cases where you have submitted a workflow with workflowOnHold = true.
-          parameters:
-            - name: version
-              description: API Version
-              required: true
-              type: string
-              in: path
-              default: v1
-            - name: id
-              description: A workflow ID
-              required: true
-              type: string
-              in: path
-          tags:
-            - Workflows
-          responses:
-            '200':
-              description: Successful Request
-              schema:
-                $ref: '#/definitions/WorkflowStatusResponse'
-            '400':
-              description: Malformed Workflow ID
-            '403':
-              description: Malformed Request
-            '404':
-              description: Workflow ID Not Found
-            '500':
-              description: Internal Error
-          security:
-            - google_oauth:
-                - openid
+    post:
+      summary: Switch a workflow from 'On Hold' to 'Submitted' status
+      description: Request Cromwell to release the hold on a workflow. It will switch the status of a workflow from 'On Hold' to 'Submitted' so it can be picked for running. For instance this might be necessary in cases where you have submitted a workflow with workflowOnHold = true.
+      parameters:
+        - name: version
+          description: API Version
+          required: true
+          type: string
+          in: path
+          default: v1
+        - name: id
+          description: A workflow ID
+          required: true
+          type: string
+          in: path
+      tags:
+        - Workflows
+      responses:
+        '200':
+          description: Successful Request
+          schema:
+            $ref: '#/definitions/WorkflowStatusResponse'
+        '400':
+          description: Malformed Workflow ID
+        '403':
+          description: Malformed Request
+        '404':
+          description: Workflow ID Not Found
+        '500':
+          description: Internal Error
+      security:
+        - googleoauth:
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/logs':
     get:
       summary: Query for the standard output and error of all calls in a workflow
@@ -489,9 +491,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/{id}/metadata':
     get:
       summary: Query for workflow and call-level metadata for a specified workflow
@@ -676,9 +678,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/callcaching/diff':
     get:
       summary: Return the hash differential between two calls
@@ -732,9 +734,9 @@ paths:
           description: Internal Error
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/api/workflows/{version}/backends':
     get:
       summary: Returns the backends supported by this Cromwell.
@@ -754,9 +756,9 @@ paths:
             $ref: '#/definitions/BackendResponse'
       security:
         - googleoauth:
-            - openid
-            - email
-            - profile
+          - openid
+          - email
+          - profile
   '/engine/{version}/version':
     get:
       summary: Returns the version of the Cromwell Engine
@@ -774,11 +776,6 @@ paths:
           description: Successful Request
           schema:
             $ref: '#/definitions/VersionResponse'
-      security:
-        - googleoauth:
-            - openid
-            - email
-            - profile
   '/engine/{version}/status':
         get:
           summary: Returns the current health status of any monitored subsystems

backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala

@@ -4,8 +4,7 @@ import akka.actor.ActorLogging
 import akka.event.LoggingReceive
 import cromwell.backend.BackendJobExecutionActor._
 import cromwell.backend.BackendLifecycleActor._
-import cromwell.backend.wdl.OutputEvaluator
-import cromwell.backend.wdl.OutputEvaluator.EvaluatedJobOutputs
+import cromwell.backend.OutputEvaluator.EvaluatedJobOutputs
 import cromwell.core.path.Path
 import cromwell.core._
 import wom.expression.IoFunctionSet

backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala

@@ -3,8 +3,8 @@ package cromwell.backend
 import akka.actor.{ActorRef, Props}
 import com.typesafe.config.Config
 import cromwell.backend.io.WorkflowPathsWithDocker
-import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import cromwell.core.CallOutputs
+import cromwell.core.JobExecutionToken.JobExecutionTokenType
 import cromwell.core.path.Path
 import cromwell.core.path.PathFactory.PathBuilders
 import net.ceedubs.ficus.Ficus._

backend/src/main/scala/cromwell/backend/wdl/Command.scala → backend/src/main/scala/cromwell/backend/Command.scala

@@ -1,8 +1,7 @@
-package cromwell.backend.wdl
+package cromwell.backend
 
 import common.validation.ErrorOr._
 import common.validation.Validation._
-import cromwell.backend.BackendJobDescriptor
 import wom.InstantiatedCommand
 import wom.callable.RuntimeEnvironment
 import wom.expression.IoFunctionSet

backend/src/main/scala/cromwell/backend/wdl/FileSizeTooBig.scala → backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala

@@ -1,4 +1,4 @@
-package cromwell.backend.wdl
+package cromwell.backend
 
 case class FileSizeTooBig(override val getMessage: String) extends Exception
 

backend/src/main/scala/cromwell/backend/wdl/OutputEvaluator.scala → backend/src/main/scala/cromwell/backend/OutputEvaluator.scala

@@ -1,4 +1,4 @@
-package cromwell.backend.wdl
+package cromwell.backend
 
 import cats.data.EitherT._
 import cats.data.Validated.{Invalid, Valid}
@@ -10,7 +10,6 @@ import cats.syntax.validated._
 import common.util.TryUtil
 import common.validation.Checked._
 import common.validation.ErrorOr.ErrorOr
-import cromwell.backend.BackendJobDescriptor
 import cromwell.core.CallOutputs
 import wom.expression.IoFunctionSet
 import wom.graph.GraphNodePort.{ExpressionBasedOutputPort, OutputPort}

backend/src/main/scala/cromwell/backend/wdl/ReadLikeFunctions.scala → backend/src/main/scala/cromwell/backend/ReadLikeFunctions.scala

@@ -1,4 +1,4 @@
-package cromwell.backend.wdl
+package cromwell.backend
 
 import cromwell.core.io.AsyncIoFunctions
 import cromwell.core.path.PathFactory

backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala

@@ -5,6 +5,9 @@ import java.util.UUID
 import cromwell.backend.io.JobPaths
 import cromwell.backend.validation.{CpuValidation, MemoryValidation}
 import cromwell.core.path.Path
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.Positive
+import eu.timepit.refined.refineMV
 import wdl4s.parser.MemoryUnit
 import wom.callable.RuntimeEnvironment
 import wom.format.MemorySize
@@ -23,12 +26,12 @@ object RuntimeEnvironmentBuilder {
         callRoot.resolve(s"tmp.$hash").pathAsString
       }
 
-      val cores: Int = CpuValidation.instanceMin.validate(runtimeAttributes).getOrElse(minimums.cores)
+      val cores: Int Refined Positive = CpuValidation.instanceMin.validate(runtimeAttributes).getOrElse(minimums.cores)
 
       val memoryInMiB: Double =
         MemoryValidation.instance().
           validate(runtimeAttributes).
-          map(_.to(MemoryUnit.MiB).amount).
+          map(_.toMebibytes).
           getOrElse(minimums.ram.amount)
 
       //TODO: Read these from somewhere else
@@ -50,7 +53,7 @@ object RuntimeEnvironmentBuilder {
   }
 }
 
-case class MinimumRuntimeSettings(cores: Int = 1,
+case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1),
                                   ram: MemorySize = MemorySize(4, MemoryUnit.GiB),
                                   outputPathSize: Long = Long.MaxValue,
                                   tempPathSize: Long = Long.MaxValue)