From dcd596a8d2db0f82977149d06fc07bfe6b9a9090 Mon Sep 17 00:00:00 2001
From: coreone <teixeira@broadinstitute.org>
Date: Fri, 8 Jan 2016 15:19:10 -0500
Subject: [PATCH 01/31] Refactor proxy-compose.yaml.ctmpl to use multiple Vault
 keys

---
 src/main/config/cromwell-compose.yaml.ctmpl | 47 +++++++++++++--------
 1 file changed, 29 insertions(+), 18 deletions(-)

diff --git a/src/main/config/cromwell-compose.yaml.ctmpl b/src/main/config/cromwell-compose.yaml.ctmpl
index bf63b053568..eaff76fa0ce 100644
--- a/src/main/config/cromwell-compose.yaml.ctmpl
+++ b/src/main/config/cromwell-compose.yaml.ctmpl
@@ -1,33 +1,44 @@
-{{with $environment := env "ENVIRONMENT"}}
-{{$keyname := printf "secret/dsde/%s/cromwell/cromwell-compose.yaml" $environment}}
-{{with vault $keyname}}
+{{with $environment := env "ENVIRONMENT"}}{{$cromkey := printf "secret/dsde/%s/cromwell/cromwell-compose.yaml" $environment}}{{with vault $cromkey}}
+{{$cromwell_image := .Data.cromwell_image}}
+{{$cromwell_dns := .Data.cromwell_dns}}
+{{$env_log_driver := .Data.env_log_driver}}
+{{$env_java_opts := .Data.env_java_opts}}
+{{$cromwell_volumes := .Data.cromwell_volumes}}
+{{$proxy_hostname := .Data.proxy_hostname}}
+{{$proxy_log_driver := .Data.proxy_log_driver}}
+{{$proxy_volumes := .Data.proxy_volumes}}
+{{$env_callback_uri := .Data.env_callback_uri}}
+{{$env_log_level := .Data.env_log_level}}
+{{$env_server_name := .Data.env_server_name}}
 
+{{$commonkey := printf "secret/dsde/%s/common/proxy-ldap" $environment}}{{with vault $commonkey}}
+{{$proxy_ldap_group := .Data.proxy_ldap_group}}
+{{$proxy_ldap_url := .Data.proxy_ldap_url}}
 app:
-  image: {{.Data.cromwell_image}}
-  {{.Data.cromwell_dns}}
-  log_driver: "{{.Data.env_log_driver}}"
+  image: {{ $cromwell_image }}
+  {{ $cromwell_dns }}
+  log_driver: "{{ $env_log_driver }}"
   environment:
-    JAVA_OPTS: {{.Data.env_java_opts}}
-  {{.Data.cromwell_volumes}}
+    JAVA_OPTS: {{ $env_java_opts }}
+  {{ $cromwell_volumes }}
 proxy:
   image: broadinstitute/openidc-proxy:latest
-  hostname: {{.Data.proxy_hostname}}
-  log_driver: "{{.Data.proxy_log_driver}}"
+  hostname: {{ $proxy_hostname }}
+  log_driver: "{{ $proxy_log_driver }}"
   links:
     - app:app
   ports:
     - "80:80"
     - "443:443"
-  {{.Data.proxy_volumes}}
+  {{ $proxy_volumes }}
   environment:
-    CALLBACK_URI: {{.Data.env_callback_uri}}
-    LOG_LEVEL: {{.Data.env_log_level}}
+    CALLBACK_URI: {{ $env_callback_uri }}
+    LOG_LEVEL: {{ $env_log_level }}
     PROXY_URL: http://app:8000/
     PROXY_URL2: http://app:8000/api
-    SERVER_NAME: {{.Data.env_server_name}}
-    AUTH_REQUIRE2: Require ldap-group {{.Data.proxy_ldap_group}}
-    AUTH_LDAP_URL2: 'AuthLDAPURL "{{.Data.proxy_ldap_url}}"'
+    SERVER_NAME: {{ $env_server_name }}
+    AUTH_REQUIRE2: Require ldap-group {{ $proxy_ldap_group }}
+    AUTH_LDAP_URL2: 'AuthLDAPURL "{{ $proxy_ldap_url }}"'
     AUTH_LDAP_GROUP_ATTR2: 'AuthLDAPGroupAttribute member'
     REMOTE_USER_CLAIM: sub
-{{end}}
-{{end}}
+{{end}}{{end}}{{end}}

From d13975db21efb0b45906d13179df2bcf816d5553 Mon Sep 17 00:00:00 2001
From: coreone <teixeira@broadinstitute.org>
Date: Fri, 8 Jan 2016 15:47:59 -0500
Subject: [PATCH 02/31] Add config to require LDAP auth to query LDAP
 membership in proxy

---
 src/main/config/cromwell-compose.yaml.ctmpl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/config/cromwell-compose.yaml.ctmpl b/src/main/config/cromwell-compose.yaml.ctmpl
index eaff76fa0ce..1d865707d94 100644
--- a/src/main/config/cromwell-compose.yaml.ctmpl
+++ b/src/main/config/cromwell-compose.yaml.ctmpl
@@ -10,10 +10,11 @@
 {{$env_callback_uri := .Data.env_callback_uri}}
 {{$env_log_level := .Data.env_log_level}}
 {{$env_server_name := .Data.env_server_name}}
-
 {{$commonkey := printf "secret/dsde/%s/common/proxy-ldap" $environment}}{{with vault $commonkey}}
 {{$proxy_ldap_group := .Data.proxy_ldap_group}}
 {{$proxy_ldap_url := .Data.proxy_ldap_url}}
+{{$proxy_ldap_bind_dn := .Data.proxy_ldap_bind_dn}}
+{{$proxy_ldap_bind_password := .Data.proxy_ldap_bind_password}}
 app:
   image: {{ $cromwell_image }}
   {{ $cromwell_dns }}
@@ -40,5 +41,7 @@ proxy:
     AUTH_REQUIRE2: Require ldap-group {{ $proxy_ldap_group }}
     AUTH_LDAP_URL2: 'AuthLDAPURL "{{ $proxy_ldap_url }}"'
     AUTH_LDAP_GROUP_ATTR2: 'AuthLDAPGroupAttribute member'
+    AUTH_LDAP_BIND_DN2: 'AuthLDAPBindDN "{{ $proxy_ldap_bind_dn }}"'
+    AUTH_LDAP_BIND_PASSWORD2: 'AuthLDAPBindPassword {{ $proxy_ldap_bind_password }}'
     REMOTE_USER_CLAIM: sub
 {{end}}{{end}}{{end}}

From fe30beef0e81861e186d62abceee45698940b0fc Mon Sep 17 00:00:00 2001
From: geoffjentry <geoffjentry@gmail.com>
Date: Mon, 11 Jan 2016 15:01:37 -0500
Subject: [PATCH 03/31] add gitter badge

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 2f4efb9a71a..4f8231e3b1b 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 [![Build Status](https://travis-ci.org/broadinstitute/cromwell.svg?branch=develop)](https://travis-ci.org/broadinstitute/cromwell?branch=develop)
 [![Coverage Status](https://coveralls.io/repos/broadinstitute/cromwell/badge.svg?branch=develop)](https://coveralls.io/r/broadinstitute/cromwell?branch=develop)
+[![Join the chat at https://gitter.im/broadinstitute/cromwell](https://badges.gitter.im/broadinstitute/cromwell.svg)](https://gitter.im/broadinstitute/cromwell?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
 
 Cromwell
 ========

From 13f59c8fb255fa9513dedf49c1c9bb8c180ccb3d Mon Sep 17 00:00:00 2001
From: Scott Frazer <scott.d.frazer@gmail.com>
Date: Mon, 11 Jan 2016 15:06:45 -0500
Subject: [PATCH 04/31] version bump

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index afc05b9017e..dc609b6061b 100644
--- a/build.sbt
+++ b/build.sbt
@@ -3,7 +3,7 @@ import sbtrelease.ReleasePlugin._
 
 name := "cromwell"
 
-version := "0.16"
+version := "0.17"
 
 organization := "org.broadinstitute"
 

From 72481fb2e3b9dbd33caa2f7758173db9f5a1e4d3 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sat, 19 Dec 2015 10:40:19 -0500
Subject: [PATCH 05/31] `CromwellServer` startup now checks that there are no
 additional args. The server also exits with a 1 if the server does not start
 up, using a synchronous error message. Still could use clean termination
 handling (via lenthall?). Refactored the `WorkflowManagerSystem` passing into
 `Main`, including ensuring that during tests the internal actor system does
 get shut down, but not prematurely. Made the `PromiseActor` more generic.
 Eventually may move to lenthall, and possibly make it even easier to use, as
 right now it requires using `tell` instead of `!` to ensure the `sender` is
 set correctly. `sys.exit()` is only called in the `object Main`, not
 conditionally in the `class Main`. Moved `getAction()` from the `Main` class
 to the object.

---
 src/main/scala/cromwell/Main.scala            | 90 +++++++++----------
 .../cromwell/server/CromwellServer.scala      | 24 ++++-
 .../server/WorkflowManagerSystem.scala        | 13 ++-
 .../scala/cromwell/CromwellTestkitSpec.scala  | 19 ++++
 src/test/scala/cromwell/MainSpec.scala        | 65 +++++++-------
 .../engine/backend/jes/JesBackendSpec.scala   | 18 ++--
 .../engine/db/slick/SlickDataAccessSpec.scala | 10 ++-
 .../cromwell/logging/WorkflowLoggerSpec.scala |  9 +-
 .../CromwellApiServiceIntegrationSpec.scala   |  5 ++
 9 files changed, 160 insertions(+), 93 deletions(-)

diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala
index 59b1d6cb769..3259a759b10 100644
--- a/src/main/scala/cromwell/Main.scala
+++ b/src/main/scala/cromwell/Main.scala
@@ -3,7 +3,7 @@ package cromwell
 import java.io.{File => JFile}
 import java.nio.file.{Files, Path, Paths}
 
-import akka.actor.{Actor, ActorRef, Props, Status}
+import akka.actor.{Actor, Props, Status}
 import better.files._
 import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter}
 import wdl4s.{AstTools, _}
@@ -17,7 +17,7 @@ import org.slf4j.LoggerFactory
 import spray.json._
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, Promise}
+import scala.concurrent.{Future, Await, Promise}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -43,7 +43,7 @@ object Main extends App {
    * Also now passing args to runAction instead of the constructor, as even sbt seemed to have issues with the args
    * array becoming null in "new Main(args)" when used with: sbt 'run run ...'
    */
-  new Main().runAction(args)
+  sys.exit(new Main().runAction(args))
 
   /**
     * If a cromwell server is going to be run, makes adjustments to the default logback configuration.
@@ -55,48 +55,55 @@ object Main extends App {
     * @param args The command line arguments.
     */
   private def setupServerLogging(args: Array[String]): Unit = {
-    args.headOption.map(_.capitalize) match {
-      case Some("Server") => sys.props.getOrElseUpdate("LOG_MODE", "STANDARD")
+    getAction(args) match {
+      case Some(Actions.Server) => sys.props.getOrElseUpdate("LOG_MODE", "STANDARD")
       case _ =>
     }
   }
+
+  private def getAction(args: Seq[String]): Option[Actions.Value] = for {
+    arg <- args.headOption
+    argCapitalized = arg.capitalize
+    action <- Actions.values find (_.toString == argCapitalized)
+  } yield action
 }
 
 /** A simplified version of the Akka `PromiseActorRef` that doesn't time out. */
-private class PromiseWorkflowActor(promise: Promise[Any], runner: ActorRef) extends Actor {
-  runner ! RunWorkflow
-
+private class PromiseActor(promise: Promise[Any]) extends Actor {
   override def receive = {
-    case Status.Failure(f) => promise.tryFailure(f)
-    case success => promise.trySuccess(success)
+    case Status.Failure(f) =>
+      promise.tryFailure(f)
+      context.stop(self)
+    case success =>
+      promise.trySuccess(success)
+      context.stop(self)
   }
 }
 
-
-class Main private[cromwell](enableTermination: Boolean, managerSystem: () => WorkflowManagerSystem) {
+class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
   private[cromwell] val initLoggingReturnCode = initLogging()
 
   lazy val Log = LoggerFactory.getLogger("cromwell")
   Monitor.start()
 
-  def this() = this(enableTermination = true, managerSystem = () => new WorkflowManagerSystem {})
+  def this() = this(managerSystem = new WorkflowManagerSystem {})
 
   // CromwellServer still doesn't clean up... so => Any
-  def runAction(args: Seq[String]): Any = {
-    getAction(args.headOption) match {
+  def runAction(args: Seq[String]): Int = {
+    Main.getAction(args) match {
       case Some(x) if x == Actions.Validate => validate(args.tail)
       case Some(x) if x == Actions.Highlight => highlight(args.tail)
       case Some(x) if x == Actions.Inputs => inputs(args.tail)
       case Some(x) if x == Actions.Run => run(args.tail)
       case Some(x) if x == Actions.Parse => parse(args.tail)
-      case Some(x) if x == Actions.Server => CromwellServer
+      case Some(x) if x == Actions.Server => runServer(args.tail)
       case _ => usageAndExit()
     }
   }
 
   def validate(args: Seq[String]): Int = {
     continueIf(args.length == 1) {
-      loadWdl(args.head) { _ => exit(0) }
+      loadWdl(args.head) { _ => 0 }
     }
   }
 
@@ -105,7 +112,7 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
       loadWdl(args.head) { namespace =>
         val formatter = new SyntaxFormatter(if (args(1) == "html") HtmlSyntaxHighlighter else AnsiSyntaxHighlighter)
         println(formatter.format(namespace))
-        exit(0)
+        0
       }
     }
   }
@@ -118,11 +125,15 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
           case x: NamespaceWithWorkflow => println(x.workflow.inputs.toJson.prettyPrint)
           case _ => println("WDL does not have a local workflow")
         }
-        exit(0)
+        0
       }
     }
   }
 
+  def runServer(args: Seq[String]): Int = {
+    continueIf(args.isEmpty)(waitAndExit(CromwellServer.run(), CromwellServer))
+  }
+
   /* Begin .run() method and utilities */
 
   val WdlLabel = "WDL file"
@@ -147,7 +158,7 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
         case Success(workflowSourceFiles) => runWorkflow(workflowSourceFiles, metadataPath)
         case Failure(ex) =>
           Console.err.println(ex.getMessage)
-          exit(1)
+          1
       }
     }
   }
@@ -176,23 +187,27 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
   }
 
   private[this] def runWorkflow(workflowSourceFiles: WorkflowSourceFiles, metadataPath: Option[Path]): Int = {
-    val workflowManagerSystem = managerSystem()
+    val workflowManagerSystem = managerSystem
     val runnerProps = SingleWorkflowRunnerActor.props(workflowSourceFiles, metadataPath,
       workflowManagerSystem.workflowManagerActor)
     val runner = workflowManagerSystem.actorSystem.actorOf(runnerProps, "SingleWorkflowRunnerActor")
 
     val promise = Promise[Any]()
-    workflowManagerSystem.actorSystem.actorOf(Props(classOf[PromiseWorkflowActor], promise, runner))
-    val futureResult = promise.future
+    val promiseActor = workflowManagerSystem.actorSystem.actorOf(Props(classOf[PromiseActor], promise))
+    runner.tell(RunWorkflow, promiseActor)
+    waitAndExit(promise.future, workflowManagerSystem)
+  }
+
+  private[this] def waitAndExit(futureResult: Future[Any], workflowManagerSystem: WorkflowManagerSystem): Int = {
     Await.ready(futureResult, Duration.Inf)
 
-    if (enableTermination) workflowManagerSystem.actorSystem.shutdown()
+    workflowManagerSystem.shutdownActorSystem()
 
     futureResult.value.get match {
-      case Success(_) => exit(0)
+      case Success(_) => 0
       case Failure(e) =>
         Console.err.println(e.getMessage)
-        exit(1)
+        1
     }
   }
 
@@ -260,7 +275,7 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
   def parse(args: Seq[String]): Int = {
     continueIf(args.length == 1) {
       println(AstTools.getAst(new JFile(args.head)).toPrettyString)
-      exit(0)
+      0
     }
   }
 
@@ -314,7 +329,7 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
         |  will output colorized text to the terminal
         |
       """.stripMargin)
-    exit(-1)
+    -1
   }
 
   private[this] def initLogging(): Int = {
@@ -337,33 +352,18 @@ class Main private[cromwell](enableTermination: Boolean, managerSystem: () => Wo
       case e: Throwable =>
         Console.err.println(s"Could not create log directory: $logRoot")
         e.printStackTrace()
-        exit(1)
+        1
     }
   }
 
   private[this] def continueIf(valid: => Boolean)(block: => Int): Int = if (valid) block else usageAndExit()
 
-  private[this] def getAction(firstArg: Option[String]): Option[Actions.Value] = for {
-    arg <- firstArg
-    argCapitalized = arg.capitalize
-    a <- Actions.values find (_.toString == argCapitalized)
-  } yield a
-
   private[this] def loadWdl(path: String)(f: WdlNamespace => Int): Int = {
     Try(WdlNamespace.load(new JFile(path))) match {
       case Success(namespace) => f(namespace)
       case Failure(t) =>
         println(t.getMessage)
-        exit(1)
-    }
-  }
-
-  private[this] def exit(returnCode: Int): Int = {
-    if (enableTermination) {
-      // $COVERAGE-OFF$Exit not allowed during tests
-      sys.exit(returnCode)
-      // $COVERAGE-ON$
+        1
     }
-    returnCode
   }
 }
diff --git a/src/main/scala/cromwell/server/CromwellServer.scala b/src/main/scala/cromwell/server/CromwellServer.scala
index b0a790db1e7..8a825f6fd6f 100644
--- a/src/main/scala/cromwell/server/CromwellServer.scala
+++ b/src/main/scala/cromwell/server/CromwellServer.scala
@@ -5,7 +5,9 @@ import com.typesafe.config.ConfigFactory
 import cromwell.webservice.CromwellApiServiceActor
 import lenthall.spray.SprayCanHttpService._
 
+import scala.concurrent.Future
 import scala.concurrent.duration._
+import scala.util.{Failure, Success}
 
 // Note that as per the language specification, this is instantiated lazily and only used when necessary (i.e. server mode)
 object CromwellServer extends WorkflowManagerSystem {
@@ -15,7 +17,25 @@ object CromwellServer extends WorkflowManagerSystem {
   val conf = ConfigFactory.load()
   val service = actorSystem.actorOf(CromwellApiServiceActor.props(workflowManagerActor, conf), "cromwell-service")
   val webserviceConf = conf.getConfig("webservice")
-  service.bindOrShutdown(interface = webserviceConf.getString("interface"), port = webserviceConf.getInt("port")) onSuccess {
-    case _ => actorSystem.log.info("Cromwell service started...")
+
+  def run(): Future[Any] = {
+    val interface = webserviceConf.getString("interface")
+    val port = webserviceConf.getInt("port")
+    val futureBind = service.bind(interface = interface, port = port)
+    futureBind andThen {
+      case Success(_) =>
+        actorSystem.log.info("Cromwell service started...")
+        actorSystem.awaitTermination()
+      case Failure(throwable) =>
+        /*
+        TODO:
+        If/when CromwellServer behaves like a better async citizen, we may be less paranoid about our async log messages
+        not appearing due to the actor system shutdown. For now, synchronously print to the stderr so that the user has
+        some idea of why the server failed to start up.
+         */
+        Console.err.println(s"Binding failed interface $interface port $port")
+        throwable.printStackTrace(Console.err)
+        shutdownActorSystem()
+    }
   }
 }
diff --git a/src/main/scala/cromwell/server/WorkflowManagerSystem.scala b/src/main/scala/cromwell/server/WorkflowManagerSystem.scala
index f2f9a32b258..b1e5b1b3f4c 100644
--- a/src/main/scala/cromwell/server/WorkflowManagerSystem.scala
+++ b/src/main/scala/cromwell/server/WorkflowManagerSystem.scala
@@ -2,15 +2,22 @@ package cromwell.server
 
 import akka.actor.ActorSystem
 import com.typesafe.config.ConfigFactory
-import cromwell.engine.backend.{CromwellBackend, Backend}
+import cromwell.engine.backend.{Backend, CromwellBackend}
 import cromwell.engine.workflow.WorkflowManagerActor
-import cromwell.engine.backend.BackendType
 
 trait WorkflowManagerSystem {
   protected def systemName = "cromwell-system"
+
   protected def newActorSystem(): ActorSystem = ActorSystem(systemName)
-  implicit final val actorSystem = newActorSystem()
+
+  implicit final lazy val actorSystem = newActorSystem()
+
+  def shutdownActorSystem(): Unit = {
+    actorSystem.shutdown()
+  }
+
   def backendType: String = ConfigFactory.load.getConfig("backend").getString("backend")
+
   lazy val backend: Backend = CromwellBackend.initBackend(backendType, actorSystem)
   // For now there's only one WorkflowManagerActor so no need to dynamically name it
   lazy val workflowManagerActor = actorSystem.actorOf(WorkflowManagerActor.props(backend), "WorkflowManagerActor")
diff --git a/src/test/scala/cromwell/CromwellTestkitSpec.scala b/src/test/scala/cromwell/CromwellTestkitSpec.scala
index f96a1b16325..35936f910fb 100644
--- a/src/test/scala/cromwell/CromwellTestkitSpec.scala
+++ b/src/test/scala/cromwell/CromwellTestkitSpec.scala
@@ -57,6 +57,25 @@ object CromwellTestkitSpec {
     override protected def newActorSystem() = ActorSystem(systemName, ConfigFactory.parseString(CromwellTestkitSpec.ConfigText))
     override val backendType = "local"
     backend // Force initialization
+    /**
+      * Do NOT shut down the test actor system inside the normal flow.
+      * The actor system will be externally shutdown outside the block.
+      */
+    override def shutdownActorSystem() = {}
+
+    def shutdownTestActorSystem() = super.shutdownActorSystem()
+  }
+
+  /**
+    * Loans a test actor system. NOTE: This should be run OUTSIDE of a wait block, never within one.
+    */
+  def withTestWorkflowManagerSystem[T](block: WorkflowManagerSystem => T): T = {
+    val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem
+    try {
+      block(testWorkflowManagerSystem)
+    } finally {
+      TestKit.shutdownActorSystem(testWorkflowManagerSystem.actorSystem, timeoutDuration)
+    }
   }
 
   /**
diff --git a/src/test/scala/cromwell/MainSpec.scala b/src/test/scala/cromwell/MainSpec.scala
index e8364079e85..921fe0f32df 100644
--- a/src/test/scala/cromwell/MainSpec.scala
+++ b/src/test/scala/cromwell/MainSpec.scala
@@ -4,7 +4,7 @@ import java.io.{ByteArrayOutputStream, OutputStream}
 import java.nio.file.Path
 import java.text.SimpleDateFormat
 import java.util.Date
-import akka.testkit.TestKit
+
 import akka.util.Timeout
 import better.files._
 import cromwell.util.FileUtil._
@@ -14,6 +14,7 @@ import org.apache.commons.io.output.TeeOutputStream
 import org.scalatest.concurrent.TimeLimitedTests
 import org.scalatest.time.Span
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
@@ -371,12 +372,13 @@ object MainSpec {
    * @return The return code of run.
    */
   def traceInfoRun(args: String*)(pattern: String): Int = {
-    val workflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem
-    waitForInfo(pattern)(
-      printBlock("run", args) {
-        new Main(enableTermination = false, () => workflowManagerSystem).run(args)
-      }
-    )(workflowManagerSystem.actorSystem)
+    withTestWorkflowManagerSystem { workflowManagerSystem =>
+      waitForInfo(pattern)(
+        printBlock("run", args) {
+          new Main(workflowManagerSystem).run(args)
+        }
+      )(workflowManagerSystem.actorSystem)
+    }
   }
 
   /**
@@ -388,17 +390,13 @@ object MainSpec {
    * @return The return code of run.
    */
   def traceErrorWithExceptionRun(args: String*)(pattern: String, throwableClass: Class[_ <: Throwable] = classOf[Throwable]): Int = {
-    val workflowManagerSystem = new TestWorkflowManagerSystem
-    val result = waitForErrorWithException(pattern, throwableClass = throwableClass)(
-      printBlock("run", args) {
-        // Explicitly disable shutting down the actor system from within Main, there's a race condition to deliver
-        // log messages to the TestKit filter before the system is torn down.  Wait until we see the message we
-        // want, then shutdown the system ourselves.
-        new Main(enableTermination = false, () => workflowManagerSystem).run(args)
-      }
-    )(workflowManagerSystem.actorSystem)
-    TestKit.shutdownActorSystem(workflowManagerSystem.actorSystem, timeoutDuration)
-    result
+    withTestWorkflowManagerSystem { workflowManagerSystem =>
+      waitForErrorWithException(pattern, throwableClass = throwableClass)(
+        printBlock("run", args) {
+          new Main(workflowManagerSystem).run(args)
+        }
+      )(workflowManagerSystem.actorSystem)
+    }
   }
 
   /**
@@ -409,14 +407,15 @@ object MainSpec {
    * @return The return code of run.
    */
   def traceInfoAction(args: String*)(pattern: String): Int = {
-    val workflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem
-    waitForInfo(pattern)(
-      printBlock("runAction", args) {
-        new Main(enableTermination = false, () => workflowManagerSystem).runAction(args) match {
-          case status: Int => status
+    withTestWorkflowManagerSystem { workflowManagerSystem =>
+      waitForInfo(pattern)(
+        printBlock("runAction", args) {
+          new Main(workflowManagerSystem).runAction(args) match {
+            case status: Int => status
+          }
         }
-      }
-    )(workflowManagerSystem.actorSystem)
+      )(workflowManagerSystem.actorSystem)
+    }
   }
 
   /**
@@ -427,16 +426,18 @@ object MainSpec {
    * @return return code plus Console.out/Console.err during the block.
    */
   def traceMain(block: Main => Int): TraceResult = {
-    val outStream = TeeStream(Console.out)
-    val errStream = TeeStream(Console.err)
-    val status = {
-      Console.withOut(outStream.teed) {
-        Console.withErr(errStream.teed) {
-          block(new Main(enableTermination = false, () => new CromwellTestkitSpec.TestWorkflowManagerSystem))
+    withTestWorkflowManagerSystem { workflowManagerSystem =>
+      val outStream = TeeStream(Console.out)
+      val errStream = TeeStream(Console.err)
+      val status = {
+        Console.withOut(outStream.teed) {
+          Console.withErr(errStream.teed) {
+            block(new Main(workflowManagerSystem))
+          }
         }
       }
+      TraceResult(status, outStream.captured, errStream.captured)
     }
-    TraceResult(status, outStream.captured, errStream.captured)
   }
 
   /**
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index 4b1f6456fcd..8af06064d33 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -15,18 +15,22 @@ import cromwell.engine.backend.jes.authentication._
 import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
 import cromwell.util.EncryptionSpec
-import org.scalatest.{FlatSpec, Matchers}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 import org.specs2.mock.Mockito
 
 import scala.util.Success
 
-object JesBackendSpec {
-  val ActorSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem().actorSystem
-}
+class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndAfterAll {
+  val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()
+  val actorSystem = testWorkflowManagerSystem.actorSystem
+
+  override protected def afterAll() = {
+    testWorkflowManagerSystem.shutdownTestActorSystem()
+    super.afterAll()
+  }
 
-class JesBackendSpec extends FlatSpec with Matchers with Mockito {
   val clientSecrets = SimpleClientSecrets("id", "secrets")
-  val jesBackend = new JesBackend(JesBackendSpec.ActorSystem) {
+  val jesBackend = new JesBackend(actorSystem) {
     private val anyString = ""
     private val anyURL: URL = null
     override lazy val jesConf = new JesAttributes(
@@ -55,7 +59,7 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito {
       gcsFileKey -> gcsFileVal
     )
 
-    val mappedInputs: CallInputs  = new JesBackend(JesBackendSpec.ActorSystem).adjustInputPaths(ignoredCall, emptyRuntimeAttributes, inputs, mock[WorkflowDescriptor])
+    val mappedInputs: CallInputs  = new JesBackend(actorSystem).adjustInputPaths(ignoredCall, emptyRuntimeAttributes, inputs, mock[WorkflowDescriptor])
 
     mappedInputs.get(stringKey).get match {
       case WdlString(v) => assert(v.equalsIgnoreCase(stringVal.value))
diff --git a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
index 7b050223709..c9ded80282c 100644
--- a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
+++ b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
@@ -27,8 +27,9 @@ import org.scalactic.StringNormalizations._
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.time.{Millis, Seconds, Span}
-import org.scalatest.{FlatSpec, Matchers}
 import org.scalatest.PartialFunctionValues._
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+
 import scala.concurrent.{ExecutionContext, Future}
 import Hashing._
 
@@ -37,12 +38,17 @@ object SlickDataAccessSpec {
   val AllowTrue = Seq(webservice.QueryParameter("allow", "true"))
 }
 
-class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures {
+class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with BeforeAndAfterAll {
 
   import TableDrivenPropertyChecks._
 
   val workflowManagerSystem = new TestWorkflowManagerSystem
 
+  override protected def afterAll() = {
+    workflowManagerSystem.shutdownTestActorSystem()
+    super.afterAll()
+  }
+
   implicit val ec = ExecutionContext.global
 
   implicit val defaultPatience = PatienceConfig(timeout = Span(5, Seconds), interval = Span(100, Millis))
diff --git a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
index 6800dd7f875..c7e7311b3ae 100644
--- a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
+++ b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
@@ -7,11 +7,16 @@ import wdl4s.values.WdlValue
 import cromwell.engine.backend.local.{LocalBackend, LocalBackendCall}
 import cromwell.engine.workflow.CallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor, WorkflowId, WorkflowSourceFiles}
-import org.scalatest.{FlatSpec, Matchers}
+import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 
-class WorkflowLoggerSpec extends FlatSpec with Matchers {
+class WorkflowLoggerSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
   val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()
 
+  override protected def afterAll() = {
+    testWorkflowManagerSystem.shutdownTestActorSystem()
+    super.afterAll()
+  }
+
   val descriptor = WorkflowDescriptor(
     WorkflowId(UUID.fromString("fc6cfad9-65e9-4eb7-853f-7e08c1c8cf8e")),
     WorkflowSourceFiles(
diff --git a/src/test/scala/cromwell/webservice/CromwellApiServiceIntegrationSpec.scala b/src/test/scala/cromwell/webservice/CromwellApiServiceIntegrationSpec.scala
index a8e447b2dfa..15101feebdf 100644
--- a/src/test/scala/cromwell/webservice/CromwellApiServiceIntegrationSpec.scala
+++ b/src/test/scala/cromwell/webservice/CromwellApiServiceIntegrationSpec.scala
@@ -17,6 +17,11 @@ class CromwellApiServiceIntegrationSpec extends FlatSpec with CromwellApiService
   override val workflowManager = TestActorRef(new WorkflowManagerActor(new LocalBackend(actorRefFactory)))
   val version = "v1"
 
+  override protected def afterAll() = {
+    testWorkflowManagerSystem.shutdownTestActorSystem()
+    super.afterAll()
+  }
+
   it should "return 400 for a malformed WDL " in {
     Post(s"/workflows/$version", FormData(Seq("wdlSource" -> CromwellApiServiceSpec.MalformedWdl, "workflowInputs" -> HelloWorld.rawInputs.toJson.toString()))) ~>
       submitRoute ~>

From 8543c8cfe5ac1d6e1ae0f79421edaf7aed7e23ab Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Wed, 23 Dec 2015 15:48:44 -0500
Subject: [PATCH 06/31] Copy workflow outputs using final calls.

---
 README.md                                     |  13 +-
 build.sbt                                     |   1 +
 .../scala/cromwell/engine/CallActor.scala     |   3 -
 .../cromwell/engine/CopyWorkflowOutputs.scala |  19 ++
 .../scala/cromwell/engine/FinalCall.scala     |  37 ++
 .../cromwell/engine/WorkflowDescriptor.scala  |  99 +++++-
 .../engine/backend/jes/JesBackend.scala       |   2 +-
 .../backend/jes/JesEngineFunctions.scala      |   8 +-
 .../backend/local/LocalEngineFunctions.scala  |   4 +-
 .../backend/local/SharedFileSystem.scala      |   7 +-
 .../engine/db/slick/SlickDataAccess.scala     |   3 +-
 .../engine/io/gcs/GcsFileAttributes.scala     |  16 +
 .../engine/io/gcs/GcsFileSystem.scala         |  59 ++++
 .../engine/io/gcs/GcsFileSystemProvider.scala | 136 ++++++++
 .../engine/io/gcs/GoogleCloudStorage.scala    |  22 +-
 .../cromwell/engine/io/gcs/NioGcsPath.scala   | 158 +++++++++
 .../shared/SharedFileSystemIoInterface.scala  |   3 +-
 src/main/scala/cromwell/engine/package.scala  |  26 +-
 .../engine/workflow/ExecutionStoreKey.scala   |   6 +-
 .../engine/workflow/WorkflowActor.scala       |  65 +++-
 .../engine/workflow/WorkflowOptions.scala     |   4 +-
 src/main/scala/cromwell/util/PathUtil.scala   |  11 -
 src/main/scala/cromwell/util/TryUtil.scala    |   6 +-
 .../cromwell/CopyWorkflowOutputsSpec.scala    |  47 +++
 .../scala/cromwell/CromwellTestkitSpec.scala  |   3 +-
 .../engine/WorkflowDescriptorSpec.scala       |  32 +-
 .../engine/io/gcs/NioGcsPathSpec.scala        | 318 ++++++++++++++++++
 src/test/scala/cromwell/util/SampleWdl.scala  |  69 ++++
 28 files changed, 1104 insertions(+), 73 deletions(-)
 create mode 100644 src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
 create mode 100644 src/main/scala/cromwell/engine/FinalCall.scala
 create mode 100644 src/main/scala/cromwell/engine/io/gcs/GcsFileAttributes.scala
 create mode 100644 src/main/scala/cromwell/engine/io/gcs/GcsFileSystem.scala
 create mode 100644 src/main/scala/cromwell/engine/io/gcs/GcsFileSystemProvider.scala
 create mode 100644 src/main/scala/cromwell/engine/io/gcs/NioGcsPath.scala
 delete mode 100644 src/main/scala/cromwell/util/PathUtil.scala
 create mode 100644 src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
 create mode 100644 src/test/scala/cromwell/engine/io/gcs/NioGcsPathSpec.scala

diff --git a/README.md b/README.md
index 9ebaff91610..c558c149151 100644
--- a/README.md
+++ b/README.md
@@ -221,16 +221,6 @@ Only a few workflow options are available currently and are all to be used with
 $ java -jar cromwell.jar run my_jes_wf.wdl my_jes_wf.json wf_options.json
 ```
 
-Where `wf_options.json` would contain:
-
-```
-{
-  "jes_gcs_root": "gs://my-bucket/workflows",
-  "google_project": "my_google_project",
-  "refresh_token": "1/Fjf8gfJr5fdfNf9dk26fdn23FDm4x"
-}
-```
-
 The fourth, optional parameter to the 'run' subcommand is a path where the workflow metadata will be written.  By default, no workflow metadata will be written.
 
 ```
@@ -955,7 +945,6 @@ Example workflow options file:
 
 ```json
 {
-  "default_backend": "jes",
   "jes_gcs_root": "gs://my-bucket/workflows",
   "google_project": "my_google_project",
   "refresh_token": "1/Fjf8gfJr5fdfNf9dk26fdn23FDm4x"
@@ -964,9 +953,9 @@ Example workflow options file:
 
 Valid keys and their meanings:
 
-* **default_backend** - Backend to use to run this workflow.  Accepts values `jes`, `local`, or `sge`.
 * **write_to_cache** - Accepts values `true` or `false`.  If `false`, the completed calls from this workflow will not be added to the cache.  See the [Call Caching](#call-caching) section for more details.
 * **read_from_cache** - Accepts values `true` or `false`.  If `false`, Cromwell will not search the cache when invoking a call (i.e. every call will be executed unconditionally).  See the [Call Caching](#call-caching) section for more details.
+* **outputs_path** - Specifies a path where final workflow outputs will be written.  If this is not specified, workflow outputs will not be copied out of the Cromwell workflow execution directory/path.
 * **jes_gcs_root** - (JES backend only) Specifies where outputs of the workflow will be written.  Expects this to be a GCS URL (e.g. `gs://my-bucket/workflows`).  If this is not set, this defaults to the value within `backend.jes.baseExecutionBucket` in the [configuration](#configuring-cromwell).
 * **google_project** - (JES backend only) Specifies which google project to execute this workflow.
 * **refresh_token** - (JES backend only) Only used if `localizeWithRefreshToken` is specified in the [configuration file](#configuring-cromwell).  See the [Data Localization](#data-localization) section below for more details.
diff --git a/build.sbt b/build.sbt
index c1501ebe3ac..584d7717089 100644
--- a/build.sbt
+++ b/build.sbt
@@ -59,6 +59,7 @@ libraryDependencies ++= Seq(
   "com.google.api-client" % "google-api-client-java6" % googleClientApiV,
   "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV,
   "com.google.oauth-client" % "google-oauth-client" % googleClientApiV,
+  "com.google.cloud.bigdataoss" % "gcsio" % "1.4.3",
   "mysql" % "mysql-connector-java" % "5.1.36",
   "org.scalaz" % "scalaz-core_2.11" % "7.1.3",
   "com.github.pathikrit" %% "better-files" % "2.13.0",
diff --git a/src/main/scala/cromwell/engine/CallActor.scala b/src/main/scala/cromwell/engine/CallActor.scala
index 42a6c091ac8..d9ffb6c008c 100644
--- a/src/main/scala/cromwell/engine/CallActor.scala
+++ b/src/main/scala/cromwell/engine/CallActor.scala
@@ -9,7 +9,6 @@ import wdl4s.values.WdlValue
 import cromwell.engine.CallActor.{CallActorData, CallActorState}
 import cromwell.engine.CallExecutionActor.CallExecutionActorMessage
 import cromwell.engine.backend._
-import cromwell.engine.db.slick.Execution
 import cromwell.engine.workflow.{CallKey, WorkflowActor}
 import cromwell.instrumentation.Instrumentation.Monitor
 import cromwell.logging.WorkflowLogger
@@ -18,8 +17,6 @@ import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
-import scala.util.{Try, Success, Failure}
-
 
 object CallActor {
 
diff --git a/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala b/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
new file mode 100644
index 00000000000..10c17040a31
--- /dev/null
+++ b/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
@@ -0,0 +1,19 @@
+package cromwell.engine
+
+
+import wdl4s.Workflow
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object CopyWorkflowOutputs {
+  val Name: LocallyQualifiedName = "$final_call$copy_workflow_outputs"
+}
+
+/**
+  * Final call implementation that copies workflow outputs to a specified destination.
+  */
+case class CopyWorkflowOutputs(workflow: WorkflowDescriptor) extends FinalCall {
+  override def unqualifiedName = CopyWorkflowOutputs.Name
+  override def rootWorkflow: Workflow = workflow.namespace.workflow
+  override def execute(implicit ec: ExecutionContext): Future[Unit] = workflow.copyWorkflowOutputs
+}
diff --git a/src/main/scala/cromwell/engine/FinalCall.scala b/src/main/scala/cromwell/engine/FinalCall.scala
new file mode 100644
index 00000000000..14566e24156
--- /dev/null
+++ b/src/main/scala/cromwell/engine/FinalCall.scala
@@ -0,0 +1,37 @@
+package cromwell.engine
+
+import cromwell.engine.workflow.{ExecutionStoreKey, FinalCallKey}
+import wdl4s.Scope
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object FinalCall {
+  implicit class FinalCallString(val fqn: FullyQualifiedName) extends AnyVal {
+    /** Does this FQN conform to a final call? */
+    def isFinalCall = fqn startsWith "$final_call"
+    /** Convert this FQN to a `FinalCall` or throw if it can't be converted. */
+    def toStoreKey(workflow: WorkflowDescriptor): ExecutionStoreKey = {
+      fqn match {
+        case CopyWorkflowOutputs.Name => FinalCallKey(CopyWorkflowOutputs(workflow))
+        case _ => throw new IllegalArgumentException(s"Unrecognized call key $fqn")
+      }
+    }
+  }
+}
+
+/** Scope representing a "final call" that is inserted after all other workflow executions. */
+trait FinalCall extends Scope {
+  def execute(implicit ec: ExecutionContext): Future[Unit]
+
+  def prerequisiteCallNames: Set[LocallyQualifiedName] = throw new UnsupportedOperationException("prerequisiteCallNames not supported for FinalCalls")
+
+  /**
+    * Note: This implementation makes final calls dependent on all "real" scopes in the workflow.  For a system with
+    * at most one final call this should be fine, but if there are > 1 final calls should they all be made runnable
+    * at the same time (consuming multiple slots in the thread pool with potentially time consuming work) or run
+    * serially?  This implementation starts all FinalCalls at the same time.
+    */
+  def prerequisiteScopes: Set[Scope] = rootWorkflow.children.toSet
+
+  val parent: Option[Scope] = None
+}
diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index 957e59d3e13..d53c21e8992 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -1,26 +1,33 @@
 package cromwell.engine
 
-import java.nio.file.{Path, Paths}
+import java.nio.file.{FileAlreadyExistsException, Files, Path, Paths}
 
 import ch.qos.logback.classic.encoder.PatternLayoutEncoder
 import ch.qos.logback.classic.spi.ILoggingEvent
 import ch.qos.logback.classic.{Level, LoggerContext}
 import ch.qos.logback.core.FileAppender
 import com.typesafe.config.{Config, ConfigFactory}
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s._
-import wdl4s.values.WdlFile
-import cromwell.engine.backend.{BackendType, CromwellBackend, Backend}
 import cromwell.engine.backend.jes.JesBackend
-import cromwell.engine.io.{IoInterface, IoManager}
-import cromwell.engine.io.gcs.GoogleCloudStorage
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
+import cromwell.engine.backend.{Backend, BackendType, CromwellBackend}
+import cromwell.engine.db.DataAccess._
+import cromwell.engine.io.gcs.{GcsFileSystem, GoogleCloudStorage}
 import cromwell.engine.io.shared.SharedFileSystemIoInterface
+import cromwell.engine.io.{IoInterface, IoManager}
 import cromwell.engine.workflow.WorkflowOptions
+import cromwell.logging.WorkflowLogger
+import cromwell.util.TryUtil
 import lenthall.config.ScalaConfig._
 import org.slf4j.helpers.NOPLogger
 import org.slf4j.{Logger, LoggerFactory}
 import spray.json.{JsObject, _}
+import wdl4s._
+import wdl4s.values.{WdlFile, WdlSingleFile}
 
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 import scalaz.Scalaz._
 
@@ -44,10 +51,30 @@ case class WorkflowDescriptor(id: WorkflowId,
   val name = namespace.workflow.unqualifiedName
   val actualInputs: WorkflowCoercedInputs = coercedInputs ++ declarations
   val props = sys.props
+  val relativeWorkflowRootPath = s"$name/$id"
+  private val log = WorkflowLogger("WorkflowDescriptor", this)
+  val workflowOutputsPath = workflowOptions.get("outputs_path") recover { case e: IllegalArgumentException =>
+    log.warn("outputs_path expected to be of type String", e)
+    throw e
+  }
   lazy val fileHasher: FileHasher = { wdlFile: WdlFile => SymbolHash(ioManager.hash(wdlFile.value)) }
 
-  private lazy val optionCacheWriting = workflowOptions.getBoolean("write_to_cache") getOrElse configCallCaching
-  private lazy val optionCacheReading = workflowOptions.getBoolean("read_from_cache") getOrElse configCallCaching
+  // GCS FS with the workflow working directory as root
+  val gcsFilesystem = for {
+    interface <- gcsInterface
+    fs <- Try(GcsFileSystem.instance(interface, wfContext.root))
+  } yield fs
+
+  // GCS FS with the workflow outputs directory as root
+  val gcsOutputsFilesystem = for {
+    root <- workflowOutputsPath
+    interface <- gcsInterface
+    fs <- Try(GcsFileSystem.instance(interface, root))
+  } yield fs
+
+
+  private lazy val optionCacheWriting = workflowOptions getBoolean "write_to_cache" getOrElse configCallCaching
+  private lazy val optionCacheReading = workflowOptions getBoolean "read_from_cache" getOrElse configCallCaching
 
   if (!configCallCaching) {
     if (optionCacheWriting) logWriteDisabled()
@@ -66,6 +93,58 @@ case class WorkflowDescriptor(id: WorkflowId,
     case _ => NOPLogger.NOP_LOGGER
   }
 
+  def copyWorkflowOutputs(implicit executionContext: ExecutionContext): Future[Unit] = {
+    // Try to copy outputs to final destination
+    workflowOutputsPath map copyOutputFiles getOrElse Future.successful(())
+  }
+
+  private def copyOutputFiles(destDirectory: String)(implicit executionContext: ExecutionContext): Future[Unit] = {
+    import PathString._
+    val logger = backend.workflowLogger(this)
+
+    def copyFile(file: WdlFile): Try[Unit] = {
+      val src = file.valueString.toPath(workflowLogger, gcsFilesystem)
+      val wfPath = wfContext.root.toPath(workflowLogger, gcsFilesystem).toAbsolutePath
+      val relativeFilePath = src.subpath(wfPath.getNameCount, src.getNameCount)
+      val dest = destDirectory.toPath(workflowLogger, gcsOutputsFilesystem).resolve(relativeWorkflowRootPath).resolve(relativeFilePath)
+
+      def copy(): Unit = {
+        logger.info(s"Trying to copy output file $src to $dest")
+        Files.createDirectories(dest.getParent)
+        Files.copy(src, dest)
+      }
+
+      TryUtil.retryBlock(
+        fn = (retries: Option[Unit]) => copy(),
+        retryLimit = Option(5),
+        pollingInterval = 5 seconds,
+        pollingBackOffFactor = 1,
+        maxPollingInterval = 10 seconds,
+        logger = logger,
+        failMessage = Option(s"Failed to copy file $src to $dest"),
+        fatalExceptions = Seq(classOf[FileAlreadyExistsException])
+      ) recover {
+        case _: FileAlreadyExistsException => logger.info(s"Tried to copy the same file multiple times. Skipping subsequent copies for $src")
+      }
+    }
+
+    def processOutputs(outputs: Traversable[SymbolStoreEntry]): Unit = {
+      // All outputs should have wdl values at this point, if they don't there's nothing we can do here
+      val copies = TryUtil.sequence(outputs map { o => Try(o.wdlValue.get) } toSeq) match {
+        case Success(wdlValues) => wdlValues flatMap { _ collectAsSeq { case f: WdlSingleFile => f } } map copyFile
+        case Failure(e) => throw new Throwable(s"Unable to resolve the following workflow outputs for workflow $id: ${e.getMessage}")
+      }
+
+      // Throw an exception if one or more of the copies failed.
+      TryUtil.sequence(copies) match {
+        case Success(_) => ()
+        case Failure(e) => throw new Throwable(s"Output copy failed for the following files:\n ${e.getMessage}")
+      }
+    }
+
+    globalDataAccess.getWorkflowOutputs(id) map processOutputs
+  }
+
   private def makeFileLogger(root: Path, name: String, level: Level): Logger = {
     val ctx = LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]
     val encoder = new PatternLayoutEncoder()
@@ -229,4 +308,4 @@ object WorkflowDescriptor {
       value <- config.getBooleanOption(key)
     } yield value) getOrElse default
   }
-}
\ No newline at end of file
+}
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index 919fc40af5a..cd99d5e5744 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -2,7 +2,7 @@ package cromwell.engine.backend.jes
 
 import java.math.BigInteger
 import java.net.SocketTimeoutException
-import java.nio.file.{Path, Paths}
+import java.nio.file.{Files, Path, Paths}
 
 import akka.actor.ActorSystem
 import com.google.api.services.genomics.model.Parameter
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesEngineFunctions.scala b/src/main/scala/cromwell/engine/backend/jes/JesEngineFunctions.scala
index 5fb88f7f5b2..d57db35ed3b 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesEngineFunctions.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesEngineFunctions.scala
@@ -1,12 +1,12 @@
 package cromwell.engine.backend.jes
 
+import cromwell.engine.Hashing._
+import cromwell.engine.PathString._
+import cromwell.engine._
 import cromwell.engine.io.IoInterface
 import wdl4s.values.WdlValue
-import cromwell.engine._
-import cromwell.util.PathUtil._
-import cromwell.engine.Hashing._
-import scala.util.Try
 
+import scala.util.Try
 class JesWorkflowEngineFunctions(interface: IoInterface, context: WorkflowContext) extends WorkflowEngineFunctions(interface, context) {
   override def globPath(glob: String) = s"${context.root}/glob-${glob.md5Sum}/"
 
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalEngineFunctions.scala b/src/main/scala/cromwell/engine/backend/local/LocalEngineFunctions.scala
index da1a1c1f1ff..4fd518166dd 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalEngineFunctions.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalEngineFunctions.scala
@@ -3,7 +3,7 @@ package cromwell.engine.backend.local
 import java.nio.file.Paths
 import wdl4s.values.WdlValue
 import cromwell.engine.io.IoInterface
-import cromwell.engine.{CallContext, CallEngineFunctions, WorkflowContext, WorkflowEngineFunctions}
+import cromwell.engine.{CallContext, CallEngineFunctions, WorkflowContext, WorkflowEngineFunctions, _}
 
 import scala.language.postfixOps
 import scala.util.Try
@@ -11,7 +11,7 @@ import scala.util.Try
 class LocalWorkflowEngineFunctions(interface: IoInterface, context: WorkflowContext) extends WorkflowEngineFunctions(interface, context)
 
 class LocalCallEngineFunctions(interface: IoInterface, context: CallContext) extends LocalWorkflowEngineFunctions(interface ,context) with CallEngineFunctions {
-  import cromwell.util.PathUtil._
+  import PathString._
 
   override def fileContentsToString(path: String) = {
     if (!Paths.get(path).isAbsolute && !path.isUriWithProtocol)
diff --git a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
index 16b04ef5b60..e9c0d5561ed 100644
--- a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
+++ b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
@@ -6,7 +6,7 @@ import java.nio.file.{Files, Path, Paths}
 import better.files.{File => ScalaFile, _}
 import com.typesafe.config.ConfigFactory
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s._
+import wdl4s.{Call, TaskOutput}
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType}
 import wdl4s.values.{WdlValue, _}
 import cromwell.engine.ExecutionIndex.ExecutionIndex
@@ -14,7 +14,6 @@ import cromwell.engine.backend.{CallLogs, LocalFileSystemBackendCall, _}
 import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs.{GcsPath, GoogleCloudStorage}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId, _}
 import cromwell.engine._
 import cromwell.util.TryUtil
 import org.apache.commons.io.FileUtils
@@ -48,7 +47,7 @@ object SharedFileSystem {
   }).+:(localizeFromGcs _)
 
   private def localizeFromGcs(originalPath: String, executionPath: Path, descriptor: WorkflowDescriptor): Try[Unit] = Try {
-    import cromwell.util.PathUtil._
+    import PathString._
     assert(originalPath.isGcsUrl)
     val content = descriptor.gcsInterface.get.downloadObject(GcsPath(originalPath))
     new ScalaFile(executionPath).createIfNotExists().write(content)
@@ -175,7 +174,7 @@ trait SharedFileSystem {
                        runtimeAttributes: CromwellRuntimeAttributes,
                        inputs: CallInputs,
                        workflowDescriptor: WorkflowDescriptor): CallInputs = {
-    import cromwell.util.PathUtil._
+    import PathString._
 
     val strategies = if (runtimeAttributes.docker.isDefined) DockerLocalizers else Localizers
 
diff --git a/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala b/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
index fbd1b071679..ed3420ead5f 100644
--- a/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
+++ b/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
@@ -18,7 +18,7 @@ import cromwell.engine.backend.local.LocalBackend
 import cromwell.engine.backend.sge.SgeBackend
 import cromwell.engine.backend.{Backend, WorkflowQueryResult}
 import cromwell.engine.db._
-import cromwell.engine.workflow.{CallKey, ExecutionStoreKey, OutputKey, ScatterKey}
+import cromwell.engine.workflow._
 import cromwell.engine.{SymbolHash, CallOutput, WorkflowOutputs}
 import cromwell.webservice.{CallCachingParameters, WorkflowQueryParameters, WorkflowQueryResponse}
 import lenthall.config.ScalaConfig._
@@ -187,6 +187,7 @@ class SlickDataAccess(databaseConfig: Config) extends DataAccess {
     val scopeKeys: Traversable[ExecutionStoreKey] = scopes collect {
       case call: Call => CallKey(call, None)
       case scatter: Scatter => ScatterKey(scatter, None)
+      case finalCall: FinalCall => FinalCallKey(finalCall)
     }
 
     val action = for {
diff --git a/src/main/scala/cromwell/engine/io/gcs/GcsFileAttributes.scala b/src/main/scala/cromwell/engine/io/gcs/GcsFileAttributes.scala
new file mode 100644
index 00000000000..fa34aaffd19
--- /dev/null
+++ b/src/main/scala/cromwell/engine/io/gcs/GcsFileAttributes.scala
@@ -0,0 +1,16 @@
+package cromwell.engine.io.gcs
+
+import java.nio.file.Path
+import java.nio.file.attribute.{FileTime, BasicFileAttributes}
+
+class GcsFileAttributes(path: Path) extends BasicFileAttributes {
+  override def fileKey(): AnyRef = throw new NotImplementedError("To be implemented when/if needed")
+  override def isRegularFile: Boolean = throw new NotImplementedError("To be implemented when/if needed")
+  override def isOther: Boolean = throw new NotImplementedError("To be implemented when/if needed")
+  override def lastModifiedTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed")
+  override def size(): Long = throw new NotImplementedError("To be implemented when/if needed")
+  override def isDirectory: Boolean = false
+  override def isSymbolicLink: Boolean = false
+  override def creationTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed")
+  override def lastAccessTime(): FileTime = throw new NotImplementedError("To be implemented when/if needed")
+}
diff --git a/src/main/scala/cromwell/engine/io/gcs/GcsFileSystem.scala b/src/main/scala/cromwell/engine/io/gcs/GcsFileSystem.scala
new file mode 100644
index 00000000000..561c8d8433b
--- /dev/null
+++ b/src/main/scala/cromwell/engine/io/gcs/GcsFileSystem.scala
@@ -0,0 +1,59 @@
+package cromwell.engine.io.gcs
+
+import java.lang.Iterable
+import java.nio.file._
+import java.nio.file.attribute.UserPrincipalLookupService
+import java.nio.file.spi.FileSystemProvider
+import java.util.{Collections, Set => JSet}
+
+import cromwell.engine.PathString
+
+import scala.concurrent.ExecutionContext
+
+object GcsFileSystem {
+  import PathString._
+  def instance(interface: GoogleCloudStorage, root: String)(implicit executionContext: ExecutionContext) = {
+    if (root.isGcsUrl) new GcsFileSystem(GcsFileSystemProvider.instance(interface), root)
+    else throw new IllegalArgumentException(s"$root is not am absolute GCS path")
+  }
+  val Separator = "/"
+  private [io] val Protocol = "gs://"
+  private val GsUriRegex = s"""$Protocol(.*)""".r
+  private val AttributeViews = Collections.singleton("basic")
+}
+
+/**
+  * Implements the java.nio.FileSystem interface for GoogleCloudStorage.
+  *
+  */
+class GcsFileSystem private (gcsFileSystemProvider: GcsFileSystemProvider, gcsRoot: String) extends FileSystem {
+
+  import GcsFileSystem._
+
+  val root = gcsRoot match {
+    case GsUriRegex(chunks) => new NioGcsPath(chunks.split(Separator), true)(this)
+    case _ => throw new InvalidPathException(gcsRoot, s"Root of GCS file system must be an absolute GCS path: $gcsRoot")
+  }
+
+  override def supportedFileAttributeViews(): JSet[String] = AttributeViews
+  override def getSeparator: String = Separator
+  override def getRootDirectories: Iterable[Path] = Collections.singleton(root)
+  override def newWatchService(): WatchService = throw new NotImplementedError("GCS FS does not support Watch Service at this time")
+  override def getFileStores: Iterable[FileStore] = Collections.emptyList()
+  override def isReadOnly: Boolean = false
+  override def provider(): FileSystemProvider = gcsFileSystemProvider
+  override def isOpen: Boolean = true
+  override def close(): Unit = throw new UnsupportedOperationException("GCS FS cannot be closed")
+  override def getPathMatcher(syntaxAndPattern: String): PathMatcher = FileSystems.getDefault.getPathMatcher(syntaxAndPattern)
+  override def getUserPrincipalLookupService: UserPrincipalLookupService = throw new UnsupportedOperationException()
+  override def getPath(first: String, more: String*): Path = {
+    first match {
+      case GsUriRegex(chunks) =>
+        val path = new NioGcsPath(chunks.split(Separator) ++ more.toArray[String], true)(this)
+        if (path.startsWith(root)) path
+        else throw new InvalidPathException(first, s"Path $path has a different root than this filesystem: $root")
+      case empty if empty.isEmpty => new NioGcsPath(Array.empty[String] ++ more.toArray[String], false)(this)
+      case _ => new NioGcsPath(first.split(Separator) ++ more.toArray[String], false)(this)
+    }
+  }
+}
diff --git a/src/main/scala/cromwell/engine/io/gcs/GcsFileSystemProvider.scala b/src/main/scala/cromwell/engine/io/gcs/GcsFileSystemProvider.scala
new file mode 100644
index 00000000000..fa612fd35cb
--- /dev/null
+++ b/src/main/scala/cromwell/engine/io/gcs/GcsFileSystemProvider.scala
@@ -0,0 +1,136 @@
+package cromwell.engine.io.gcs
+
+import java.io.{FileNotFoundException, OutputStream}
+import java.net.URI
+import java.nio.channels.{Channels, SeekableByteChannel}
+import java.nio.file.DirectoryStream.Filter
+import java.nio.file._
+import java.nio.file.attribute.{BasicFileAttributes, FileAttribute, FileAttributeView}
+import java.nio.file.spi.FileSystemProvider
+import java.util
+import java.util.Collections
+import java.util.concurrent.{AbstractExecutorService, TimeUnit}
+
+import com.google.api.services.storage.model.StorageObject
+import com.google.cloud.hadoop.gcsio.{GoogleCloudStorageReadChannel, GoogleCloudStorageWriteChannel, ObjectWriteConditions}
+import com.google.cloud.hadoop.util.{ApiErrorExtractor, AsyncWriteChannelOptions, ClientRequestHelper}
+
+import scala.collection.JavaConverters._
+import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService}
+
+object GcsFileSystemProvider {
+  def instance(gcsInterface: GoogleCloudStorage)(implicit executionContext: ExecutionContext) = new GcsFileSystemProvider(gcsInterface, executionContext)
+}
+
+/**
+  * Converts a Scala ExecutionContext to a Java ExecutorService.
+  * https://groups.google.com/forum/#!topic/scala-user/ZyHrfzD7eX8
+  */
+object ExecutionContextExecutorServiceBridge {
+  def apply(ec: ExecutionContext): ExecutionContextExecutorService = ec match {
+    case null => throw new Throwable("Execution context cannot be null")
+    case eces: ExecutionContextExecutorService => eces
+    case executionContext => new AbstractExecutorService with ExecutionContextExecutorService {
+      override def prepare(): ExecutionContext = executionContext
+      override def isShutdown = false
+      override def isTerminated = false
+      override def shutdown() = ()
+      override def shutdownNow() = Collections.emptyList[Runnable]
+      override def execute(runnable: Runnable): Unit = executionContext execute runnable
+      override def reportFailure(t: Throwable): Unit = executionContext reportFailure t
+      override def awaitTermination(length: Long,unit: TimeUnit): Boolean = false
+    }
+  }
+}
+
+/**
+  * Implements java.nio.FileSystemProvider for GoogleCloudStorage
+  * This implementation is not complete and mostly a proof of concept that it's possible to *copy* around files from/to local/gcs.
+  * Copying is the only functionality that has been successfully tested (same and cross filesystems).
+  * @param googleCloudStorage must be properly set up (credentials) according to the context. Might be absorbed by this class eventually.
+  * @param executionContext executionContext, will be used to perform async writes to GCS after being converted to a Java execution service
+  */
+class GcsFileSystemProvider private (googleCloudStorage: GoogleCloudStorage, executionContext: ExecutionContext) extends FileSystemProvider {
+
+  private val executionService = ExecutionContextExecutorServiceBridge(executionContext)
+  private val errorExtractor = new ApiErrorExtractor()
+
+  private def checkExists(path: Path) = {
+    if (!googleCloudStorage.exists(path.toString)) throw new FileNotFoundException(path.toString)
+  }
+
+  /**
+    * Note: options and attributes are not honored.
+    */
+  override def newByteChannel(path: Path, options: util.Set[_ <: OpenOption], attrs: FileAttribute[_]*): SeekableByteChannel = {
+    path match {
+      case gcsPath: NioGcsPath =>
+        new GoogleCloudStorageReadChannel(googleCloudStorage.client,
+          gcsPath.bucket,
+          gcsPath.objectName,
+          errorExtractor,
+          new ClientRequestHelper[StorageObject]()
+        )
+      case _ => throw new UnsupportedOperationException("Only Gcs paths are supported.")
+    }
+  }
+
+  /**
+    * Overrides the default implementation to provide a writable channel (which newByteChannel doesn't).
+    * NOTE: options are not honored.
+    */
+  override def newOutputStream(path: Path, options: OpenOption*): OutputStream = {
+    def initializeOutputStream(gcsPath: NioGcsPath) = {
+      val channel = new GoogleCloudStorageWriteChannel(
+        executionService,
+        googleCloudStorage.client,
+        new ClientRequestHelper[StorageObject](),
+        gcsPath.bucket,
+        gcsPath.objectName,
+        AsyncWriteChannelOptions.newBuilder().build(),
+        new ObjectWriteConditions(),
+        Map.empty[String, String].asJava,
+        "text/plain")
+      channel.initialize()
+      Channels.newOutputStream(channel)
+    }
+
+    path match {
+      case gcsPath: NioGcsPath => initializeOutputStream(gcsPath)
+      case _ => throw new UnsupportedOperationException("Only Gcs paths are supported.")
+    }
+  }
+
+  override def copy(source: Path, target: Path, options: CopyOption*): Unit = {
+    (source, target) match {
+      case (s: NioGcsPath, d: NioGcsPath) => googleCloudStorage.copy(s, d)
+      case _ => throw new UnsupportedOperationException(s"Can only copy from GCS to GCS: $source or $target is not a GCS path")
+    }
+  }
+
+  override def delete(path: Path): Unit = {
+    path match {
+      case gcs: GcsPath => googleCloudStorage.deleteObject(gcs)
+    }
+  }
+
+  override def readAttributes[A <: BasicFileAttributes](path: Path, `type`: Class[A], options: LinkOption*): A = {
+    checkExists(path)
+    new GcsFileAttributes(path).asInstanceOf[A]
+  }
+
+  override def move(source: Path, target: Path, options: CopyOption*): Unit = throw new NotImplementedError()
+  override def checkAccess(path: Path, modes: AccessMode*): Unit = {checkExists(path)}
+  override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = {}
+  override def getFileSystem(uri: URI): FileSystem = throw new UnsupportedOperationException()
+  override def isHidden(path: Path): Boolean = throw new NotImplementedError()
+  override def newDirectoryStream(dir: Path, filter: Filter[_ >: Path]): DirectoryStream[Path] = throw new NotImplementedError()
+  override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = throw new NotImplementedError()
+  override def getPath(uri: URI): Path = throw new NotImplementedError()
+  override def newFileSystem(uri: URI, env: util.Map[String, _]): FileSystem = throw new NotImplementedError()
+  override def readAttributes(path: Path, attributes: String, options: LinkOption*): util.Map[String, AnyRef] = throw new NotImplementedError()
+  override def isSameFile(path: Path, path2: Path): Boolean = throw new NotImplementedError()
+  override def getFileAttributeView[V <: FileAttributeView](path: Path, `type`: Class[V], options: LinkOption*): V = throw new NotImplementedError()
+  override def getFileStore(path: Path): FileStore = throw new NotImplementedError()
+  override def getScheme: String = GcsFileSystem.Protocol
+}
diff --git a/src/main/scala/cromwell/engine/io/gcs/GoogleCloudStorage.scala b/src/main/scala/cromwell/engine/io/gcs/GoogleCloudStorage.scala
index d487078b40d..0eae69fc131 100644
--- a/src/main/scala/cromwell/engine/io/gcs/GoogleCloudStorage.scala
+++ b/src/main/scala/cromwell/engine/io/gcs/GoogleCloudStorage.scala
@@ -11,12 +11,12 @@ import com.google.api.client.util.DateTime
 import com.google.api.services.storage.Storage
 import com.google.api.services.storage.model.Bucket.Owner
 import com.google.api.services.storage.model.{Bucket, StorageObject}
+import cromwell.engine.PathString
 import cromwell.engine.io.IoInterface
 import cromwell.engine.workflow.WorkflowOptions
 import cromwell.util.TryUtil
 import cromwell.util.google.GoogleCredentialFactory
 
-
 import scala.collection.JavaConverters._
 import scala.concurrent.duration._
 import scala.language.postfixOps
@@ -52,7 +52,7 @@ object GoogleCloudStorage {
 case class GoogleCloudStorage private(client: Storage) extends IoInterface {
 
   import GcsPath._
-  import cromwell.util.PathUtil._
+  import PathString._
 
   def isValidPath(path: String) = path.isGcsUrl
 
@@ -120,6 +120,18 @@ case class GoogleCloudStorage private(client: Storage) extends IoInterface {
     client.objects.copy(from.bucket, from.objectName, to.bucket, to.objectName, storageObject).execute
   }
 
+  /**
+    * Copy file from one GCS path to another
+    *
+    * @param from - source GCS path (must exist and point to a file)
+    * @param to - destination GCS path
+    * @return a Try[StorageObject] which is a result of the call to Storage.Objects.copy()
+    */
+  def copy(from: NioGcsPath, to: NioGcsPath): Try[StorageObject] = Try {
+    val storageObject = client.objects.get(from.bucket, from.objectName).execute
+    client.objects.copy(from.bucket, from.objectName, to.bucket, to.objectName, storageObject).execute
+  }
+
   /**
     * Copy files with prefix `from` to files with prefix `to`
     *
@@ -187,9 +199,9 @@ case class GoogleCloudStorage private(client: Storage) extends IoInterface {
     insertObject.execute()
   }
 
-  def deleteObject(gcsPath: GcsPath): Unit = {
-    client.objects.delete(gcsPath.bucket, gcsPath.objectName).execute()
-  }
+  def deleteObject(gcsPath: GcsPath): Unit = client.objects.delete(gcsPath.bucket, gcsPath.objectName).execute()
+
+  def deleteObject(gcsPath: NioGcsPath): Unit = client.objects.delete(gcsPath.bucket, gcsPath.objectName).execute()
 
   def downloadObject(gcsPath: GcsPath): Array[Byte] = {
     val outputStream: ByteArrayOutputStream = new ByteArrayOutputStream()
diff --git a/src/main/scala/cromwell/engine/io/gcs/NioGcsPath.scala b/src/main/scala/cromwell/engine/io/gcs/NioGcsPath.scala
new file mode 100644
index 00000000000..7dd698b6814
--- /dev/null
+++ b/src/main/scala/cromwell/engine/io/gcs/NioGcsPath.scala
@@ -0,0 +1,158 @@
+package cromwell.engine.io.gcs
+
+import java.io.File
+import java.net.URI
+import java.nio.file.WatchEvent.{Kind, Modifier}
+import java.nio.file._
+import java.util
+
+import scala.collection.JavaConverters._
+import scala.language.implicitConversions
+import scala.language.postfixOps
+import scala.util.Try
+
+object NioGcsPath {
+  def apply(path: String)(implicit gcsFileSystem: GcsFileSystem)= gcsFileSystem.getPath(path)
+
+  implicit class PathEnhanced(val path: Path) extends AnyVal {
+    def asGcsPath(implicit gcsFileSystem: GcsFileSystem) = path match {
+      case gcsPath: NioGcsPath => gcsPath
+      case otherPath: Path => gcsFileSystem.getPath(otherPath.toString).asInstanceOf[NioGcsPath]
+      case _ => throw new IllegalArgumentException("Only GcsPaths are supported.")
+    }
+  }
+
+  val Protocol = GcsFileSystem.Protocol
+}
+
+/**
+  * NOTE: Currently called NioGcsPath so it can exist alongside the current GcsPath class.
+  *       If this approach was to be validated the current GcsPath class would be replaced by this one.
+  * This class proposes an implementation of the java.nio.Path interface for GoogleCloudStorage.
+  * The following methods are yet to be implemented:
+  *   relativize
+  *   compareTo
+  * @param chunks array containing all parts of the path in between separators - except the protocol (gs://)
+  *               eg: gs://path/to/resource.txt -> chunks = [path, to, resource.txt]
+  * @param absolute true if this path is to be considered absolute.
+  *                 Only absolute GCS paths can be used to actually locate resources.
+  *                 Calling methods on an absolute path can return a relative paths (eg subpath).
+  * @param gcsFileSystem the gcsFileSystem to be used when performing operations on this path
+  */
+class NioGcsPath(private val chunks: Array[String], absolute: Boolean)(implicit gcsFileSystem: GcsFileSystem) extends Path {
+  import NioGcsPath._
+
+  private val separator = GcsFileSystem.Separator
+
+  private val objectChunks = chunks match {
+    case values if isAbsolute && values.nonEmpty => values.tail
+    case _ => chunks
+  }
+
+  private val fullPath = chunksToString(chunks)
+
+  lazy val bucket: String = chunks match {
+    case values if values.isEmpty && isAbsolute => throw new IllegalStateException("An absolute gcs path cannot be empty")
+    case _ => if(isAbsolute) chunks.head else gcsFileSystem.root.asGcsPath.bucket
+  }
+
+  val objectName = chunksToString(objectChunks)
+
+  private def chunksToString(chunksArray: Array[String]): String = chunksArray.mkString(separator)
+
+  override def subpath(beginIndex: Int, endIndex: Int): Path = {
+    new NioGcsPath(chunks.slice(beginIndex, endIndex), isAbsolute && beginIndex == 0)
+  }
+
+  override def toFile: File = throw new UnsupportedOperationException("A GCS path cannot be converted to a File.")
+
+  override def resolveSibling(other: Path): Path = new NioGcsPath(getParent.asGcsPath.chunks ++ other.asGcsPath.chunks, isAbsolute)
+
+  override def resolveSibling(other: String): Path = new NioGcsPath(getParent.asGcsPath.chunks ++ gcsFileSystem.getPath(other).asGcsPath.chunks, isAbsolute)
+
+  override def getFileSystem: FileSystem = gcsFileSystem
+
+  override def getName(index: Int): Path = new NioGcsPath(Array(chunks(index)), isAbsolute && index == 0)
+
+  override def getParent: Path = chunks match {
+    case values if values.isEmpty || values.length == 1 => null
+    case values => new NioGcsPath(values.init, isAbsolute)
+  }
+
+  override def toAbsolutePath: Path = if (isAbsolute) this else gcsFileSystem.root.resolve(this)
+
+  override def relativize(other: Path): Path = throw new NotImplementedError()
+
+  override def getNameCount: Int = chunks.length
+
+  override def toUri: URI = throw new UnsupportedOperationException()
+
+  override def compareTo(other: Path): Int = throw new NotImplementedError()
+
+  override def register(watcher: WatchService, events: Array[Kind[_]], modifiers: Modifier*): WatchKey = throw new UnsupportedOperationException()
+
+  override def register(watcher: WatchService, events: Kind[_]*): WatchKey = throw new UnsupportedOperationException()
+
+  override def getFileName: Path = chunks match {
+    case values if values.isEmpty => null
+    case _ => new NioGcsPath(Array(chunks.last), isAbsolute && chunks.length == 1)
+  }
+
+  override def getRoot: Path = new NioGcsPath(Array(bucket), true)
+
+  override def iterator(): util.Iterator[Path] = (chunks map { elt => new NioGcsPath(Array(elt), false).asInstanceOf[Path] } iterator).asJava
+
+  override def normalize(): Path = if (isAbsolute) this else throw new UnsupportedOperationException("Cannot normalize a relative GCS path.")
+
+  override def endsWith(other: Path): Boolean = {
+    other match {
+      case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false
+      case _: NioGcsPath => chunks.endsWith(other.asGcsPath.chunks)
+      case _ => false
+    }
+  }
+
+  override def endsWith(other: String): Boolean = {
+    Try(gcsFileSystem.getPath(other)) map {
+      case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false
+      case path@(_: NioGcsPath) => chunks.endsWith(path.asGcsPath.chunks)
+      case _ => false
+    } getOrElse false
+  }
+
+  override def resolve(other: Path): Path = {
+    if (other.isAbsolute) other
+    else new NioGcsPath(chunks ++ other.asGcsPath.chunks, isAbsolute)
+  }
+
+  override def resolve(other: String): Path = {
+    val otherPath = gcsFileSystem.getPath(other)
+    if (otherPath.isAbsolute) otherPath
+    else new NioGcsPath(chunks ++ otherPath.asGcsPath.chunks, isAbsolute)
+  }
+
+  override def toRealPath(options: LinkOption*): Path = this
+
+  override def startsWith(other: Path): Boolean = {
+    other match {
+      case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false
+      case _: NioGcsPath => chunks.startsWith(other.asGcsPath.chunks)
+      case _ => false
+    }
+  }
+
+  override def startsWith(other: String): Boolean = {
+    Try(gcsFileSystem.getPath(other)) map {
+      case rel: NioGcsPath if !isAbsolute && rel.isAbsolute => false
+      case path@(_: NioGcsPath) => chunks.startsWith(path.asGcsPath.chunks)
+      case _ => false
+    } getOrElse false
+  }
+
+  override def toString: String = {
+    if (absolute) s"$Protocol$fullPath"
+    else fullPath
+  }
+
+  override def isAbsolute: Boolean = absolute
+}
diff --git a/src/main/scala/cromwell/engine/io/shared/SharedFileSystemIoInterface.scala b/src/main/scala/cromwell/engine/io/shared/SharedFileSystemIoInterface.scala
index a571382f734..d0b142378df 100644
--- a/src/main/scala/cromwell/engine/io/shared/SharedFileSystemIoInterface.scala
+++ b/src/main/scala/cromwell/engine/io/shared/SharedFileSystemIoInterface.scala
@@ -2,6 +2,7 @@ package cromwell.engine.io.shared
 
 import java.nio.file.{Files, Paths}
 
+import cromwell.engine.PathString
 import cromwell.engine.io.IoInterface
 
 import scala.language.postfixOps
@@ -13,8 +14,8 @@ object SharedFileSystemIoInterface {
 
 class SharedFileSystemIoInterface private() extends IoInterface {
 
+  import PathString._
   import better.files._
-  import cromwell.util.PathUtil._
 
   override def readFile(path: String): String = Paths.get(path).contentAsString
 
diff --git a/src/main/scala/cromwell/engine/package.scala b/src/main/scala/cromwell/engine/package.scala
index 8a6a85e560f..3e3e41d42ae 100644
--- a/src/main/scala/cromwell/engine/package.scala
+++ b/src/main/scala/cromwell/engine/package.scala
@@ -1,10 +1,15 @@
 package cromwell
 
-import wdl4s._
+import java.nio.file.{Path, Paths}
+
+import cromwell.engine.io.gcs.GcsFileSystem
 import org.joda.time.DateTime
-import wdl4s.values.{WdlValue, WdlFile}
+import org.slf4j.Logger
+import wdl4s._
+import wdl4s.values.{WdlFile, WdlValue}
 
 import scala.language.implicitConversions
+import scala.util.{Failure, Try}
 import scalaz.ValidationNel
 
 package object engine {
@@ -52,4 +57,21 @@ package object engine {
       case (k, CallOutput(wdlValue, hash)) => (k, wdlValue)
     }
   }
+
+  object PathString {
+    implicit class UriString(val str: String) extends AnyVal {
+      def isGcsUrl: Boolean = str.startsWith("gs://")
+
+      def isUriWithProtocol: Boolean = "^[a-z]+://".r.findFirstIn(str).nonEmpty
+
+      def toPath(workflowLogger: Logger, gcsFileSystem: Try[GcsFileSystem] = Failure(new Throwable("No GCS Filesystem"))): Path = {
+        str match {
+          case path if path.isGcsUrl && gcsFileSystem.isSuccess => gcsFileSystem.get.getPath(str)
+          case path if path.isGcsUrl => throw new Throwable(s"Unable to parse GCS path $path: ${gcsFileSystem.failed.get.getMessage}")
+          case path if !path.isUriWithProtocol => Paths.get(path)
+          case path => throw new Throwable(s"Unable to parse $path")
+        }
+      }
+    }
+  }
 }
diff --git a/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala b/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
index fe025b10caa..9363d170717 100644
--- a/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
+++ b/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
@@ -1,7 +1,7 @@
 package cromwell.engine.workflow
 
 import wdl4s._
-import cromwell.engine.ExecutionStatus
+import cromwell.engine.{FinalCall, ExecutionStatus}
 import cromwell.engine.workflow.WorkflowActor.ExecutionStore
 
 import scala.language.postfixOps
@@ -46,3 +46,7 @@ case class ScatterKey(scope: Scatter, index: Option[Int]) extends ExecutionStore
     }
   }
 }
+
+case class FinalCallKey(scope: FinalCall) extends OutputKey {
+  override def index: Option[Int] = None
+}
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index cefeca919f7..3e9205c1582 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -223,8 +223,10 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   def createWorkflow(inputs: HostInputs): Future[Unit] = {
     val symbolStoreEntries = buildSymbolStoreEntries(workflow, inputs)
     symbolCache = symbolStoreEntries.groupBy(entry => SymbolCacheKey(entry.scope, entry.isInput))
+    // Currently assumes there is at most one possible final call, a `CopyWorkflowOutputs`.
+    val finalCall = workflow.workflowOutputsPath.toOption map { _ => CopyWorkflowOutputs(workflow) }
     globalDataAccess.createWorkflow(
-      workflow, symbolStoreEntries, workflow.namespace.workflow.children, backend)
+      workflow, symbolStoreEntries, workflow.namespace.workflow.children ++ finalCall, backend)
   }
 
   // This is passed as an implicit parameter to methods of classes in the companion object.
@@ -408,11 +410,31 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     }
   }
 
+  private def handleFinalCallStarted(finalCallKey: FinalCallKey): State = {
+    executionStore += finalCallKey -> ExecutionStatus.Running
+    val finalCallWork = for {
+      _ <- persistStatus(finalCallKey, ExecutionStatus.Running)
+      _ <- finalCallKey.scope.execute
+      _ = self ! CallCompleted(finalCallKey, callOutputs = Map.empty, executionEvents = Seq.empty, returnCode = 0, hash = None, resultsClonedFrom = None)
+    } yield ()
+
+    finalCallWork onFailure {
+      case t =>
+        log.error("Final call work failed", t)
+        scheduleTransition(WorkflowFailed)
+    }
+
+    val updatedData = stateData.addPersisting(finalCallKey, ExecutionStatus.Running)
+    stay() using updatedData
+  }
+
   when(WorkflowRunning) {
     case Event(StartRunnableCalls, data) =>
       val updatedData = startRunnableCalls(data)
       if (isWorkflowDone) scheduleTransition(WorkflowSucceeded)
       stay() using updatedData
+    case Event(CallStarted(finalCallKey: FinalCallKey), _) =>
+      handleFinalCallStarted(finalCallKey)
     case Event(CallStarted(callKey), data) if !data.isPending(callKey) =>
       executionStore += callKey -> ExecutionStatus.Running
       persistStatus(callKey, ExecutionStatus.Running)
@@ -421,6 +443,12 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     case Event(message: CallStarted, _) =>
       resendDueToPendingExecutionWrites(message)
       stay()
+    case Event(message @ CallCompleted(finalCallKey: FinalCallKey, _, _, _, _, _), data) if !data.isPersistedRunning(finalCallKey) =>
+      // Unlike "real" calls which are managed by a CallActor, the final call completion message is triggered
+      // internally and there isn't an automatic retry if the message isn't processed.  So if the message can't be
+      // processed due to pending writes, schedule it to be resent here.
+      resendDueToPendingExecutionWrites(message)
+      stay()
     case Event(message @ CallCompleted(callKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if data.isPersistedRunning(callKey) =>
       handleCallCompleted(callKey, outputs, executionEvents, returnCode, message, hash, resultsClonedFrom, data)
     case Event(message @ CallCompleted(collectorKey: CollectorKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if !data.isPending(collectorKey) =>
@@ -489,6 +517,12 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   }
 
   when(WorkflowAborting) {
+    case Event(message @ CallCompleted(finalCallKey: FinalCallKey, _, _, _, _, _), data) if !data.isPersistedRunning(finalCallKey) =>
+      // Unlike "real" calls which are managed by a CallActor, the final call completion message is triggered
+      // internally and there isn't an automatic retry if the message isn't processed.  So if the message can't be
+      // processed due to pending writes, schedule it to be resent here.
+      resendDueToPendingExecutionWrites(message)
+      stay()
     case Event(message @ CallCompleted(callKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if data.isPersistedRunning(callKey) =>
       handleCallCompleted(callKey, outputs, executionEvents, returnCode, message, hash, resultsClonedFrom, data)
     case Event(message @ CallCompleted(collectorKey: CollectorKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if !data.isPending(collectorKey) =>
@@ -688,9 +722,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       status == ExecutionStatus.NotStarted && arePrerequisitesDone(key)
     }
 
-    def findRunnableEntries: Traversable[ExecutionStoreEntry] = executionStore filter isRunnable
-
-    val runnableEntries = findRunnableEntries
+    val runnableEntries = executionStore filter isRunnable
 
     val runnableCalls = runnableEntries collect { case(k: CallKey, v) => k.scope }
     if (runnableCalls.nonEmpty)
@@ -700,6 +732,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       case (k: ScatterKey, _) => processRunnableScatter(k)
       case (k: CollectorKey, _) => processRunnableCollector(k)
       case (k: CallKey, _) => processRunnableCall(k)
+      case (k: FinalCallKey, _) => processRunnableFinalCall(k)
       case (k, v) =>
         val message = s"Unknown entry in execution store:\nKEY: $k\nVALUE:$v"
         logger.error(message)
@@ -880,15 +913,21 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   private def createCaches: Future[(ExecutionStore, SymbolCache)] = {
 
     def isInScatterBlock(c: Call) = c.ancestry.exists(_.isInstanceOf[Scatter])
+    import FinalCall._
 
     val futureExecutionCache = globalDataAccess.getExecutionStatuses(workflow.id) map { statuses =>
       statuses map { case (k, v) =>
-        val key: ExecutionStoreKey = (workflow.namespace.resolve(k.fqn), k.index) match {
-          case (Some(c: Call), Some(i)) => CallKey(c, Option(i))
-          case (Some(c: Call), None) if isInScatterBlock(c) => CollectorKey(c)
-          case (Some(c: Call), None) => CallKey(c, None)
-          case (Some(s: Scatter), None) => ScatterKey(s, None)
-          case _ => throw new UnsupportedOperationException(s"Execution entry invalid: $k -> $v")
+        val key: ExecutionStoreKey = if (k.fqn.isFinalCall) {
+          // Final calls are not part of the workflow namespace, handle these differently from other keys.
+          k.fqn.toStoreKey(workflow)
+        } else {
+          (workflow.namespace.resolve(k.fqn), k.index) match {
+            case (Some(c: Call), Some(i)) => CallKey(c, Option(i))
+            case (Some(c: Call), None) if isInScatterBlock(c) => CollectorKey(c)
+            case (Some(c: Call), None) => CallKey(c, None)
+            case (Some(s: Scatter), None) => ScatterKey(s, None)
+            case _ => throw new UnsupportedOperationException(s"Execution entry invalid: $k -> $v")
+          }
         }
         key -> v.executionStatus
       }
@@ -988,6 +1027,12 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     Success(ExecutionStartResult(Set(StartEntry(collector, ExecutionStatus.Starting))))
   }
 
+  private def processRunnableFinalCall(finalCallKey: FinalCallKey): Try[ExecutionStartResult] = {
+    executionStore += finalCallKey -> ExecutionStatus.Starting
+    persistStatus(finalCallKey, ExecutionStatus.Starting) map { _ => self ! CallStarted(finalCallKey) }
+    Success(ExecutionStartResult(Set(StartEntry(finalCallKey, ExecutionStatus.Starting))))
+  }
+
   private def sendStartMessage(callKey: CallKey, callInputs: Map[String, WdlValue]) = {
     def registerAbortFunction(abortFunction: AbortFunction): Unit = {}
     val backendCall = backend.bindCall(workflow, callKey, callInputs, AbortRegistrationFunction(registerAbortFunction))
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowOptions.scala b/src/main/scala/cromwell/engine/workflow/WorkflowOptions.scala
index 5f9075f3b83..88358efc907 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowOptions.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowOptions.scala
@@ -100,13 +100,13 @@ case class WorkflowOptions(jsObject: JsObject) {
   def get(key: String): Try[String] = jsObject.fields.get(key) match {
     case Some(jsStr: JsString) => Success(jsStr.value)
     case Some(jsObj: JsObject) if isEncryptedField(jsObj) => decryptField(jsObj)
-    case Some(jsVal: JsValue) => Failure(new Throwable(s"Unsupported value as JsValue: $jsVal"))
+    case Some(jsVal: JsValue) => Failure(new IllegalArgumentException(s"Unsupported value as JsValue: $jsVal"))
     case None => Failure(new Throwable(s"Field not found: $key"))
   }
 
   def getBoolean(key: String): Try[Boolean] = jsObject.fields.get(key) match {
     case Some(jsBool: JsBoolean) => Success(jsBool.value)
-    case Some(jsVal: JsValue) => Failure(new Throwable(s"Unsupported JsValue as JsBoolean: $jsVal"))
+    case Some(jsVal: JsValue) => Failure(new IllegalArgumentException(s"Unsupported JsValue as JsBoolean: $jsVal"))
     case None => Failure(new Throwable(s"Field not found: $key"))
   }
 
diff --git a/src/main/scala/cromwell/util/PathUtil.scala b/src/main/scala/cromwell/util/PathUtil.scala
deleted file mode 100644
index c9e04fa9a0a..00000000000
--- a/src/main/scala/cromwell/util/PathUtil.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package cromwell.util
-
-
-object PathUtil {
-
-  implicit class UriString(val str: String) extends AnyVal {
-    def isGcsUrl: Boolean = str.startsWith("gs://")
-    def isUriWithProtocol: Boolean = "^[a-z]+://".r.findFirstIn(str).nonEmpty
-  }
-
-}
diff --git a/src/main/scala/cromwell/util/TryUtil.scala b/src/main/scala/cromwell/util/TryUtil.scala
index 97f72f40d65..554be3281c6 100644
--- a/src/main/scala/cromwell/util/TryUtil.scala
+++ b/src/main/scala/cromwell/util/TryUtil.scala
@@ -4,7 +4,7 @@ import java.io.{PrintWriter, StringWriter}
 
 import cromwell.logging.WorkflowLogger
 
-import scala.concurrent.duration.Duration
+import scala.concurrent.duration.{Duration, _}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -53,7 +53,8 @@ object TryUtil {
                     maxPollingInterval: Duration,
                     logger: WorkflowLogger,
                     failMessage: Option[String] = None,
-                    priorValue: Option[T] = None): Try[T] = {
+                    priorValue: Option[T] = None,
+                    fatalExceptions: Seq[Class[_ <: Throwable]] = Seq.empty): Try[T] = {
 
     def logFailures(attempt: Try[T]): Unit = {
       attempt recover {
@@ -63,6 +64,7 @@ object TryUtil {
 
     Try { fn(priorValue) } match {
       case Success(x) if isSuccess(x) => Success(x)
+      case Failure(f) if fatalExceptions.contains(f.getClass) => Failure(f)
       case value if (retryLimit.isDefined && retryLimit.get > 1) || retryLimit.isEmpty =>
         logFailures(value)
         val retryCountMessage = if (retryLimit.getOrElse(0) > 0) s" (${retryLimit.getOrElse(0) - 1} more retries) " else ""
diff --git a/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
new file mode 100644
index 00000000000..a754baf213f
--- /dev/null
+++ b/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
@@ -0,0 +1,47 @@
+package cromwell
+
+import java.nio.file.{Files, Paths}
+
+import akka.testkit.EventFilter
+import cromwell.util.SampleWdl
+import org.scalatest.prop.TableDrivenPropertyChecks._
+import org.scalatest.prop.Tables.Table
+
+import scala.language.postfixOps
+
+
+class CopyWorkflowOutputsSpec extends CromwellTestkitSpec {
+
+  "CopyWorkflowOutputs" should {
+    "copy workflow outputs" in {
+      val workflowOutputsPath = "copy-workflow-outputs"
+
+      val tmpDir = Files.createTempDirectory(workflowOutputsPath).toAbsolutePath
+
+      val outputs = Table(
+        ("call", "file"),
+        ("call-A", "out"),
+        ("call-A", "out2"),
+        ("call-B", "out"),
+        ("call-B", "out2")
+      )
+
+      val workflowId = runWdlAndAssertOutputs(
+        sampleWdl = SampleWdl.WorkflowOutputsWithFiles,
+        eventFilter = EventFilter.info(pattern = "transitioning from Running to Succeeded.", occurrences = 1),
+        runtime = "",
+        workflowOptions = s""" { "outputs_path": "$tmpDir" } """,
+        expectedOutputs = Seq("A.out", "A.out2", "B.outs") map { o => ("wfoutputs." + o) ->  CromwellTestkitSpec.AnyValueIsFine } toMap,
+        allowOtherOutputs = false
+      )
+
+      forAll(outputs) { (call, file) =>
+        val path = tmpDir.resolve(Paths.get("wfoutputs", workflowId.id.toString, call, file))
+        Files.exists(path) shouldBe true
+      }
+      val path = tmpDir.resolve(Paths.get("wfoutputs", workflowId.id.toString, "call-C", "out"))
+      Files.exists(path) shouldBe false
+    }
+  }
+
+}
diff --git a/src/test/scala/cromwell/CromwellTestkitSpec.scala b/src/test/scala/cromwell/CromwellTestkitSpec.scala
index f96a1b16325..2ad4d8cc84d 100644
--- a/src/test/scala/cromwell/CromwellTestkitSpec.scala
+++ b/src/test/scala/cromwell/CromwellTestkitSpec.scala
@@ -215,7 +215,7 @@ with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with Bef
                              runtime: String = "",
                              workflowOptions: String = "{}",
                              allowOtherOutputs: Boolean = true,
-                             terminalState: WorkflowState = WorkflowSucceeded): Unit = {
+                             terminalState: WorkflowState = WorkflowSucceeded): WorkflowId = {
     val wma = buildWorkflowManagerActor(sampleWdl, runtime)
     val wfSources = sampleWdl.asWorkflowSources(runtime, workflowOptions)
     val submitMessage = WorkflowManagerActor.SubmitWorkflow(wfSources)
@@ -238,6 +238,7 @@ with DefaultTimeout with ImplicitSender with WordSpecLike with Matchers with Bef
             if (expectedValue != AnyValueIsFine) validateOutput(actualValue.wdlValue, expectedValue)
           }
         }
+        workflowId
       }
     }
   }
diff --git a/src/test/scala/cromwell/engine/WorkflowDescriptorSpec.scala b/src/test/scala/cromwell/engine/WorkflowDescriptorSpec.scala
index e4656a01ae9..87ddff19186 100644
--- a/src/test/scala/cromwell/engine/WorkflowDescriptorSpec.scala
+++ b/src/test/scala/cromwell/engine/WorkflowDescriptorSpec.scala
@@ -1,11 +1,15 @@
 package cromwell.engine
 
-import com.typesafe.config.{Config, ConfigFactory}
+import java.nio.file.{Files, Paths}
+
+import com.typesafe.config.ConfigFactory
+import cromwell.util.SampleWdl
 import org.scalatest.prop.TableDrivenPropertyChecks._
 import org.scalatest.prop.Tables.Table
 import org.scalatest.{FlatSpec, Matchers}
 
 import scala.collection.JavaConverters._
+import scala.util.{Failure, Success}
 
 class WorkflowDescriptorSpec extends FlatSpec with Matchers {
 
@@ -69,4 +73,30 @@ class WorkflowDescriptorSpec extends FlatSpec with Matchers {
 
   }
 
+  it should "copy workflow outputs to their final (local) destination" in {
+    import scala.concurrent.ExecutionContext.Implicits.global
+
+    val tmpDir = Files.createTempDirectory("wf-outputs").toAbsolutePath
+    val sources = WorkflowSourceFiles(SampleWdl.WorkflowOutputsWithFiles.wdlSource(), "{}", s"""{ "outputs_path": "$tmpDir" }""")
+
+    val outputs = Table(
+      ("call", "file"),
+      ("call-A", "out"),
+      ("call-A", "out2"),
+      ("call-B", "out"),
+      ("call-B", "out2")
+    )
+
+    val descriptor = WorkflowDescriptor(WorkflowId.randomId(), sources, ConfigFactory.load)
+
+    descriptor.copyWorkflowOutputs onComplete {
+      case Success(_) =>
+        forAll(outputs) { (call, file) =>
+          val path = tmpDir.resolve(Paths.get(descriptor.name, descriptor.id.toString, call, file))
+          Files.exists(path) shouldBe true
+        }
+      case Failure(f) => fail(f)
+    }
+  }
+
 }
diff --git a/src/test/scala/cromwell/engine/io/gcs/NioGcsPathSpec.scala b/src/test/scala/cromwell/engine/io/gcs/NioGcsPathSpec.scala
new file mode 100644
index 00000000000..e93b01e0ef1
--- /dev/null
+++ b/src/test/scala/cromwell/engine/io/gcs/NioGcsPathSpec.scala
@@ -0,0 +1,318 @@
+package cromwell.engine.io.gcs
+
+import java.nio.file.Path
+
+import org.scalatest.mock.MockitoSugar
+import org.scalatest.prop.TableDrivenPropertyChecks._
+import org.scalatest.prop.Tables.Table
+import org.scalatest.{FlatSpec, Matchers}
+
+import scala.concurrent.ExecutionContext
+
+class NioGcsPathSpec extends FlatSpec with Matchers with MockitoSugar {
+
+  behavior of "NioGcsPath"
+
+  implicit val GCSFs = GcsFileSystem.instance(mock[GoogleCloudStorage], "gs://absolute")(mock[ExecutionContext])
+
+  it should "implement toString" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    absPath1.toString shouldBe "gs://absolute/path/to/somewhere"
+    relPath1.toString shouldBe "some/relative/path"
+  }
+
+  it should "implement subpath" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    val absSub1 = absPath1.subpath(0, 2)
+    absSub1.isAbsolute shouldBe true
+    absSub1.toString shouldBe "gs://absolute/path"
+
+    val absSub2 = absPath1.subpath(1, 2)
+    absSub2.isAbsolute shouldBe false
+    absSub2.toString shouldBe "path"
+
+    val relSub1 = relPath1.subpath(0, 2)
+    relSub1.isAbsolute shouldBe false
+    relSub1.toString shouldBe "some/relative"
+  }
+
+  it should "implement resolveSibling" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val absPath2 = new NioGcsPath(Array("absolute", "location"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+    val relPath2 = new NioGcsPath(Array("another", "relative", "resource", "path"), false)
+
+    val absSibling = absPath1.resolveSibling("somewhere else")
+    absSibling.isAbsolute shouldBe true
+    absSibling.toString shouldBe "gs://absolute/path/to/somewhere else"
+
+    val absSiblingPath = absPath1.resolveSibling(relPath1)
+    absSiblingPath.isAbsolute shouldBe true
+    absSiblingPath.toString shouldBe "gs://absolute/path/to/some/relative/path"
+
+    val absRel = relPath1.resolveSibling("other path")
+    absRel.isAbsolute shouldBe false
+    absRel.toString shouldBe "some/relative/other path"
+
+    val absRelPath = relPath1.resolveSibling(relPath2)
+    absRelPath.isAbsolute shouldBe false
+    absRelPath.toString shouldBe "some/relative/another/relative/resource/path"
+  }
+
+  it should "implement resolve" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val absPath2 = new NioGcsPath(Array("absolute", "location"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+    val relPath2 = new NioGcsPath(Array("another", "relative", "resource", "path"), false)
+
+    val absToRel = absPath1.resolve(relPath1)
+    absToRel.isAbsolute shouldBe true
+    absToRel.toString shouldBe "gs://absolute/path/to/somewhere/some/relative/path"
+
+    val absToAbs = absPath1.resolve(absPath2)
+    absToAbs.isAbsolute shouldBe true
+    absToAbs.toString shouldBe "gs://absolute/location"
+
+    val relToAbs = relPath1.resolve(absPath1)
+    relToAbs.isAbsolute shouldBe true
+    relToAbs.toString shouldBe "gs://absolute/path/to/somewhere"
+
+    val relToRel = relPath1.resolve(relPath2)
+    relToRel.isAbsolute shouldBe false
+    relToRel.toString shouldBe "some/relative/path/another/relative/resource/path"
+  }
+
+  it should "implement getName" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    val nameAbs1 = absPath1.getName(0)
+    nameAbs1.isAbsolute shouldBe true
+    nameAbs1.toString shouldBe "gs://absolute"
+
+    val nameAbs2 = absPath1.getName(1)
+    nameAbs2.isAbsolute shouldBe false
+    nameAbs2.toString shouldBe "path"
+
+    val nameRel1 = relPath1.getName(0)
+    nameRel1.isAbsolute shouldBe false
+    nameRel1.toString shouldBe "some"
+
+    val nameRel2 = relPath1.getName(1)
+    nameRel2.isAbsolute shouldBe false
+    nameRel2.toString shouldBe "relative"
+  }
+
+  it should "implement getParent" in {
+    val empty = new NioGcsPath(Array.empty[String], true)
+    val singleton = new NioGcsPath(Array("singleton"), true)
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    val parentAbs1 = absPath1.getParent
+    parentAbs1.isAbsolute shouldBe true
+    parentAbs1.toString shouldBe "gs://absolute/path/to"
+
+    empty.getParent shouldBe null
+    singleton.getParent shouldBe null
+
+    val nameRel1 = relPath1.getParent
+    nameRel1.isAbsolute shouldBe false
+    nameRel1.toString shouldBe "some/relative"
+  }
+
+  it should "implement toAbsolutePath" in {
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    val abs = absPath1.toAbsolutePath
+    abs.isAbsolute shouldBe true
+    abs.toString shouldBe "gs://absolute/path/to/somewhere"
+
+    val rel = relPath1.toAbsolutePath
+    rel.isAbsolute shouldBe true
+    rel.toString shouldBe "gs://absolute/some/relative/path"
+  }
+
+  it should "implement getNameCount" in {
+    val empty = new NioGcsPath(Array.empty[String], true)
+    val singleton = new NioGcsPath(Array("singleton"), true)
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    absPath1.getNameCount shouldBe 4
+    relPath1.getNameCount shouldBe 3
+    empty.getNameCount shouldBe 0
+    singleton.getNameCount shouldBe 1
+  }
+
+  it should "implement getFileName" in {
+    val empty = new NioGcsPath(Array.empty[String], true)
+    val singletonAbs = new NioGcsPath(Array("singleton"), true)
+    val singletonRel = new NioGcsPath(Array("singleton"), false)
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    val emptyFileName = empty.getFileName
+    emptyFileName shouldBe null
+
+    val singletonAbsFileName = singletonAbs.getFileName
+    singletonAbsFileName.isAbsolute shouldBe true
+    singletonAbsFileName.toString shouldBe "gs://singleton"
+
+    val singletonRelFileName = singletonRel.getFileName
+    singletonRelFileName.isAbsolute shouldBe false
+    singletonRelFileName.toString shouldBe "singleton"
+
+    val relFileName = relPath1.getFileName
+    relFileName.isAbsolute shouldBe false
+    relFileName.toString shouldBe "path"
+
+    val absFileName = absPath1.getFileName
+    absFileName.isAbsolute shouldBe false
+    absFileName.toString shouldBe "somewhere"
+  }
+
+  it should "implement getRoot" in {
+    val empty = new NioGcsPath(Array.empty[String], true)
+    val singletonAbs = new NioGcsPath(Array("singleton"), true)
+    val singletonRel = new NioGcsPath(Array("singleton"), false)
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    an[IllegalStateException] shouldBe thrownBy(empty.getRoot)
+
+    val singletonAbsFileName = singletonAbs.getRoot
+    singletonAbsFileName.isAbsolute shouldBe true
+    singletonAbsFileName.toString shouldBe "gs://singleton"
+
+    val singletonRelFileName = singletonRel.getRoot
+    singletonRelFileName.isAbsolute shouldBe true
+    singletonRelFileName.toString shouldBe "gs://absolute"
+
+    val relFileName = relPath1.getRoot
+    relFileName.isAbsolute shouldBe true
+    relFileName.toString shouldBe "gs://absolute"
+
+    val absFileName = absPath1.getRoot
+    absFileName.isAbsolute shouldBe true
+    absFileName.toString shouldBe "gs://absolute"
+  }
+
+  it should "implement getIterator" in {
+    val empty = new NioGcsPath(Array.empty[String], true)
+    val singletonAbs = new NioGcsPath(Array("singleton"), true)
+    val singletonRel = new NioGcsPath(Array("singleton"), false)
+    val absPath1 = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val relPath1 = new NioGcsPath(Array("some", "relative", "path"), false)
+
+    empty.iterator().hasNext shouldBe false
+
+    val singletonAbsIterator = singletonAbs.iterator()
+    val nextAbsSingleton: Path = singletonAbsIterator.next()
+    nextAbsSingleton.isAbsolute shouldBe false
+    nextAbsSingleton.toString shouldBe "singleton"
+    singletonAbsIterator.hasNext shouldBe false
+
+    val singletonRelIterator = singletonRel.iterator()
+    val nextRelSingleton: Path = singletonRelIterator.next()
+    nextRelSingleton.isAbsolute shouldBe false
+    nextRelSingleton.toString shouldBe "singleton"
+    singletonRelIterator.hasNext shouldBe false
+
+    val relIterator = relPath1.iterator()
+    val nextRel: Path = relIterator.next()
+    nextRel.isAbsolute shouldBe false
+    nextRel.toString shouldBe "some"
+    relIterator.next().toString shouldBe "relative"
+    relIterator.next().toString shouldBe "path"
+    relIterator.hasNext shouldBe false
+
+    val absIterator = absPath1.iterator()
+    val absRel: Path = absIterator.next()
+    absRel.isAbsolute shouldBe false
+    absRel.toString shouldBe "absolute"
+    absIterator.next().toString shouldBe "path"
+    absIterator.next().toString shouldBe "to"
+    absIterator.next().toString shouldBe "somewhere"
+    absIterator.hasNext shouldBe false
+  }
+
+  it should "implement startsWith" in {
+    val empty = new NioGcsPath(Array.empty[String], false)
+    val singletonAbs = new NioGcsPath(Array("absolute"), true)
+    val singletonRel = new NioGcsPath(Array("absolute"), false)
+
+    val absPath = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val startsWithAbsPath = new NioGcsPath(Array("absolute", "path", "to"), true)
+    val doesntStartsWithAbsPath = new NioGcsPath(Array("absolute", "path", "to", "another", "place"), true)
+    val absPathStartingLikeRel = new NioGcsPath(Array("some", "relative", "path"), true)
+
+    val relPath = new NioGcsPath(Array("some", "relative", "path"), false)
+    val startsWithRelPath = new NioGcsPath(Array("some", "relative"), false)
+    val doesntStartsWithRelPath = new NioGcsPath(Array("some", "relative", "other", "path"), false)
+    val relPathStartingLikeAbs = new NioGcsPath(Array("absolute", "path", "to"), false)
+
+    val paths = Table(
+      ("path1", "path2", "result"),
+      (empty, empty, true),
+      (empty, absPath, false),
+      (singletonAbs, singletonAbs, true),
+      (absPath, startsWithAbsPath, true),
+      (absPath, doesntStartsWithAbsPath, false),
+      (absPath, relPathStartingLikeAbs, true),
+      (absPath, relPath, false),
+      (relPath, startsWithRelPath, true),
+      (relPath, doesntStartsWithRelPath, false),
+      (relPath, absPathStartingLikeRel, false),
+      (relPath, absPath, false)
+    )
+
+    forAll(paths) { (p1, p2, res) =>
+      val startsWith: Boolean = p1.startsWith(p2)
+      startsWith shouldBe res
+      val startsWith1: Boolean = p1.startsWith(p2.toString)
+      startsWith1 shouldBe res
+    }
+  }
+
+  it should "implement endsWith" in {
+    val empty = new NioGcsPath(Array.empty[String], false)
+    val singletonAbs = new NioGcsPath(Array("absolute"), true)
+    val singletonRel = new NioGcsPath(Array("absolute"), false)
+
+    val absPath = new NioGcsPath(Array("absolute", "path", "to", "somewhere"), true)
+    val doesntEndWithAbsPath = new NioGcsPath(Array("absolute", "path", "to", "another", "place"), true)
+    val absPathEndingLikeRel = new NioGcsPath(Array("relative", "path"), true)
+
+    val relPath = new NioGcsPath(Array("some", "relative", "path"), false)
+    val endsWithRelPath = new NioGcsPath(Array("relative", "path"), false)
+    val doesntStartsWithRelPath = new NioGcsPath(Array("relative", "other", "path"), false)
+    val relPathEndingLikeAbs = new NioGcsPath(Array("path", "to", "somewhere"), false)
+
+    val paths = Table(
+      ("path1", "path2", "result"),
+      (empty, empty, true),
+      (empty, absPath, false),
+      (singletonAbs, singletonAbs, true),
+      (absPath, absPath, true),
+      (absPath, doesntEndWithAbsPath, false),
+      (absPath, relPathEndingLikeAbs, true),
+      (absPath, relPath, false),
+      (relPath, endsWithRelPath, true),
+      (relPath, doesntStartsWithRelPath, false),
+      (relPath, absPathEndingLikeRel, false),
+      (relPath, absPath, false)
+    )
+
+    forAll(paths) { (p1, p2, res) =>
+      p1.endsWith(p2) shouldBe res
+      p1.endsWith(p2.toString) shouldBe res
+    }
+  }
+
+}
diff --git a/src/test/scala/cromwell/util/SampleWdl.scala b/src/test/scala/cromwell/util/SampleWdl.scala
index b3df1cac692..f36b55e7b1f 100644
--- a/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/src/test/scala/cromwell/util/SampleWdl.scala
@@ -298,6 +298,75 @@ object SampleWdl {
     override lazy val rawInputs = Map(ThreeStep.PatternKey -> "." * 10000)
   }
 
+  object WorkflowOutputsWithFiles extends SampleWdl {
+    // ASCII art from http://www.chris.com/ascii/joan/www.geocities.com/SoHo/7373/flag.html with pipes
+    // replaced by exclamation points to keep stripMargin from removing the flagpole.
+    override def wdlSource(runtime: String = "") =
+      """
+        task A {
+          command {
+            echo "Enfin un peu de francais pour contrer ce raz-de-marée anglais !" > out
+            echo "Jacques Chirac fait du jetski sur la Seine en costume traditionnel russe" > out2
+          }
+          output {
+            File out = "out"
+            File out2 = "out2"
+          }
+        }
+        task B {
+          command {
+             echo "Je contre avec un bonnet peruvien et tire une carte chance" > out
+             echo "Kamoulox !" > out2
+          }
+          output {
+             Array[File] outs = ["out", "out2"]
+          }
+        }
+        task C {
+          command {
+            cat > out <<END
+            (_)
+             !_________________________________________
+             !*  *  *  *  * |##########################|
+             ! *  *  *  *  *|                          |
+             !*  *  *  *  * |##########################|
+             ! *  *  *  *  *|                          |
+             !*  *  *  *  * |##########################|
+             ! *  *  *  *  *|                          |
+             !*  *  *  *  * |##########################|
+             !~~~~~~~~~~~~~~~                          |
+             !#########################################|
+             !                                         |
+             !#########################################|
+             !                                         |
+             !###################################JGS###|
+             !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+             !
+             !
+             !
+             !
+             !
+             !
+             !
+            END
+          }
+          output {
+            File out = "out"
+          }
+        }
+        workflow wfoutputs {
+          call A
+          call B
+          call C
+          output {
+            A.*
+            B.outs
+          }
+        }
+      """.stripMargin
+    override lazy val rawInputs = Map.empty[String, String]
+  }
+
   object NestedScatterWdl extends SampleWdl {
     override def wdlSource(runtime: String = "") =
       """

From a2145ac306f1360556139f90f2d16aa23817fd1a Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Tue, 5 Jan 2016 18:30:10 -0500
Subject: [PATCH 07/31] Outputs decls can reference previous outputs

---
 .../cromwell/engine/backend/Backend.scala     | 17 ++++-
 .../cromwell/engine/backend/BackendCall.scala |  5 +-
 .../engine/backend/jes/JesBackend.scala       | 64 ++++++++++---------
 .../backend/local/SharedFileSystem.scala      | 31 +++++----
 src/main/scala/cromwell/engine/package.scala  |  1 -
 .../ReferencingPreviousInputsAndOutputs.scala | 24 +++++++
 src/test/scala/cromwell/util/SampleWdl.scala  | 29 +++++++++
 7 files changed, 126 insertions(+), 45 deletions(-)
 create mode 100644 src/test/scala/cromwell/ReferencingPreviousInputsAndOutputs.scala

diff --git a/src/main/scala/cromwell/engine/backend/Backend.scala b/src/main/scala/cromwell/engine/backend/Backend.scala
index d048e980a82..bfe525613de 100644
--- a/src/main/scala/cromwell/engine/backend/Backend.scala
+++ b/src/main/scala/cromwell/engine/backend/Backend.scala
@@ -2,7 +2,6 @@ package cromwell.engine.backend
 
 import akka.actor.ActorSystem
 import com.typesafe.config.Config
-import cromwell.engine.{CallInputs, CallOutputs}
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import wdl4s._
 import cromwell.engine.ExecutionIndex.ExecutionIndex
@@ -17,9 +16,11 @@ import cromwell.engine.{HostInputs, CallOutputs}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.docker.SprayDockerRegistryApiClient
 import org.slf4j.LoggerFactory
+import wdl4s.values.WdlValue
+import scala.language.postfixOps
 
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.Try
+import scala.util.{Success, Try}
 
 object Backend {
   class StdoutStderrException(message: String) extends RuntimeException(message)
@@ -53,6 +54,18 @@ object Backend {
 
 trait JobKey
 
+final case class AttemptedLookupResult(name: String, value: Try[WdlValue]) {
+  def toPair = name -> value
+}
+
+object AttemptedLookupResult {
+  implicit class AugmentedAttemptedLookupSequence(s: Seq[AttemptedLookupResult]) {
+    def toLookupMap: Map[String, WdlValue] = s collect {
+      case AttemptedLookupResult(name, Success(value)) => (name, value)
+    } toMap
+  }
+}
+
 /**
  * Trait to be implemented by concrete backends.
  */
diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index 7556c953d52..b1ad32e3921 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -106,7 +106,10 @@ trait BackendCall {
    * expression `read_lines(my_file_var)` would have to call lookupFunction()("my_file_var")
    * during expression evaluation
    */
-  def lookupFunction: String => WdlValue = WdlExpression.standardLookupFunction(locallyQualifiedInputs, key.scope.task.declarations, engineFunctions)
+  def lookupFunction(evaluatedValues: Map[String, WdlValue]): String => WdlValue = {
+    val currentlyKnownValues = locallyQualifiedInputs ++ evaluatedValues
+    WdlExpression.standardLookupFunction(currentlyKnownValues, key.scope.task.declarations, engineFunctions)
+  }
 
   /** Initiate execution, callers can invoke `poll` once this `Future` completes successfully. */
   def execute(implicit ec: ExecutionContext): Future[ExecutionHandle]
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index cd99d5e5744..ec48cd00961 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -8,7 +8,8 @@ import akka.actor.ActorSystem
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s.{Scatter, UnsatisfiedInputsException, Call, CallInputs}
+import wdl4s.CallInputs
+import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
 import cromwell.engine.ExecutionIndex.{ExecutionIndex, IndexEnhancedInt}
@@ -355,7 +356,7 @@ case class JesBackend(actorSystem: ActorSystem)
   def generateJesOutputs(backendCall: BackendCall): Seq[JesOutput] = {
     val log = workflowLoggerWithCall(backendCall)
     val wdlFileOutputs = backendCall.call.task.outputs flatMap { taskOutput =>
-      taskOutput.expression.evaluateFiles(backendCall.lookupFunction, NoFunctions, taskOutput.wdlType) match {
+      taskOutput.expression.evaluateFiles(backendCall.lookupFunction(Map.empty), NoFunctions, taskOutput.wdlType) match {
         case Success(wdlFiles) => wdlFiles map gcsPathToLocal
         case Failure(ex) =>
           log.warn(s"Could not evaluate $taskOutput: ${ex.getMessage}")
@@ -457,8 +458,8 @@ case class JesBackend(actorSystem: ActorSystem)
     }
   }
 
-  private def customLookupFunction(backendCall: BackendCall) = { toBeLookedUp: String =>
-    val originalLookup = backendCall.lookupFunction
+  private def customLookupFunction(backendCall: BackendCall, alreadyGeneratedOutputs: Map[String, WdlValue]): String => WdlValue = toBeLookedUp => {
+    val originalLookup = backendCall.lookupFunction(alreadyGeneratedOutputs)
     gcsInputToGcsOutput(backendCall, originalLookup(toBeLookedUp))
   }
 
@@ -475,31 +476,9 @@ case class JesBackend(actorSystem: ActorSystem)
   }
 
   def postProcess(backendCall: BackendCall): Try[CallOutputs] = {
-    val outputMappings = backendCall.call.task.outputs.map({ taskOutput =>
-      /**
-        * This will evaluate the task output expression and coerces it to the task output's type.
-        * If the result is a WdlFile, then attempt to find the JesOutput with the same path and
-        * return a WdlFile that represents the GCS path and not the local path.  For example,
-        *
-        * <pre>
-        * output {
-        *   File x = "out" + ".txt"
-        * }
-        * </pre>
-        *
-        * "out" + ".txt" is evaluated to WdlString("out.txt") and then coerced into a WdlFile("out.txt")
-        * Then, via wdlFileToGcsPath(), we attempt to find the JesOutput with .name == "out.txt".
-        * If it is found, then WdlFile("gs://some_bucket/out.txt") will be returned.
-        */
-      val attemptedValue = for {
-        wdlValue <- taskOutput.expression.evaluate(customLookupFunction(backendCall), backendCall.engineFunctions)
-        coercedValue <- taskOutput.wdlType.coerceRawValue(wdlValue)
-        value = wdlValueToGcsPath(generateJesOutputs(backendCall))(coercedValue)
-      } yield value
-
-      taskOutput.name -> attemptedValue
-    }).toMap
-
+    val outputs = backendCall.call.task.outputs
+    val outputFoldingFunction = getOutputFoldingFunction(backendCall)
+    val outputMappings = outputs.foldLeft(Seq.empty[AttemptedLookupResult])(outputFoldingFunction).map(_.toPair).toMap
     TryUtil.sequenceMap(outputMappings) map { outputMap =>
       outputMap mapValues { v =>
         CallOutput(v, v.getHash(backendCall.workflowDescriptor))
@@ -507,6 +486,33 @@ case class JesBackend(actorSystem: ActorSystem)
     }
   }
 
+  private def getOutputFoldingFunction(backendCall: BackendCall): (Seq[AttemptedLookupResult], TaskOutput) => Seq[AttemptedLookupResult] = {
+    (currentList: Seq[AttemptedLookupResult], taskOutput: TaskOutput) => {
+      currentList ++ Seq(AttemptedLookupResult(taskOutput.name, outputLookup(taskOutput, backendCall, currentList)))
+    }
+  }
+
+  private def outputLookup(taskOutput: TaskOutput, backendCall: BackendCall, currentList: Seq[AttemptedLookupResult]) = for {
+  /**
+    * This will evaluate the task output expression and coerces it to the task output's type.
+    * If the result is a WdlFile, then attempt to find the JesOutput with the same path and
+    * return a WdlFile that represents the GCS path and not the local path.  For example,
+    *
+    * <pre>
+    * output {
+    *   File x = "out" + ".txt"
+    * }
+    * </pre>
+    *
+    * "out" + ".txt" is evaluated to WdlString("out.txt") and then coerced into a WdlFile("out.txt")
+    * Then, via wdlFileToGcsPath(), we attempt to find the JesOutput with .name == "out.txt".
+    * If it is found, then WdlFile("gs://some_bucket/out.txt") will be returned.
+    */
+    wdlValue <- taskOutput.expression.evaluate(customLookupFunction(backendCall, currentList.toLookupMap), backendCall.engineFunctions)
+    coercedValue <- taskOutput.wdlType.coerceRawValue(wdlValue)
+    value = wdlValueToGcsPath(generateJesOutputs(backendCall))(coercedValue)
+  } yield value
+
   def executionResult(status: RunStatus, handle: JesPendingExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future {
     val log = workflowLoggerWithCall(handle.backendCall)
 
diff --git a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
index e9c0d5561ed..accf3b70cee 100644
--- a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
+++ b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
@@ -6,7 +6,7 @@ import java.nio.file.{Files, Path, Paths}
 import better.files.{File => ScalaFile, _}
 import com.typesafe.config.ConfigFactory
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s.{Call, TaskOutput}
+import wdl4s.{CallInputs, Call, TaskOutput}
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType}
 import wdl4s.values.{WdlValue, _}
 import cromwell.engine.ExecutionIndex.ExecutionIndex
@@ -14,6 +14,7 @@ import cromwell.engine.backend.{CallLogs, LocalFileSystemBackendCall, _}
 import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs.{GcsPath, GoogleCloudStorage}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
+import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId}
 import cromwell.engine._
 import cromwell.util.TryUtil
 import org.apache.commons.io.FileUtils
@@ -24,6 +25,7 @@ import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 import Hashing._
+import cromwell.engine.backend.AttemptedLookupResult
 
 object SharedFileSystem {
   type LocalizationStrategy = (String, Path, WorkflowDescriptor) => Try[Unit]
@@ -121,19 +123,12 @@ trait SharedFileSystem {
 
   def postProcess(backendCall: LocalFileSystemBackendCall): Try[CallOutputs] = {
     implicit val hasher = backendCall.workflowDescriptor.fileHasher
-    // Evaluate output expressions, performing conversions from String -> File where required.
-    val outputMappings = backendCall.call.task.outputs map { taskOutput =>
-      val tryConvertedValue =
-        for {
-          expressionValue <- taskOutput.expression.evaluate(backendCall.lookupFunction, backendCall.engineFunctions)
-          convertedValue <- outputAutoConversion(backendCall, taskOutput, expressionValue)
-          pathAdjustedValue <- Success(absolutizeOutputWdlFile(convertedValue, backendCall.callRootPath))
-        } yield pathAdjustedValue
-      taskOutput.name -> tryConvertedValue
-    }
 
-    val taskOutputFailures = outputMappings filter { _._2.isFailure }
+    val outputs = backendCall.call.task.outputs
+    val outputFoldingFunction = getOutputFoldingFunction(backendCall)
+    val outputMappings = outputs.foldLeft(Seq.empty[AttemptedLookupResult])(outputFoldingFunction).map(_.toPair).toMap
 
+    val taskOutputFailures = outputMappings filter { _._2.isFailure }
     if (taskOutputFailures.isEmpty) {
       val unwrappedMap = outputMappings collect { case (name, Success(wdlValue)) =>
         name -> CallOutput(wdlValue, wdlValue.getHash(backendCall.workflowDescriptor))
@@ -145,6 +140,18 @@ trait SharedFileSystem {
     }
   }
 
+  private def getOutputFoldingFunction(backendCall: LocalFileSystemBackendCall): (Seq[AttemptedLookupResult], TaskOutput) => Seq[AttemptedLookupResult] = {
+    (currentList: Seq[AttemptedLookupResult], taskOutput: TaskOutput) => {
+      currentList ++ Seq(AttemptedLookupResult(taskOutput.name, outputLookup(taskOutput, backendCall, currentList)))
+    }
+  }
+
+  private def outputLookup(taskOutput: TaskOutput, backendCall: LocalFileSystemBackendCall, currentList: Seq[AttemptedLookupResult]) = for {
+    expressionValue <- taskOutput.expression.evaluate(backendCall.lookupFunction(currentList.toLookupMap), backendCall.engineFunctions)
+    convertedValue <- outputAutoConversion(backendCall, taskOutput, expressionValue)
+    pathAdjustedValue <- Success(absolutizeOutputWdlFile(convertedValue, backendCall.callRootPath))
+  } yield pathAdjustedValue
+
   def adjustOutputPaths(call: Call, outputs: CallOutputs): CallOutputs = outputs
 
   def stdoutStderr(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): CallLogs = {
diff --git a/src/main/scala/cromwell/engine/package.scala b/src/main/scala/cromwell/engine/package.scala
index 3e3e41d42ae..bf532699783 100644
--- a/src/main/scala/cromwell/engine/package.scala
+++ b/src/main/scala/cromwell/engine/package.scala
@@ -39,7 +39,6 @@ package object engine {
   type WorkflowOutputs = Map[FullyQualifiedName, CallOutput]
   type FullyQualifiedName = String
   type LocallyQualifiedName = String
-  type CallInputs = Map[String, WdlValue]
   case class CallOutput(wdlValue: WdlValue, hash: Option[SymbolHash])
   type CallOutputs = Map[LocallyQualifiedName, CallOutput]
   type HostInputs = Map[String, WdlValue]
diff --git a/src/test/scala/cromwell/ReferencingPreviousInputsAndOutputs.scala b/src/test/scala/cromwell/ReferencingPreviousInputsAndOutputs.scala
new file mode 100644
index 00000000000..cfa934746b4
--- /dev/null
+++ b/src/test/scala/cromwell/ReferencingPreviousInputsAndOutputs.scala
@@ -0,0 +1,24 @@
+package cromwell
+
+import akka.testkit._
+import cromwell.util.SampleWdl
+import wdl4s.values.WdlFloat
+
+import scala.language.postfixOps
+
+class ReferencingPreviousInputsAndOutputs extends CromwellTestkitSpec {
+  "A task with outputs which reference other outputs" should {
+    "run without let or hindrance" in {
+      runWdlAndAssertOutputs(
+        sampleWdl = SampleWdl.ReferencingPreviousInputsAndOutputs,
+        eventFilter = EventFilter.info(pattern = s"starting calls: wf.golden_pie", occurrences = 1),
+        expectedOutputs = Map(
+          "wf.golden_pie.Au" -> WdlFloat(1.6180339887),
+          "wf.golden_pie.doubleAu" -> WdlFloat(1.6180339887 * 2),
+          "wf.golden_pie.tauValue" -> WdlFloat(3.1415926 * 2),
+          "wf.golden_pie.goldenPie" -> WdlFloat(3.1415926 * 1.6180339887)
+        )
+      )
+    }
+  }
+}
diff --git a/src/test/scala/cromwell/util/SampleWdl.scala b/src/test/scala/cromwell/util/SampleWdl.scala
index f36b55e7b1f..32d4223f70a 100644
--- a/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/src/test/scala/cromwell/util/SampleWdl.scala
@@ -1737,4 +1737,33 @@ object SampleWdl {
       """.stripMargin
     override val rawInputs = Map.empty[String, String]
   }
+
+  /**
+    * Inputs referencing other inputs and outputs referencing other outputs.
+    */
+  object ReferencingPreviousInputsAndOutputs extends SampleWdl {
+    override def wdlSource(runtime: String = "") =
+      """task golden_pie {
+        |  Float pi = 3.1415926
+        |  Float tau = pi + pi
+        |
+        |  command {
+        |    echo 1.6180339887
+        |    echo ${tau} 1>&2
+        |  }
+        |
+        |  output {
+        |    Float Au = read_float(stdout())
+        |    Float doubleAu = Au + Au
+        |    Float tauValue = read_float(stderr())
+        |    Float goldenPie = Au * pi
+        |  }
+        |}
+        |
+        |workflow wf {
+        |  call golden_pie
+        |}
+      """.stripMargin
+    override val rawInputs = Map.empty[FullyQualifiedName, Any]
+  }
 }

From de4911037d166abd1ced152df711d68191b40867 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Wed, 13 Jan 2016 05:55:29 -0500
Subject: [PATCH 08/31] warn not debug on blocked async failures

---
 src/main/scala/cromwell/engine/workflow/WorkflowActor.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 3e9205c1582..4fce13cb561 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -609,10 +609,10 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       scheduleTransition(WorkflowFailed)
       stay()
     case Event(message @ AsyncFailure(t), _) =>
-      // This is the unusual combination of debug + throwable logging since the expectation is that this will eventually
+      // This is the unusual combination of warn + throwable logging since the expectation is that this will eventually
       // be logged in the case above as an error, but if for some weird reason this actor never ends up in that
       // state we don't want to be completely blind to the cause of the AsyncFailure.
-      logger.debug(t.getMessage, t)
+      logger.warn(t.getMessage, t)
       resendDueToPendingExecutionWrites(message)
       stay()
     case Event(Terminate, data) if data.pendingExecutions.isEmpty && stateName.isTerminal =>

From 4f19751fdcde487b2687d1fc4b96f34252d81e70 Mon Sep 17 00:00:00 2001
From: Kristian Cibulskis <kcibul@broadinstitute.org>
Date: Wed, 13 Jan 2016 12:49:28 -0500
Subject: [PATCH 09/31] added stanza related to workflow option encryption

---
 src/main/config/cromwell.conf.ctmpl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/main/config/cromwell.conf.ctmpl b/src/main/config/cromwell.conf.ctmpl
index 8bac5d4f191..10c95b9359a 100644
--- a/src/main/config/cromwell.conf.ctmpl
+++ b/src/main/config/cromwell.conf.ctmpl
@@ -13,6 +13,14 @@ backend {
   }
 }
 
+workflow-options {
+  // These workflow options will be encrypted when stored in the database
+  encrypted-fields: ["refresh_token"]
+
+  // AES-256 key to use to encrypt the values in `encrypted-fields`
+  base64-encryption-key: "{{.Data.workflow_options_encryption_key}}"
+}
+
 docker {
   dockerAccount = "{{.Data.docker_account}}"
   dockerToken = "{{.Data.docker_token}}"

From 7fdf4441aea303561c6300087ecf8bd4a75b9846 Mon Sep 17 00:00:00 2001
From: Miguel Covarrubias <mcovarr@broadinstitute.org>
Date: Wed, 13 Jan 2016 05:52:07 -0500
Subject: [PATCH 10/31] Fix abort.

---
 .../cromwell/engine/backend/BackendCall.scala |  5 ++++
 .../engine/backend/jes/JesBackend.scala       | 27 +++++++++----------
 .../engine/workflow/WorkflowActor.scala       |  6 ++---
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index b1ad32e3921..dd29c99f1a5 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -70,6 +70,11 @@ final case class FailedExecutionHandle(throwable: Throwable, returnCode: Option[
   override val result = FailedExecution(throwable, returnCode)
 }
 
+case object AbortedExecutionHandle extends ExecutionHandle {
+  override def isDone: Boolean = true
+  override def result: ExecutionResult = AbortedExecution
+}
+
 trait BackendCall {
   /**
    * The Workflow and Call to invoke.  It is assumed that in the creation
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index ec48cd00961..a6b5349c4d1 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -2,39 +2,36 @@ package cromwell.engine.backend.jes
 
 import java.math.BigInteger
 import java.net.SocketTimeoutException
-import java.nio.file.{Files, Path, Paths}
+import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorSystem
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s.CallInputs
-import wdl4s._
-import wdl4s.expression.NoFunctions
-import wdl4s.values._
 import cromwell.engine.ExecutionIndex.{ExecutionIndex, IndexEnhancedInt}
 import cromwell.engine.ExecutionStatus.ExecutionStatus
-import cromwell.engine.backend._
+import cromwell.engine.Hashing._
+import cromwell.engine.backend.{BackendType, _}
 import cromwell.engine.backend.jes.JesBackend._
 import cromwell.engine.backend.jes.Run.RunStatus
 import cromwell.engine.backend.jes.authentication._
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import cromwell.engine.db.DataAccess.globalDataAccess
 import cromwell.engine.db.ExecutionDatabaseKey
 import cromwell.engine.db.slick.Execution
 import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs._
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.engine.{AbortRegistrationFunction, _}
-import cromwell.engine.{HostInputs, CallOutput, CallOutputs}
+import cromwell.engine.{AbortRegistrationFunction, CallOutput, CallOutputs, HostInputs, _}
 import cromwell.logging.WorkflowLogger
-import cromwell.engine.backend.BackendType
 import cromwell.util.{AggregatedException, TryUtil}
+import wdl4s.{CallInputs, _}
+import wdl4s.expression.NoFunctions
+import wdl4s.values._
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
-import Hashing._
 
 object JesBackend {
   /*
@@ -539,12 +536,12 @@ case class JesBackend(actorSystem: ActorSystem)
         case Run.Success(events) =>
           backendCall.hash map { h => handleSuccess(outputMappings, backendCall.workflowDescriptor, events, returnCode.get, h, handle) }
         case Run.Failed(errorCode, errorMessage) =>
-          val throwable = if (errorMessage contains "Operation canceled at") {
-            new TaskAbortedException()
+          if (errorMessage contains "Operation canceled at") {
+            AbortedExecutionHandle.future
           } else {
-            new Throwable(s"Task ${backendCall.workflowDescriptor.id}:${backendCall.call.unqualifiedName} failed: error code $errorCode. Message: $errorMessage")
+            val e = new Throwable(s"Task ${backendCall.workflowDescriptor.id}:${backendCall.call.unqualifiedName} failed: error code $errorCode. Message: $errorMessage")
+            FailedExecutionHandle(e, Option(errorCode)).future
           }
-          FailedExecutionHandle(throwable, Option(errorCode)).future
       }
     } catch {
       case e: Exception =>
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 4fce13cb561..7374534f4d1 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -466,6 +466,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       // Something funky's going on if aborts are coming through while the workflow's still running. But don't second-guess
       // by transitioning the whole workflow - the message is either still in the queue or this command was maybe
       // cancelled by some external system.
+      executionStore += callKey -> ExecutionStatus.Aborted
       persistStatusThenAck(callKey, ExecutionStatus.Aborted, sender(), message)
       logger.warn(s"Call ${callKey.scope.unqualifiedName} was aborted but the workflow should still be running.")
       val updatedData = data.addPersisting(callKey, ExecutionStatus.Aborted)
@@ -529,6 +530,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       // Collector keys are weird internal things and never go to Running state.
       handleCallCompleted(collectorKey, outputs, executionEvents, returnCode, message, hash, resultsClonedFrom, data)
     case Event(message @ CallAborted(callKey), data) if data.isPersistedRunning(callKey) =>
+      executionStore += callKey -> ExecutionStatus.Aborted
       persistStatusThenAck(callKey, ExecutionStatus.Aborted, sender(), message)
       val updatedData = data.addPersisting(callKey, ExecutionStatus.Aborted)
       if (isWorkflowAborted) scheduleTransition(WorkflowAborted)
@@ -545,10 +547,6 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       if (isWorkflowAborted) scheduleTransition(WorkflowAborted)
       val updatedData = data.removePersisting(callKey, ExecutionStatus.Done)
       stay() using updatedData
-    case Event(m, _) =>
-      logger.error("Unexpected message in Aborting state: " + m.getClass.getSimpleName)
-      if (isWorkflowAborted) scheduleTransition(WorkflowAborted)
-      stay()
   }
 
   /**

From 8767cc4c4dca859427464858a9cf62030d65cd52 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Wed, 13 Jan 2016 14:29:17 -0500
Subject: [PATCH 11/31] Robustity of workflowTiming diagrams improved

---
 .../workflowTimings/workflowTimings.html      | 20 +++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/main/resources/workflowTimings/workflowTimings.html b/src/main/resources/workflowTimings/workflowTimings.html
index 1d7150fe83a..80e04885918 100644
--- a/src/main/resources/workflowTimings/workflowTimings.html
+++ b/src/main/resources/workflowTimings/workflowTimings.html
@@ -7,6 +7,14 @@
 <script type="text/javascript">
 
     google.setOnLoadCallback(drawChart);
+    function addDataTableRow(dataTable, callName, callPartName, startDate, endDate) {
+        if (startDate <= endDate) {
+            dataTable.addRow([callName, callPartName, startDate, endDate]);
+        } else {
+            console.error("Unable to add '" + callName + "'s entry: '" + callPartName + "' because start-time '" + startDate + "'' is greater than end-time '" + endDate + "'");
+        }
+    }
+
     function drawChart() {
         $.getJSON("./metadata", function( data ) {
 
@@ -69,7 +77,7 @@
                                 finalEventEnd = endDate
                             }
                             // NB: See the column definitions above.
-                            dataTable.addRow([thisCallName, description, startDate, endDate]);
+                            addDataTableRow(dataTable, thisCallName, description, startDate, endDate);
                         }
 
                         if (callList[callIndex].hasOwnProperty("start")) {
@@ -86,19 +94,19 @@
                         }
 
                         if (firstEventStart == null || finalEventEnd == null) {
-                            dataTable.addRow([thisCallName, callList[callIndex].executionStatus, callStart, callEnd]);
+                            addDataTableRow(dataTable, thisCallName, callList[callIndex].executionStatus, callStart, callEnd);
                         } else {
-                            if (callStart < firstEventStart) dataTable.addRow([thisCallName, "cromwell starting overhead", callStart, firstEventStart]);
-                            if (callEnd > finalEventEnd) dataTable.addRow([thisCallName, "cromwell final overhead", finalEventEnd, callEnd]);
+                            if (callStart < firstEventStart) addDataTableRow(dataTable, thisCallName, "cromwell starting overhead", callStart, firstEventStart);
+                            if (callEnd > finalEventEnd) addDataTableRow(dataTable, thisCallName, "cromwell final overhead", finalEventEnd, callEnd);
                         }
                     } else if (callList[callIndex].executionStatus == "Running") {
                         executionCallsCount++;
                         var endDate = workflowEnd;
                         if(endDate == null) {
-                            dataTable.addRow([thisCallName, "Running", new Date(callList[callIndex].start), new Date(Date.now())]);
+                            addDataTableRow(dataTable, thisCallName, "Running", new Date(callList[callIndex].start), new Date(Date.now()));
                         }
                         else {
-                            dataTable.addRow([thisCallName, "Still running when workflow ended", new Date(callList[callIndex].start), endDate]);
+                            addDataTableRow(dataTable, thisCallName, "Still running when workflow ended", new Date(callList[callIndex].start), endDate);
                         }
                     }
                 }

From cbefada21547a5ba0a4e6ee58370765590b26c75 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Thu, 7 Jan 2016 17:39:14 -0500
Subject: [PATCH 12/31] Add json wrapping for error response

---
 README.md                                     |  55 +++++++
 build.sbt                                     |   2 +-
 .../cromwell/engine/WorkflowDescriptor.scala  |  12 +-
 .../engine/backend/CromwellBackend.scala      |   2 +-
 .../engine/workflow/ValidateActor.scala       |   7 +-
 .../workflow/WorkflowManagerActor.scala       |   2 +
 .../cromwell/webservice/ApiDataModels.scala   |  31 +++-
 .../webservice/CromwellApiHandler.scala       |  53 +++---
 .../webservice/CromwellApiService.scala       |  57 ++++---
 .../webservice/WorkflowJsonSupport.scala      |   5 +-
 .../webservice/CromwellApiServiceSpec.scala   | 153 ++++++++++++++++--
 11 files changed, 303 insertions(+), 76 deletions(-)

diff --git a/README.md b/README.md
index 78c53dcf4cf..19ef09e39a1 100644
--- a/README.md
+++ b/README.md
@@ -60,6 +60,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
   * [POST /api/workflows/:version/:id/abort](#post-apiworkflowsversionidabort)
   * [POST /api/workflows/:version/:id/call-caching](#post-apiworkflowsversionidcall-caching)
   * [POST /api/workflows/:version/:id/call-caching/:call](#post-apiworkflowsversionidcall-cachingcall)
+  * [Error handling](#error-handling)
 * [Developer](#developer)
   * [Generating table of contents on Markdown files](#generating-table-of-contents-on-markdown-files)
   * [Generating and Hosting ScalaDoc](#generating-and-hosting-scaladoc)
@@ -1696,6 +1697,60 @@ Server: spray-can/1.3.3
 
 ```
 
+## Error handling
+Requests that Cromwell can't process return a failure in the form of a JSON response respecting the following JSON schema:
+
+```
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Error response schema",
+  "type": "object",
+  "properties": {
+    "status": {
+      "enum": [ "fail", "error"]
+    },
+    "message": {
+      "type": "string"
+    },
+    "errors": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string" },
+      "uniqueItems": true
+    }
+  },
+  "required": ["status", "message"]
+}
+```
+
+The `status` field can take two values:
+> "fail" means that the request was invalid and/or data validation failed. "fail" status is most likely returned with a 4xx HTTP Status code.
+e.g.
+
+```
+{
+  "status": "fail",
+  "message": "Workflow input processing failed.",
+  "errors": [
+    "Required workflow input 'helloworld.input' not specified."
+  ]
+}
+```
+
+> "error" means that an error occurred while processing the request. "error" status is most likely returned with a 5xx HTTP Status code.
+e.g.
+
+```
+{
+  "status": "error",
+  "message": "Connection to the database failed."
+}
+```
+
+The `message` field contains a short description of the error.
+
+The `errors` field is optional and may contain additional information about why the request failed.
+
 # Developer
 
 ## Generating table of contents on Markdown files
diff --git a/build.sbt b/build.sbt
index 37ad58012f9..1b28cc2469b 100644
--- a/build.sbt
+++ b/build.sbt
@@ -11,7 +11,7 @@ scalaVersion := "2.11.7"
 
 val lenthallV = "0.15"
 
-val wdl4sV = "0.1"
+val wdl4sV = "0.2-d949ac2-SNAPSHOT"
 
 val sprayV = "1.3.2"
 
diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index d53c21e8992..0b74a5d1f85 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -30,6 +30,7 @@ import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 import scalaz.Scalaz._
+import scalaz.Validation.FlatMap._
 
 case class WorkflowDescriptor(id: WorkflowId,
                               sourceFiles: WorkflowSourceFiles,
@@ -180,7 +181,10 @@ object WorkflowDescriptor {
     validateWorkflowDescriptor(id, sourceFiles, CromwellBackend.backend(), conf) match {
       case scalaz.Success(w) => w
       case scalaz.Failure(f) =>
-        throw new IllegalArgumentException(s"""Workflow $id failed to process inputs:\n${f.toList.mkString("\n")}""")
+        throw new IllegalArgumentException() with ThrowableWithErrors {
+          val message = s"Workflow input processing failed."
+          val errors = f
+        }
     }
   }
 
@@ -196,9 +200,8 @@ object WorkflowDescriptor {
       n <- namespace.disjunction
     } yield validateRuntimeAttributes(id, n, backend.backendType)
 
-    (namespace |@| rawInputs |@| options |@| runtimeAttributes.validation) { (_, _, _, _) } match {
-      case scalaz.Success((n, r, o, a)) => buildWorkflowDescriptor(id, sourceFiles, n, r, o, backend, conf)
-      case scalaz.Failure(f) => f.toList.mkString("\n").failureNel
+    (namespace |@| rawInputs |@| options |@| runtimeAttributes.validation) { (_, _, _, _) } flatMap {
+      case (n, r, o, a) => buildWorkflowDescriptor(id, sourceFiles, n, r, o, backend, conf)
     }
   }
 
@@ -263,6 +266,7 @@ object WorkflowDescriptor {
                                     namespace: NamespaceWithWorkflow): ErrorOr[WorkflowCoercedInputs] = {
     namespace.coerceRawInputs(rawInputs) match {
       case Success(r) => r.successNel
+      case Failure(e: ThrowableWithErrors) => scalaz.Failure(e.errors)
       case Failure(e) => e.getMessage.failureNel
     }
   }
diff --git a/src/main/scala/cromwell/engine/backend/CromwellBackend.scala b/src/main/scala/cromwell/engine/backend/CromwellBackend.scala
index ab6ff60f744..14f2c7d0264 100644
--- a/src/main/scala/cromwell/engine/backend/CromwellBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/CromwellBackend.scala
@@ -1,8 +1,8 @@
 package cromwell.engine.backend
 
 import akka.actor.ActorSystem
+import cromwell.engine.backend.Backend.BackendyString
 import org.slf4j.LoggerFactory
-import Backend.BackendyString
 
 /**
   * Provides a global singleton access to the instantiated backend. This isn't bulletproof and can lead to
diff --git a/src/main/scala/cromwell/engine/workflow/ValidateActor.scala b/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
index dd80a050c93..60f048ff52d 100644
--- a/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
@@ -6,7 +6,8 @@ import cromwell.engine.backend.CromwellBackend
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import wdl4s._
 import cromwell.webservice.PerRequest.RequestComplete
-import cromwell.webservice.{WorkflowJsonSupport, WorkflowValidateResponse}
+import cromwell.webservice.{APIResponse, WorkflowJsonSupport, WorkflowValidateResponse}
+import cromwell.webservice.WorkflowJsonSupport._
 import spray.http.StatusCodes
 import spray.httpx.SprayJsonSupport._
 import spray.json._
@@ -58,14 +59,14 @@ class ValidateActor(wdlSource: WdlSource, wdlJson: WdlJson)
         logger.info(s"$tag success $sentBy")
         sentBy ! RequestComplete(
           StatusCodes.OK,
-          WorkflowValidateResponse(valid = true, error = None))
+          APIResponse.success("Validation succeeded."))
 
       case Failure(ex) =>
         val messageOrBlank = Option(ex.getMessage).mkString
         logger.info(s"$tag error $sentBy: $messageOrBlank")
         sentBy ! RequestComplete(
           StatusCodes.BadRequest,
-          WorkflowValidateResponse(valid = false, error = Option(messageOrBlank)))
+          APIResponse.fail(ex))
     }
   }
 }
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 55f40e48414..bd49540c045 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -164,6 +164,7 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
     for {
         _ <- assertWorkflowExistence(workflowId)
         descriptor <- globalDataAccess.getWorkflow(workflowId)
+        _ <- assertCallExistence(workflowId, callFqn)
         callName <- Future.fromTry(assertCallFqnWellFormed(descriptor, callFqn))
         callLogKeys <- getCallLogKeys(workflowId, callFqn)
         callStandardOutput <- Future.successful(callLogKeys map { key => backend.stdoutStderr(descriptor, callName, key.index) })
@@ -299,6 +300,7 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
 
   private def callCaching(id: WorkflowId, parameters: QueryParameters, callName: Option[String]): Future[Int] = {
     for {
+      _ <- assertWorkflowExistence(id)
       cachingParameters <- CallCachingParameters.from(id, callName, parameters)
       updateCount <- globalDataAccess.updateCallCaching(cachingParameters)
     } yield updateCount
diff --git a/src/main/scala/cromwell/webservice/ApiDataModels.scala b/src/main/scala/cromwell/webservice/ApiDataModels.scala
index be1b323a8b5..ada00e7a78b 100644
--- a/src/main/scala/cromwell/webservice/ApiDataModels.scala
+++ b/src/main/scala/cromwell/webservice/ApiDataModels.scala
@@ -1,10 +1,12 @@
 package cromwell.webservice
 
-import wdl4s.FullyQualifiedName
-import wdl4s.values.WdlValue
-import cromwell.engine.backend.{WorkflowQueryResult, CallMetadata, CallLogs}
+import cromwell.engine.backend.{CallLogs, CallMetadata, WorkflowQueryResult}
 import org.joda.time.DateTime
-import spray.json.JsObject
+import spray.json._
+import wdl4s.values.WdlValue
+import wdl4s.{FullyQualifiedName, ThrowableWithErrors}
+
+import scala.language.postfixOps
 
 case class WorkflowValidateResponse(valid: Boolean, error: Option[String])
 
@@ -33,3 +35,24 @@ case class WorkflowMetadataResponse(id: String,
 case class WorkflowQueryResponse(results: Seq[WorkflowQueryResult])
 
 final case class CallCachingResponse(updateCount: Int)
+
+object APIResponse {
+  private def constructFailureResponse(status: String, ex: Throwable) ={
+    ex match {
+      case cex: ThrowableWithErrors => FailureResponse(status, cex.message, Option(JsArray(cex.errors.list.map(JsString(_)).toVector)))
+      case e: Throwable => FailureResponse(status, e.getMessage, None)
+    }
+  }
+
+  /** When the data submitted in the request is incorrect. */
+  def fail(ex: Throwable) = constructFailureResponse("fail", ex)
+
+  /** When an exception is thrown while processing the request. */
+  def error(ex: Throwable) = constructFailureResponse("error", ex)
+
+  /** When a request completes successfully. */
+  def success(message: String, data: Option[JsValue] = None) = SuccessResponse("success", message, data)
+}
+
+case class SuccessResponse(status: String, message: String, data: Option[JsValue])
+case class FailureResponse(status: String, message: String, errors: Option[JsValue])
diff --git a/src/main/scala/cromwell/webservice/CromwellApiHandler.scala b/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
index 09dfe07355c..579f538dfad 100644
--- a/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
+++ b/src/main/scala/cromwell/webservice/CromwellApiHandler.scala
@@ -13,6 +13,7 @@ import cromwell.webservice.PerRequest.RequestComplete
 import cromwell.engine
 import spray.http.StatusCodes
 import spray.httpx.SprayJsonSupport._
+import spray.json._
 
 import scala.concurrent.Future
 import scala.concurrent.duration._
@@ -45,6 +46,11 @@ class CromwellApiHandler(workflowManager: ActorRef) extends Actor {
   implicit val timeout = Timeout(2 seconds)
   val log = Logging(context.system, classOf[CromwellApiHandler])
 
+  def workflowNotFound(id: WorkflowId) = RequestComplete(StatusCodes.NotFound, APIResponse.error(new Throwable(s"Workflow '$id' not found.")))
+  def callNotFound(callFqn: String, id: WorkflowId) = {
+    RequestComplete(StatusCodes.NotFound, APIResponse.error(new Throwable(s"Call $callFqn not found for workflow '$id'.")))
+  }
+
   override def receive = {
     case WorkflowStatus(id) =>
       val futureStatus = queryWorkflowManager(WorkflowManagerActor.WorkflowStatus(id))
@@ -60,25 +66,25 @@ class CromwellApiHandler(workflowManager: ActorRef) extends Actor {
         case Failure(ex) =>
           ex match {
             case _: IllegalArgumentException =>
-              context.parent ! RequestComplete(StatusCodes.BadRequest, ex.getMessage)
+              context.parent ! RequestComplete(StatusCodes.BadRequest, APIResponse.fail(ex))
             case _ =>
-              context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+              context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
           }
       }
     case WorkflowAbort(id) =>
       val futureStatus = queryWorkflowManager(WorkflowManagerActor.WorkflowStatus(id))
       futureStatus onSuccess {
         case None =>
-          context.parent ! RequestComplete(StatusCodes.NotFound, s"Workflow ID '$id' not found.")
+          context.parent ! workflowNotFound(id)
         case Some(workflowState) if workflowState.isTerminal =>
           val message = s"Workflow ID '$id' is in terminal state '$workflowState' and cannot be aborted."
-          context.parent ! RequestComplete(StatusCodes.Forbidden, message)
+          context.parent ! RequestComplete(StatusCodes.Forbidden, APIResponse.fail(new Throwable(message)))
         case Some(workflowState) =>
           val futureAbortedStatus = queryWorkflowManager(WorkflowManagerActor.WorkflowAbort(id))
           futureAbortedStatus onComplete {
             case Failure(e) =>
               log.error(e, e.getMessage)
-              context.parent ! RequestComplete(StatusCodes.InternalServerError, e.getMessage)
+              context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(e))
             case Success(status) =>
               context.parent ! RequestComplete(StatusCodes.OK, WorkflowAbortResponse(id.toString, WorkflowAborted.toString))
           }
@@ -90,55 +96,56 @@ class CromwellApiHandler(workflowManager: ActorRef) extends Actor {
           context.parent ! RequestComplete(StatusCodes.Created, WorkflowSubmitResponse(id.toString, engine.WorkflowSubmitted.toString))
         case Failure(ex) =>
           ex match {
-            case _: IllegalArgumentException => context.parent ! RequestComplete(StatusCodes.BadRequest, ex.getMessage)
-            case _ => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+            case _: IllegalArgumentException => context.parent ! RequestComplete(StatusCodes.BadRequest, APIResponse.fail(ex))
+            case _ => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
           }
       }
     case WorkflowOutputs(id) =>
       val eventualWorkflowOutputs = ask(workflowManager, WorkflowManagerActor.WorkflowOutputs(id)).mapTo[engine.WorkflowOutputs]
       eventualWorkflowOutputs onComplete {
         case Success(outputs) => context.parent ! RequestComplete(StatusCodes.OK, WorkflowOutputResponse(id.toString, outputs.mapToValues))
-        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound)
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
     case CallOutputs(id, callFqn) =>
       val eventualCallOutputs = ask(workflowManager, WorkflowManagerActor.CallOutputs(id, callFqn)).mapTo[engine.CallOutputs]
       eventualCallOutputs onComplete {
         case Success(outputs) if outputs.nonEmpty => context.parent ! RequestComplete(StatusCodes.OK, CallOutputResponse(id.toString, callFqn, outputs.mapToValues))
-        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, s"Workflow '$id' not found.")
-        case Failure(ex: WorkflowManagerActor.CallNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, s"Call $callFqn not found for workflow '$id'.")
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex: WorkflowManagerActor.CallNotFoundException) => context.parent ! callNotFound(callFqn, id)
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
     case CallStdoutStderr(id, callFqn) =>
       val eventualCallLogs = ask(workflowManager, WorkflowManagerActor.CallStdoutStderr(id, callFqn)).mapTo[Seq[CallLogs]]
       eventualCallLogs onComplete {
         case Success(logs) => context.parent ! RequestComplete(StatusCodes.OK, CallStdoutStderrResponse(id.toString, Map(callFqn -> logs)))
-        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, ex.getMessage)
-        case Failure(ex: WorkflowManagerActor.CallNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, ex.getMessage)
-        case Failure(ex: Backend.StdoutStderrException) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex: WorkflowManagerActor.CallNotFoundException) => context.parent ! callNotFound(callFqn, id)
+        case Failure(ex: Backend.StdoutStderrException) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
     case WorkflowStdoutStderr(id) =>
       val eventualCallLogs = ask(workflowManager, WorkflowManagerActor.WorkflowStdoutStderr(id)).mapTo[Map[String, Seq[CallLogs]]]
       eventualCallLogs onComplete {
         case Success(logs) =>
           context.parent ! RequestComplete(StatusCodes.OK, CallStdoutStderrResponse(id.toString, logs))
-        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, ex.getMessage)
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
     case WorkflowMetadata(id) =>
       val eventualMetadataResponse = ask(workflowManager, WorkflowManagerActor.WorkflowMetadata(id)).mapTo[WorkflowMetadataResponse]
       eventualMetadataResponse onComplete {
         case Success(metadata) => context.parent ! RequestComplete(StatusCodes.OK, metadata)
-        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! RequestComplete(StatusCodes.NotFound, ex.getMessage)
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
     case CallCaching(id, parameters, callName) =>
       val eventualUpdateCount = ask(workflowManager, WorkflowManagerActor.CallCaching(id, parameters, callName)).mapTo[Int]
       eventualUpdateCount onComplete {
         case Success(updateCount) => context.parent ! RequestComplete(StatusCodes.OK, CallCachingResponse(updateCount))
-        case Failure(ex: IllegalArgumentException) => context.parent ! RequestComplete(StatusCodes.BadRequest, ex.getMessage)
-        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        case Failure(ex: WorkflowManagerActor.WorkflowNotFoundException) => context.parent ! workflowNotFound(id)
+        case Failure(ex: IllegalArgumentException) => context.parent ! RequestComplete(StatusCodes.BadRequest, APIResponse.fail(ex))
+        case Failure(ex) => context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
       }
   }
 
@@ -153,7 +160,7 @@ class CromwellApiHandler(workflowManager: ActorRef) extends Actor {
     future onFailure {
       case ex =>
         log.error(ex, ex.getMessage)
-        context.parent ! RequestComplete(StatusCodes.InternalServerError, ex.getMessage)
+        context.parent ! RequestComplete(StatusCodes.InternalServerError, APIResponse.error(ex))
     }
 
     future
diff --git a/src/main/scala/cromwell/webservice/CromwellApiService.scala b/src/main/scala/cromwell/webservice/CromwellApiService.scala
index c3356463196..37e18cd8217 100644
--- a/src/main/scala/cromwell/webservice/CromwellApiService.scala
+++ b/src/main/scala/cromwell/webservice/CromwellApiService.scala
@@ -2,13 +2,15 @@ package cromwell.webservice
 
 import akka.actor.{Actor, ActorRef, Props}
 import com.typesafe.config.Config
-import cromwell.engine.workflow.{ValidateActor, WorkflowOptions}
+import cromwell.engine.workflow.ValidateActor
 import cromwell.engine.{WorkflowId, WorkflowSourceFiles}
 import cromwell.instrumentation.Instrumentation.Monitor
 import cromwell.webservice.CromwellApiServiceActor.traceName
+import cromwell.webservice.WorkflowJsonSupport._
 import lenthall.config.ScalaConfig._
 import lenthall.spray.SwaggerUiResourceHttpService
 import lenthall.spray.WrappedRoute._
+import spray.http.MediaTypes._
 import spray.http.StatusCodes
 import spray.json._
 import spray.routing.Directive.pimpApply
@@ -47,17 +49,21 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
 
   val workflowManager: ActorRef
 
+  private def invalidWorkflowId(id: String) = respondWithMediaType(`application/json`) {
+    complete(StatusCodes.BadRequest, APIResponse.fail(new Throwable(s"Invalid workflow ID: '$id'.")).toJson.prettyPrint)
+  }
+
   val workflowRoutes = queryRoute ~ workflowOutputsRoute ~ submitRoute ~ workflowStdoutStderrRoute ~ abortRoute ~
     callOutputsRoute ~ callStdoutStderrRoute ~ validateRoute ~ metadataRoute ~ timingRoute ~ callCachingRoute ~ statusRoute
 
   def statusRoute =
-    path("workflows" / Segment / Segment / "status") { (version, id) =>
+    path("workflows" / Segment / Segment / "status") { (version, workflowId) =>
       traceName("status") {
         get {
-          Try(WorkflowId.fromString(id)) match {
-            case Success(workflowId) =>
-              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowStatus(workflowId))
-            case Failure(ex) => complete(StatusCodes.BadRequest)
+          Try(WorkflowId.fromString(workflowId)) match {
+            case Success(w) =>
+              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowStatus(w))
+            case Failure(ex) => invalidWorkflowId(workflowId)
           }
         }
       }
@@ -76,13 +82,13 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     }
 
   def abortRoute =
-    path("workflows" / Segment / Segment / "abort") { (version, id) =>
+    path("workflows" / Segment / Segment / "abort") { (version, workflowId) =>
       traceName("abort") {
         post {
-          Try(WorkflowId.fromString(id)) match {
-            case Success(workflowId) =>
-              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowAbort(workflowId))
-            case Failure(ex) => complete(StatusCodes.BadRequest)
+          Try(WorkflowId.fromString(workflowId)) match {
+            case Success(w) =>
+              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowAbort(w))
+            case Failure(ex) => invalidWorkflowId(workflowId)
           }
         }
       }
@@ -117,13 +123,13 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     }
 
   def workflowOutputsRoute =
-    path("workflows" / Segment / Segment / "outputs") { (version, id) =>
+    path("workflows" / Segment / Segment / "outputs") { (version, workflowId) =>
       traceName("workflowOutputs") {
         get {
-          Try(WorkflowId.fromString(id)) match {
-            case Success(workflowId) =>
-              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowOutputs(workflowId))
-            case Failure(ex) => complete(StatusCodes.BadRequest)
+          Try(WorkflowId.fromString(workflowId)) match {
+            case Success(w) =>
+              requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowOutputs(w))
+            case Failure(ex) => invalidWorkflowId(workflowId)
           }
         }
       }
@@ -136,7 +142,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
           case Success(w) =>
             // This currently does not attempt to parse the call name for conformation to any pattern.
             requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.CallOutputs(w, callFqn))
-          case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+          case Failure(_) => invalidWorkflowId(workflowId)
         }
       }
     }
@@ -148,7 +154,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
           case Success(w) =>
             // This currently does not attempt to parse the call name for conformation to any pattern.
             requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.CallStdoutStderr(w, callFqn))
-          case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+          case Failure(_) => invalidWorkflowId(workflowId)
         }
       }
     }
@@ -159,7 +165,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
         Try(WorkflowId.fromString(workflowId)) match {
           case Success(w) =>
             requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowStdoutStderr(w))
-          case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+          case Failure(_) => invalidWorkflowId(workflowId)
         }
       }
     }
@@ -170,7 +176,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
         Try(WorkflowId.fromString(workflowId)) match {
           case Success(w) =>
             requestContext => perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.WorkflowMetadata(w))
-          case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+          case Failure(_) => invalidWorkflowId(workflowId)
         }
       }
     }
@@ -180,7 +186,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
       traceName("timing") {
         Try(WorkflowId.fromString(workflowId)) match {
           case Success(_) => getFromResource("workflowTimings/workflowTimings.html")
-          case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+          case Failure(_) => invalidWorkflowId(workflowId)
         }
       }
     }
@@ -189,11 +195,12 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     path("workflows" / Segment / Segment / "call-caching" ~ (Slash ~ Segment).?) { (version, workflowId, callFqn) =>
       parameterSeq { parameters =>
         val queryParameters = parameters map { case (k, v) => QueryParameter(k, v) }
-        post { requestContext =>
+        post {
           Try(WorkflowId.fromString(workflowId)) match {
-            case Success(w) => perRequest(requestContext, CromwellApiHandler.props(workflowManager),
-              CromwellApiHandler.CallCaching(w, queryParameters, callFqn))
-            case Failure(_) => complete(StatusCodes.BadRequest, s"Invalid workflow ID: '$workflowId'.")
+            case Success(w) =>
+              requestContext =>
+                perRequest(requestContext, CromwellApiHandler.props(workflowManager), CromwellApiHandler.CallCaching(w, queryParameters, callFqn))
+            case Failure(_) => invalidWorkflowId(workflowId)
           }
         }
       }
diff --git a/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala b/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
index 5ced0b47ab6..78ca8414e59 100644
--- a/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
+++ b/src/main/scala/cromwell/webservice/WorkflowJsonSupport.scala
@@ -2,7 +2,7 @@ package cromwell.webservice
 
 import WdlFileJsonFormatter._
 import WdlValueJsonFormatter._
-import cromwell.engine.ExecutionEventEntry
+import cromwell.engine.{WorkflowSourceFiles, ExecutionEventEntry}
 import cromwell.engine.backend.{WorkflowQueryResult, CallMetadata, CallLogs}
 import org.joda.time.DateTime
 import org.joda.time.format.ISODateTimeFormat
@@ -18,6 +18,9 @@ object WorkflowJsonSupport extends DefaultJsonProtocol {
   implicit val callOutputResponseProtocol = jsonFormat3(CallOutputResponse)
   implicit val callLogsResponseProtocol = jsonFormat3(CallLogs)
   implicit val callStdoutStderrResponse = jsonFormat2(CallStdoutStderrResponse)
+  implicit val workflowSourceData = jsonFormat3(WorkflowSourceFiles)
+  implicit val errorResponse = jsonFormat3(FailureResponse)
+  implicit val successResponse = jsonFormat3(SuccessResponse)
 
   implicit object DateJsonFormat extends RootJsonFormat[DateTime] {
     private val parserISO = ISODateTimeFormat.dateTime()
diff --git a/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala b/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
index d6c3a779bc2..6116f717beb 100644
--- a/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
+++ b/src/test/scala/cromwell/webservice/CromwellApiServiceSpec.scala
@@ -7,6 +7,7 @@ import akka.pattern.pipe
 import cromwell.engine.{CallOutput, SymbolHash}
 import wdl4s._
 import wdl4s.values.{WdlFile, WdlInteger, WdlValue}
+import cromwell.CromwellTestkitSpec.TestWorkflowManagerSystem
 import cromwell.engine._
 import cromwell.engine.backend.{CallLogs, WorkflowQueryResult}
 import cromwell.engine.workflow.WorkflowManagerActor._
@@ -145,7 +146,7 @@ class MockWorkflowManagerActor extends Actor  {
         head match {
           case ("BadKey", _) =>
             // The exception text is rendered as the body, so there must be exception text or Spray will 500 (!)
-            throw new IllegalArgumentException("aw snap")
+            throw new IllegalArgumentException("Unrecognized query keys: BadKey")
           case ("status", _) =>
             WorkflowQueryResponse(
               Seq(
@@ -238,8 +239,12 @@ object CromwellApiServiceSpec {
 }
 
 class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with ScalatestRouteTest with Matchers {
-  override def actorRefFactory = system
-  override val workflowManager = system.actorOf(Props(new MockWorkflowManagerActor()))
+  import spray.httpx.SprayJsonSupport._
+  import spray.httpx.unmarshalling._
+
+  val testWorkflowManagerSystem = new TestWorkflowManagerSystem
+  override def actorRefFactory = testWorkflowManagerSystem.actorSystem
+  override val workflowManager = actorRefFactory.actorOf(Props(new MockWorkflowManagerActor()))
   val version = "v1"
 
   s"CromwellApiService $version" should "return 404 for get of unknown workflow" in {
@@ -259,6 +264,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.BadRequest) {
           status
         }
+        assertResult(
+          """{
+             |  "status": "fail",
+             |  "message": "Invalid workflow ID: 'foobar'."
+             |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -270,7 +283,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
           s"""{
               |  "id": "${MockWorkflowManagerActor.runningWorkflowId.toString}",
               |  "status": "Running"
-              |}""".stripMargin) {
+              |}""".stripMargin
+        ) {
           responseAs[String]
         }
         assertResult(StatusCodes.OK) {
@@ -286,6 +300,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.NotFound) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "error",
+              |  "message": "Workflow '${MockWorkflowManagerActor.unknownId}' not found."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -296,6 +318,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.BadRequest) {
           status
         }
+        assertResult(
+          """{
+              |  "status": "fail",
+              |  "message": "Invalid workflow ID: 'foobar'."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -306,6 +336,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       assertResult(StatusCodes.Forbidden) {
         status
       }
+      assertResult(
+        s"""{
+            |  "status": "fail",
+            |  "message": "Workflow ID '${MockWorkflowManagerActor.abortedWorkflowId}' is in terminal state 'Aborted' and cannot be aborted."
+            |}""".stripMargin
+      ) {
+        responseAs[String]
+      }
     }
   }
 
@@ -352,7 +390,17 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
           status
         }
         assertResult(true) {
-          responseAs[String].contains("contains bad inputs JSON")
+          val fields: Map[String, JsValue] = responseAs[Map[String, JsValue]]
+          fields.get("status").isDefined &&
+          fields.get("status").get.asInstanceOf[JsString].value.equals("fail") &&
+          fields.get("message").isDefined &&
+          fields.get("message").get.asInstanceOf[JsString].value.contains("failed to process inputs")
+          fields.get("errors").isDefined &&
+            (fields.get("errors").get match {
+            case array: JsArray if array.elements.length == 1 =>
+              array.elements.head.asInstanceOf[JsString].value.contains("contains bad inputs JSON")
+            case _ => false
+          })
         }
       }
   }
@@ -365,7 +413,17 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
           status
         }
         assertResult(true) {
-          responseAs[String].contains("contains bad options JSON")
+          val fields: Map[String, JsValue] = responseAs[Map[String, JsValue]]
+          fields.get("status").isDefined &&
+            fields.get("status").get.asInstanceOf[JsString].value.equals("fail") &&
+            fields.get("message").isDefined &&
+            fields.get("message").get.asInstanceOf[JsString].value.contains("failed to process inputs")
+          fields.get("errors").isDefined &&
+            (fields.get("errors").get match {
+              case array: JsArray if array.elements.length == 1 =>
+                array.elements.head.asInstanceOf[JsString].value.contains("contains bad options JSON")
+              case _ => false
+            })
         }
       }
   }
@@ -379,7 +437,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       check {
         assertResult(
           s"""{
-              |  "valid": true
+              |  "status": "success",
+              |  "message": "Validation succeeded."
               |}""".stripMargin) {
           responseAs[String]
         }
@@ -398,8 +457,9 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       check {
         assertResult(
           s"""{
-              |  "valid": false,
-              |  "error": "The following errors occurred while processing your inputs:\\n\\nRequired workflow input 'hello.hello.addressee' not specified."
+              |  "status": "fail",
+              |  "message": "Workflow input processing failed.",
+              |  "errors": ["Required workflow input 'hello.hello.addressee' not specified."]
               |}""".stripMargin) {
           responseAs[String]
         }
@@ -418,8 +478,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       check {
         assertResult(
           s"""{
-              |  "valid": false,
-              |  "error": "ERROR: Finished parsing without consuming all tokens.\\n\\nfoobar bad wdl!\\n^\\n     "
+              |  "status": "fail",
+              |  "message": "ERROR: Finished parsing without consuming all tokens.\\n\\nfoobar bad wdl!\\n^\\n     "
               |}""".stripMargin) {
           responseAs[String]
         }
@@ -438,8 +498,8 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       check {
         assertResult(
           s"""{
-              |  "valid": false,
-              |  "error": "Unexpected character 'o' at input index 0 (line 1, position 1), expected JSON Value:\\nfoobar bad json!\\n^\\n"
+              |  "status": "fail",
+              |  "message": "Unexpected character 'o' at input index 0 (line 1, position 1), expected JSON Value:\\nfoobar bad json!\\n^\\n"
               |}""".stripMargin) {
           responseAs[String]
         }
@@ -477,6 +537,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
       assertResult(StatusCodes.NotFound) {
         status
       }
+      assertResult(
+        s"""{
+            |  "status": "error",
+            |  "message": "Workflow '$unknownId' not found."
+            |}""".stripMargin
+      ) {
+        responseAs[String]
+      }
     }
   }
 
@@ -507,6 +575,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.NotFound) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "error",
+              |  "message": "Workflow '$unknownId' not found."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -517,6 +593,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.NotFound) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "error",
+              |  "message": "Call bogus_workflow.bogus_call not found for workflow '$submittedWorkflowId'."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -527,6 +611,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.BadRequest) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "fail",
+              |  "message": "Invalid workflow ID: 'foobar'."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
   it should "return 405 with POST of outputs on successful execution of workflow" in {
@@ -587,12 +679,21 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
   }
 
   it should "return 404 if the workflow ID is not found" in {
-    Get(s"/workflows/$version/${UUID.randomUUID().toString}/logs/three_step.wc") ~>
+    val randomID = {UUID.randomUUID().toString}
+    Get(s"/workflows/$version/$randomID/logs/three_step.wc") ~>
       callStdoutStderrRoute ~>
       check {
         assertResult(StatusCodes.NotFound) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "error",
+              |  "message": "Workflow '$randomID' not found."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -603,6 +704,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.NotFound) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "error",
+              |  "message": "Call three_step.wcBADBAD not found for workflow '$submittedWorkflowId'."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -613,6 +722,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.BadRequest) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "fail",
+              |  "message": "Invalid workflow ID: 'foobar'."
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 
@@ -680,6 +797,14 @@ class CromwellApiServiceSpec extends FlatSpec with CromwellApiService with Scala
         assertResult(StatusCodes.BadRequest) {
           status
         }
+        assertResult(
+          s"""{
+              |  "status": "fail",
+              |  "message": "Unrecognized query keys: BadKey"
+              |}""".stripMargin
+        ) {
+          responseAs[String]
+        }
       }
   }
 

From 270f41012bbab60957abb59000d5fdbbf2f9c56d Mon Sep 17 00:00:00 2001
From: geoffjentry <geoffjentry@gmail.com>
Date: Fri, 15 Jan 2016 13:14:54 -0500
Subject: [PATCH 13/31] more prominent gitter channel plug

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 19ef09e39a1..30423da0485 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,9 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
 
 <!---toc start-->
 
-* [Mailing List](#mailing-list)
+* [How To Reach Us](#how-to-reach-us)
+  * [Gitter](#gitter)
+  * [Mailing List](#mailing-list)
 * [Requirements](#requirements)
 * [Building](#building)
 * [Installing](#installing)
@@ -67,8 +69,12 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
 
 <!---toc end-->
 
-# Mailing List
+# How To Reach Us
 
+## Gitter
+There is a [gitter channel](https://gitter.im/broadinstitute/cromwell) where people can discuss Cromwell and related topics with both the developers and user community.
+
+## Mailing List
 The [Cromwell Mailing List](https://groups.google.com/a/broadinstitute.org/forum/?hl=en#!forum/cromwell) is cromwell@broadinstitute.org.
 
 If you have any questions, suggestions or support issues please send them to this list. To subscribe you can either join via the link above or send an email to cromwell+subscribe@broadinstitute.org.

From fb45e39c84bc6deec46bf9da12d61add72412fb1 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Thu, 14 Jan 2016 15:26:24 -0500
Subject: [PATCH 14/31] Validate RuntimeAttributes before starting a call

---
 .../cromwell/engine/backend/BackendCall.scala |  3 +-
 .../backend/local/LocalBackendCall.scala      |  2 +-
 .../CromwellRuntimeAttributes.scala           |  6 +-
 .../engine/workflow/WorkflowActor.scala       | 48 +++++++------
 .../InvalidRuntimeAttributesSpec.scala        | 67 +++++++++++++++++++
 5 files changed, 102 insertions(+), 24 deletions(-)
 create mode 100644 src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala

diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index dd29c99f1a5..e79f72fd460 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -131,7 +131,8 @@ trait BackendCall {
 
   def useCachedCall(cachedBackendCall: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] = ???
 
-  val runtimeAttributes = CromwellRuntimeAttributes(call.task.runtimeAttributes, backend.backendType)
+  @throws[IllegalArgumentException]
+  lazy val runtimeAttributes = CromwellRuntimeAttributes(call.task.runtimeAttributes, backend.backendType)
 
   /** Given the specified value for the Docker hash, return the overall hash for this `BackendCall`. */
   private def hashGivenDockerHash(dockerHash: Option[String]): ExecutionHash = {
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
index 967d981d914..023640e6f05 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
@@ -16,7 +16,7 @@ case class LocalBackendCall(backend: LocalBackend,
   val workflowRootPath = LocalBackend.hostExecutionPath(workflowDescriptor)
   val callRootPath = LocalBackend.hostCallPath(workflowDescriptor, call.unqualifiedName, key.index)
   val dockerContainerExecutionDir = LocalBackend.containerExecutionPath(workflowDescriptor)
-  val containerCallRoot = runtimeAttributes.docker match {
+  lazy val containerCallRoot = runtimeAttributes.docker match {
     case Some(docker) => LocalBackend.containerCallPath(workflowDescriptor, call.unqualifiedName, key.index)
     case None => callRootPath
   }
diff --git a/src/main/scala/cromwell/engine/backend/runtimeattributes/CromwellRuntimeAttributes.scala b/src/main/scala/cromwell/engine/backend/runtimeattributes/CromwellRuntimeAttributes.scala
index 481f34dfdbe..9a5f51cd21e 100644
--- a/src/main/scala/cromwell/engine/backend/runtimeattributes/CromwellRuntimeAttributes.scala
+++ b/src/main/scala/cromwell/engine/backend/runtimeattributes/CromwellRuntimeAttributes.scala
@@ -39,8 +39,10 @@ object CromwellRuntimeAttributes {
     validatedRuntimeAttributes match {
       case Success(r) => r
       case Failure(f) =>
-        val errorMessages = f.toList.mkString(", ")
-        throw new IllegalArgumentException(s"RuntimeAttribute is not valid: Errors: $errorMessages")
+        throw new IllegalArgumentException() with ThrowableWithErrors {
+          val message = "RuntimeAttribute is not valid."
+          val errors = f
+        }
     }
   }
 
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 7374534f4d1..7d3fa5577a1 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -1061,28 +1061,36 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
         self ! InitialStartCall(callKey, CallActor.Start)
     }
 
-    if (backendCall.workflowDescriptor.readFromCache) {
-      backendCall.hash map { hash =>
-        globalDataAccess.getExecutionsWithResuableResultsByHash(hash.overallHash) onComplete {
-          case Success(executions) if executions.nonEmpty =>
-            val cachedExecution = executions.head
-            globalDataAccess.getWorkflow(cachedExecution.workflowExecutionId) onComplete { cachedDescriptor =>
-              loadCachedCallOrInitiateCall(cachedDescriptor, cachedExecution)
-            }
-          case Success(_) =>
-            log.info(s"Call Caching: cache miss")
-            self ! InitialStartCall(callKey, CallActor.Start)
-          case Failure(ex) =>
-            log.error(s"Call Caching: Failed to look up executions that matched hash '$hash'. Falling back to normal execution", ex)
-            self ! InitialStartCall(callKey, CallActor.Start)
+    def startCall = {
+      if (backendCall.workflowDescriptor.readFromCache) {
+        backendCall.hash map { hash =>
+          globalDataAccess.getExecutionsWithResuableResultsByHash(hash.overallHash) onComplete {
+            case Success(executions) if executions.nonEmpty =>
+              val cachedExecution = executions.head
+              globalDataAccess.getWorkflow(cachedExecution.workflowExecutionId) onComplete { cachedDescriptor =>
+                loadCachedCallOrInitiateCall(cachedDescriptor, cachedExecution)
+              }
+            case Success(_) =>
+              log.info(s"Call Caching: cache miss")
+              self ! InitialStartCall(callKey, CallActor.Start)
+            case Failure(ex) =>
+              log.error(s"Call Caching: Failed to look up executions that matched hash '$hash'. Falling back to normal execution", ex)
+              self ! InitialStartCall(callKey, CallActor.Start)
+          }
+        } recover { case e =>
+          log.error(s"Failed to calculate hash for call '${backendCall.key.tag}'.", e)
+          scheduleTransition(WorkflowFailed)
         }
-      } recover { case e =>
-        log.error(s"Failed to calculate hash for call '${backendCall.key.tag}'.", e)
-        scheduleTransition(WorkflowFailed)
+      } else {
+        log.info(s"Call caching 'readFromCache' is turned off, starting call")
+        self ! InitialStartCall(callKey, CallActor.Start)
       }
-    } else {
-      log.info(s"Call caching 'readFromCache' is turned off, starting call")
-      self ! InitialStartCall(callKey, CallActor.Start)
+    }
+
+    Try(backendCall.runtimeAttributes) map { _ => startCall } recover {
+      case f =>
+        log.error(f.getMessage)
+        scheduleTransition(WorkflowFailed)
     }
   }
 
diff --git a/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
new file mode 100644
index 00000000000..513666feb5e
--- /dev/null
+++ b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
@@ -0,0 +1,67 @@
+package cromwell
+
+import akka.event.LoggingAdapter
+import akka.testkit.{EventFilter, TestActorRef}
+import cromwell.engine._
+import cromwell.engine.backend.BackendType
+import cromwell.engine.backend.jes.{JesBackend, JesBackendCall}
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
+import cromwell.engine.workflow.{CallKey, WorkflowManagerActor}
+import cromwell.logging.WorkflowLogger
+import cromwell.util.SampleWdl
+import org.slf4j.LoggerFactory
+import org.specs2.mock.Mockito
+import wdl4s.{RuntimeAttributes, ThrowableWithErrors, CallInputs}
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Try, Success}
+
+class InvalidRuntimeAttributesSpec extends CromwellTestkitSpec with Mockito {
+
+  "A workflow with a task with invalid runtime attributes" should {
+    "fail on JES Backend" in {
+      val wd = mock[WorkflowDescriptor]
+      val logger = LoggerFactory.getLogger("Fake logger")
+      wd.workflowLogger returns logger
+      wd.shortId returns "fakeShortID"
+
+      val jesBackendCall = mock[JesBackendCall]
+      def exception = {
+        Try(CromwellRuntimeAttributes(RuntimeAttributes(Map.empty[String, Seq[String]]), BackendType.JES)).failed.get
+      }
+
+      jesBackendCall.runtimeAttributes throws exception
+      jesBackendCall.workflowLoggerWithCall(any[String], any[Option[LoggingAdapter]]) answers { (params, mock) =>
+        val array: Array[Object] = params.asInstanceOf[Array[Object]]
+        WorkflowLogger("WorkflowActor", wd, akkaLogger = array(1).asInstanceOf[Option[LoggingAdapter]])
+      }
+
+      val mockJes = mock[JesBackend]
+      mockJes.initializeForWorkflow(any[WorkflowDescriptor]) returns Success(Map.empty)
+      mockJes.cleanUpForWorkflow(any[WorkflowDescriptor])(any[ExecutionContext]) returns Future.successful(())
+      mockJes.bindCall(any[WorkflowDescriptor], any[CallKey], any[CallInputs],any[AbortRegistrationFunction]) returns jesBackendCall
+
+      val workflowSources = WorkflowSourceFiles(SampleWdl.HelloWorld.wdlSource(), SampleWdl.HelloWorld.wdlJson, "{}")
+      val submitMessage = WorkflowManagerActor.SubmitWorkflow(workflowSources)
+      runWdlWithWorkflowManagerActor(
+        wma = TestActorRef(new WorkflowManagerActor(mockJes)),
+        submitMsg = submitMessage,
+        stdout = Map.empty,
+        stderr = Map.empty,
+        eventFilter = EventFilter.error(pattern = "RuntimeAttribute is not valid.", occurrences = 1),
+        terminalState = WorkflowFailed
+      )
+    }
+
+    "fail on Local Backend" in {
+      runWdl(
+        sampleWdl = SampleWdl.HelloWorld,
+        runtime =
+          """ runtime { wrongAttribute: "nop" }""".stripMargin,
+        eventFilter = EventFilter.error(pattern = ".*RuntimeAttribute is not valid.*", occurrences = 1),
+        terminalState = WorkflowFailed
+      )
+    }
+  }
+
+}

From c54b3732bd0ac990ae8dc5c17e7e6a43a3df0115 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Thu, 14 Jan 2016 17:19:15 -0500
Subject: [PATCH 15/31] Prevent infinite retries

---
 .../cromwell/engine/CallExecutionActor.scala  |  2 +-
 .../cromwell/engine/WorkflowDescriptor.scala  |  2 +-
 .../engine/backend/jes/JesBackend.scala       | 10 ++-
 .../engine/backend/jes/JesBackendCall.scala   |  4 ++
 src/main/scala/cromwell/engine/package.scala  |  2 +
 src/main/scala/cromwell/util/TryUtil.scala    | 12 ++--
 .../engine/backend/jes/JesBackendSpec.scala   | 26 ++++++--
 .../scala/cromwell/util/TryUtilSpec.scala     | 62 ++++++++++++-------
 8 files changed, 85 insertions(+), 35 deletions(-)

diff --git a/src/main/scala/cromwell/engine/CallExecutionActor.scala b/src/main/scala/cromwell/engine/CallExecutionActor.scala
index 770ef63865e..e53c45cf39c 100644
--- a/src/main/scala/cromwell/engine/CallExecutionActor.scala
+++ b/src/main/scala/cromwell/engine/CallExecutionActor.scala
@@ -75,7 +75,7 @@ class CallExecutionActor(backendCall: BackendCall) extends Actor with CromwellAc
   def withRetry(work: Future[ExecutionHandle], onSuccess: ExecutionHandle => Unit, onFailure: => Unit): Unit = {
     work onComplete {
       case Success(s) => onSuccess(s)
-      case Failure(e: GoogleJsonResponseException) if e.getStatusCode == 403 =>
+      case Failure(e: CromwellFatalException) =>
         logger.error(e.getMessage, e)
         self ! Finish(FailedExecutionHandle(e))
       case Failure(e: Exception) =>
diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index 0b74a5d1f85..278204e54bc 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -123,7 +123,7 @@ case class WorkflowDescriptor(id: WorkflowId,
         maxPollingInterval = 10 seconds,
         logger = logger,
         failMessage = Option(s"Failed to copy file $src to $dest"),
-        fatalExceptions = Seq(classOf[FileAlreadyExistsException])
+        isFatal = (t: Throwable) => t.isInstanceOf[FileAlreadyExistsException]
       ) recover {
         case _: FileAlreadyExistsException => logger.info(s"Tried to copy the same file multiple times. Skipping subsequent copies for $src")
       }
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index a6b5349c4d1..20c6393be34 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -5,6 +5,8 @@ import java.net.SocketTimeoutException
 import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorSystem
+import com.google.api.client.googleapis.json.GoogleJsonResponseException
+import com.google.api.client.http.HttpResponseException
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.engine.ExecutionIndex.{ExecutionIndex, IndexEnhancedInt}
@@ -110,6 +112,11 @@ object JesBackend {
     }
   }
 
+  def isFatalJesException(t: Throwable): Boolean = t match {
+    case e: HttpResponseException if e.getStatusCode == 403 => true
+    case _ => false
+  }
+
   protected def withRetry[T](f: Option[T] => T, logger: WorkflowLogger, failureMessage: String) = TryUtil.retryBlock(
     fn = f,
     retryLimit = Option(10),
@@ -117,7 +124,8 @@ object JesBackend {
     pollingBackOffFactor = 1,
     maxPollingInterval = 10 seconds,
     logger = logger,
-    failMessage = Option(failureMessage)
+    failMessage = Option(failureMessage),
+    isFatal = isFatalJesException
   )
 
   sealed trait JesParameter {
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
index 2177912a84c..64ea4e7bcf3 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
@@ -2,6 +2,7 @@ package cromwell.engine.backend.jes
 
 import java.nio.file.Paths
 
+import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.typesafe.scalalogging.LazyLogging
 import wdl4s._
 import wdl4s.values.WdlFile
@@ -89,6 +90,9 @@ class JesBackendCall(val backend: JesBackend,
         status match {
           case Success(s: TerminalRunStatus) => backend.executionResult(s, handle)
           case Success(s) => handle.copy(previousStatus = Option(s)).future // Copy the current handle with updated previous status.
+          case Failure(e: GoogleJsonResponseException) if e.getStatusCode == 404 =>
+            logger.error(s"JES Job ID ${handle.run.runId} has not been found, failing call")
+            FailedExecutionHandle(e).future
           case Failure(e: Exception) =>
             // Log exceptions and return the original handle to try again.
             logger.warn("Caught exception, retrying: " + e.getMessage, e)
diff --git a/src/main/scala/cromwell/engine/package.scala b/src/main/scala/cromwell/engine/package.scala
index bf532699783..0a42fdbc830 100644
--- a/src/main/scala/cromwell/engine/package.scala
+++ b/src/main/scala/cromwell/engine/package.scala
@@ -43,6 +43,8 @@ package object engine {
   type CallOutputs = Map[LocallyQualifiedName, CallOutput]
   type HostInputs = Map[String, WdlValue]
 
+  class CromwellFatalException(exception: Throwable) extends Exception(exception)
+
   implicit class EnhancedFullyQualifiedName(val fqn: FullyQualifiedName) extends AnyVal {
     def isScatter = fqn.contains(Scatter.FQNIdentifier)
     def scopeAndVariableName: (String, String) = {
diff --git a/src/main/scala/cromwell/util/TryUtil.scala b/src/main/scala/cromwell/util/TryUtil.scala
index 554be3281c6..7738ca62ac9 100644
--- a/src/main/scala/cromwell/util/TryUtil.scala
+++ b/src/main/scala/cromwell/util/TryUtil.scala
@@ -2,9 +2,10 @@ package cromwell.util
 
 import java.io.{PrintWriter, StringWriter}
 
+import cromwell.engine.CromwellFatalException
 import cromwell.logging.WorkflowLogger
 
-import scala.concurrent.duration.{Duration, _}
+import scala.concurrent.duration.Duration
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -28,6 +29,7 @@ object TryUtil {
     possibleFailures.collect { case failure: Failure[T] => stringifyFailure(failure) }
 
   private def defaultSuccessFunction(a: Any): Boolean = true
+  private def defaultIsFatal(t: Throwable): Boolean = false
 
   /**
    * Runs a block of code (`fn`) `retries` number of times until it succeeds.
@@ -54,7 +56,7 @@ object TryUtil {
                     logger: WorkflowLogger,
                     failMessage: Option[String] = None,
                     priorValue: Option[T] = None,
-                    fatalExceptions: Seq[Class[_ <: Throwable]] = Seq.empty): Try[T] = {
+                    isFatal: (Throwable) => Boolean = defaultIsFatal _): Try[T] = {
 
     def logFailures(attempt: Try[T]): Unit = {
       attempt recover {
@@ -64,7 +66,7 @@ object TryUtil {
 
     Try { fn(priorValue) } match {
       case Success(x) if isSuccess(x) => Success(x)
-      case Failure(f) if fatalExceptions.contains(f.getClass) => Failure(f)
+      case Failure(f) if isFatal(f) => Failure(new CromwellFatalException(f))
       case value if (retryLimit.isDefined && retryLimit.get > 1) || retryLimit.isEmpty =>
         logFailures(value)
         val retryCountMessage = if (retryLimit.getOrElse(0) > 0) s" (${retryLimit.getOrElse(0) - 1} more retries) " else ""
@@ -86,7 +88,9 @@ object TryUtil {
         )
       case f =>
         logFailures(f)
-        f
+        f recoverWith {
+          case e => Failure(new CromwellFatalException(e))
+        }
     }
   }
 
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index 8af06064d33..b2b8c82b27e 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -3,22 +3,23 @@ package cromwell.engine.backend.jes
 import java.net.URL
 import java.nio.file.Paths
 
+import com.google.api.client.testing.http.{HttpTesting, MockHttpTransport, MockLowLevelHttpRequest, MockLowLevelHttpResponse}
 import cromwell.CromwellTestkitSpec
-import cromwell.engine.backend.{BackendType, Backend}
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s.{RuntimeAttributes, CallInputs}
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString}
 import cromwell.engine.WorkflowDescriptor
+import cromwell.engine.backend.BackendType
 import cromwell.engine.backend.jes.JesBackend.{JesInput, JesOutput}
 import cromwell.engine.backend.jes.authentication._
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
 import cromwell.util.EncryptionSpec
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 import org.specs2.mock.Mockito
+import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
+import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString}
+import wdl4s.{CallInputs, RuntimeAttributes}
 
-import scala.util.Success
+import scala.util.{Success, Try}
 
 class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndAfterAll {
   val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()
@@ -43,6 +44,19 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
     override lazy val googleConf = GoogleConfiguration("appName", ServiceAccountMode("accountID", "p12"), Option(Refresh(clientSecrets)))
   }
 
+  it should "consider 403 as a fatal exception" in {
+    val transport = new MockHttpTransport() {
+      override def buildRequest(method: String, url: String) = {
+        new MockLowLevelHttpRequest() {
+          override def execute() = new MockLowLevelHttpResponse().setStatusCode(403)
+        }
+      }
+    }
+    val request = transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL)
+    val mockedResponse = Try(request.execute()).failed.get
+    JesBackend.isFatalJesException(mockedResponse) shouldBe true
+  }
+
   "adjustInputPaths" should "map GCS paths and *only* GCS paths to local" in {
     val ignoredCall = mock[CallKey]
     val stringKey = "abc"
diff --git a/src/test/scala/cromwell/util/TryUtilSpec.scala b/src/test/scala/cromwell/util/TryUtilSpec.scala
index 29e1c1ebe26..8f5a8e16564 100644
--- a/src/test/scala/cromwell/util/TryUtilSpec.scala
+++ b/src/test/scala/cromwell/util/TryUtilSpec.scala
@@ -2,41 +2,39 @@ package cromwell.util
 
 import java.util.UUID
 
-import cromwell.engine.{WorkflowId, WorkflowSourceFiles, WorkflowDescriptor}
+import cromwell.engine.{CromwellFatalException, WorkflowId, WorkflowSourceFiles, WorkflowDescriptor}
 import cromwell.logging.WorkflowLogger
+import org.scalatest.mock.MockitoSugar
 import org.scalatest.{FlatSpec, Matchers}
 
 import scala.language.postfixOps
 import scala.concurrent.duration._
 import scala.util.Success
 
-class TryUtilSpec extends FlatSpec with Matchers {
-  def failNTimes(n: Int): Option[Int] => Int = {
-    var counter = n
-    def func(prior: Option[Int]): Int = {
-      if (counter > 0) {
-        counter -= 1
-        throw new Exception("Failed")
+class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {
+
+  class MockWork {
+    var counter: Int = _
+
+    def failNTimes(n: Int): Option[Int] => Int = {
+      counter = n
+      def func(prior: Option[Int]): Int = {
+        if (counter > 0) {
+          counter -= 1
+          throw new IllegalArgumentException("Failed")
+        }
+        9
       }
-      9
+      func
     }
-    func
   }
 
-  val descriptor = WorkflowDescriptor(
-    WorkflowId(UUID.randomUUID()),
-    WorkflowSourceFiles(
-      "workflow w {}",
-      "{}",
-      "{}"
-    )
-  )
-
-  val logger = WorkflowLogger("Test", descriptor)
+  val logger = mock[WorkflowLogger]
 
   it should "Retry a function until it works" in {
+    val work = new MockWork
     val value = TryUtil.retryBlock(
-      fn = failNTimes(4),
+      fn = work.failNTimes(4),
       retryLimit = Some(5),
       pollingInterval = 50 milliseconds,
       pollingBackOffFactor = 1,
@@ -45,11 +43,13 @@ class TryUtilSpec extends FlatSpec with Matchers {
       failMessage = Some(s"failed attempt (on purpose)")
     )
     value shouldEqual Success(9)
+    work.counter shouldBe 0
   }
 
   it should "Fail if it hits the max retry count" in {
+    val work = new MockWork
     val value = TryUtil.retryBlock(
-      fn = failNTimes(4),
+      fn = work.failNTimes(4),
       retryLimit = Some(4),
       pollingInterval = 50 milliseconds,
       pollingBackOffFactor = 1,
@@ -58,5 +58,23 @@ class TryUtilSpec extends FlatSpec with Matchers {
       failMessage = Some(s"failed attempt (on purpose)")
     )
     value.isFailure shouldEqual true
+    value.failed.get.isInstanceOf[CromwellFatalException] shouldBe true
+  }
+
+  it should "Fail if it hits a fatal exception" in {
+    val work = new MockWork
+    val value = TryUtil.retryBlock(
+      fn = work.failNTimes(4),
+      retryLimit = Some(4),
+      pollingInterval = 50 milliseconds,
+      pollingBackOffFactor = 1,
+      maxPollingInterval = 10 seconds,
+      logger = logger,
+      failMessage = Some(s"failed attempt (on purpose)"),
+      isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException]
+    )
+    value.isFailure shouldEqual true
+    value.failed.get.isInstanceOf[CromwellFatalException] shouldBe true
+    work.counter shouldBe 3
   }
 }

From 8b5afb2a492f85dbe4de8ab989b160f1e083b16f Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Fri, 15 Jan 2016 17:34:21 -0500
Subject: [PATCH 16/31] Cleanup mocking rampage

---
 .../InvalidRuntimeAttributesSpec.scala        | 63 +++++++++----------
 .../engine/backend/jes/JesBackendSpec.scala   |  2 +-
 2 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
index 513666feb5e..23cc765278b 100644
--- a/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
+++ b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
@@ -1,50 +1,47 @@
 package cromwell
 
-import akka.event.LoggingAdapter
+import java.net.URL
+
 import akka.testkit.{EventFilter, TestActorRef}
+import cromwell.engine.ExecutionIndex._
 import cromwell.engine._
-import cromwell.engine.backend.BackendType
-import cromwell.engine.backend.jes.{JesBackend, JesBackendCall}
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import cromwell.engine.workflow.{CallKey, WorkflowManagerActor}
-import cromwell.logging.WorkflowLogger
+import cromwell.engine.backend.jes.{JesAttributes, JesBackend}
+import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
+import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.util.SampleWdl
-import org.slf4j.LoggerFactory
-import org.specs2.mock.Mockito
-import wdl4s.{RuntimeAttributes, ThrowableWithErrors, CallInputs}
+import org.scalatest.BeforeAndAfterAll
+
+class InvalidRuntimeAttributesSpec extends CromwellTestkitSpec with BeforeAndAfterAll {
 
-import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Try, Success}
+  val testWorkflowManagerSystem = new CromwellTestkitSpec.TestWorkflowManagerSystem()
+  val actorSystem = testWorkflowManagerSystem.actorSystem
 
-class InvalidRuntimeAttributesSpec extends CromwellTestkitSpec with Mockito {
+  override protected def afterAll() = {
+    testWorkflowManagerSystem.shutdownTestActorSystem()
+    super.afterAll()
+  }
 
   "A workflow with a task with invalid runtime attributes" should {
     "fail on JES Backend" in {
-      val wd = mock[WorkflowDescriptor]
-      val logger = LoggerFactory.getLogger("Fake logger")
-      wd.workflowLogger returns logger
-      wd.shortId returns "fakeShortID"
-
-      val jesBackendCall = mock[JesBackendCall]
-      def exception = {
-        Try(CromwellRuntimeAttributes(RuntimeAttributes(Map.empty[String, Seq[String]]), BackendType.JES)).failed.get
+      val jesBackend = new JesBackend(actorSystem) {
+        private val anyString = ""
+        private val anyURL: URL = null
+        override lazy val jesConf = new JesAttributes(
+          project = anyString,
+          executionBucket = anyString,
+          endpointUrl = anyURL) {
+        }
+        override def callGcsPath(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): String = "gs://fake/path"
+        override def jesUserConnection(workflow: WorkflowDescriptor) = null
+        override lazy val jesCromwellInterface = null
+        override lazy val googleConf = GoogleConfiguration("appName", ServiceAccountMode("accountID", "pem"), None)
       }
 
-      jesBackendCall.runtimeAttributes throws exception
-      jesBackendCall.workflowLoggerWithCall(any[String], any[Option[LoggingAdapter]]) answers { (params, mock) =>
-        val array: Array[Object] = params.asInstanceOf[Array[Object]]
-        WorkflowLogger("WorkflowActor", wd, akkaLogger = array(1).asInstanceOf[Option[LoggingAdapter]])
-      }
-
-      val mockJes = mock[JesBackend]
-      mockJes.initializeForWorkflow(any[WorkflowDescriptor]) returns Success(Map.empty)
-      mockJes.cleanUpForWorkflow(any[WorkflowDescriptor])(any[ExecutionContext]) returns Future.successful(())
-      mockJes.bindCall(any[WorkflowDescriptor], any[CallKey], any[CallInputs],any[AbortRegistrationFunction]) returns jesBackendCall
-
       val workflowSources = WorkflowSourceFiles(SampleWdl.HelloWorld.wdlSource(), SampleWdl.HelloWorld.wdlJson, "{}")
       val submitMessage = WorkflowManagerActor.SubmitWorkflow(workflowSources)
+
       runWdlWithWorkflowManagerActor(
-        wma = TestActorRef(new WorkflowManagerActor(mockJes)),
+        wma = TestActorRef(new WorkflowManagerActor(jesBackend)),
         submitMsg = submitMessage,
         stdout = Map.empty,
         stderr = Map.empty,
@@ -58,7 +55,7 @@ class InvalidRuntimeAttributesSpec extends CromwellTestkitSpec with Mockito {
         sampleWdl = SampleWdl.HelloWorld,
         runtime =
           """ runtime { wrongAttribute: "nop" }""".stripMargin,
-        eventFilter = EventFilter.error(pattern = ".*RuntimeAttribute is not valid.*", occurrences = 1),
+        eventFilter = EventFilter.error(pattern = "RuntimeAttribute is not valid", occurrences = 1),
         terminalState = WorkflowFailed
       )
     }
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index b2b8c82b27e..1cef8cd918f 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -41,7 +41,7 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
     }
     override def jesUserConnection(workflow: WorkflowDescriptor) = null
     override lazy val jesCromwellInterface = null
-    override lazy val googleConf = GoogleConfiguration("appName", ServiceAccountMode("accountID", "p12"), Option(Refresh(clientSecrets)))
+    override lazy val googleConf = GoogleConfiguration("appName", ServiceAccountMode("accountID", "pem"), Option(Refresh(clientSecrets)))
   }
 
   it should "consider 403 as a fatal exception" in {

From e3aa4a3e7fb8fe2913859358f173e4bf3f723832 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Thu, 14 Jan 2016 16:14:43 -0500
Subject: [PATCH 17/31] No more chilling-out

---
 src/main/scala/cromwell/engine/backend/jes/Run.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/cromwell/engine/backend/jes/Run.scala b/src/main/scala/cromwell/engine/backend/jes/Run.scala
index 19b45070222..2efd688e270 100644
--- a/src/main/scala/cromwell/engine/backend/jes/Run.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/Run.scala
@@ -102,7 +102,7 @@ object Run  {
   private case class EventStartTime(name: String, timestamp: DateTime)
 
   def getEventList(op: Operation): Seq[ExecutionEventEntry] = {
-    val starterEvents = eventIfExists("createTime", op, "chilling out, maxing") ++ eventIfExists("startTime", op, "relaxing all cool")
+    val starterEvents = eventIfExists("createTime", op, "waiting for quota") ++ eventIfExists("startTime", op, "initializing VM")
 
     val eventsList: Seq[EventStartTime] = if (op.getMetadata.containsKey("events")) {
       op.getMetadata.get("events").asInstanceOf[java.util.ArrayList[AnyRef]].asScala map { x =>
@@ -159,7 +159,7 @@ case class Run(runId: String, pipeline: Pipeline, logger: WorkflowLogger) {
       val newBackendInfo = JesCallBackendInfo(Option(JesId(runId)), Option(JesStatus(currentStatus.toString)))
       globalDataAccess.updateExecutionBackendInfo(workflowId, CallKey(call, pipeline.key.index), newBackendInfo)
 
-      // If this has transitioned to a running or complete state from a state this is not running or complete,
+      // If this has transitioned to a running or complete state from a state that is not running or complete,
       // register the abort function.
       if (currentStatus.isRunningOrComplete && (previousStatus.isEmpty || !previousStatus.get.isRunningOrComplete)) {
         backendCall.callAbortRegistrationFunction.register(AbortFunction(() => abort()))

From a03a32451f49177b783d5c365a91265b4097e13e Mon Sep 17 00:00:00 2001
From: Scott Frazer <scott.d.frazer@gmail.com>
Date: Tue, 12 Jan 2016 15:19:28 -0500
Subject: [PATCH 18/31] Leaving the redirection of stdout/stderr to JES

---
 build.sbt                                     |  2 +-
 .../scala/cromwell/engine/CallActor.scala     |  2 +-
 .../cromwell/engine/backend/Backend.scala     | 16 +++---
 .../cromwell/engine/backend/BackendCall.scala | 35 ++++++------
 .../engine/backend/jes/JesBackend.scala       | 40 +++++++-------
 .../engine/backend/jes/JesBackendCall.scala   | 55 ++++++++++---------
 .../engine/backend/jes/JesConfiguration.scala |  3 -
 .../cromwell/engine/backend/jes/Run.scala     |  4 +-
 .../engine/backend/local/LocalBackend.scala   |  9 +--
 .../backend/local/LocalBackendCall.scala      |  4 +-
 .../backend/local/SharedFileSystem.scala      | 28 ++++++----
 .../engine/backend/sge/SgeBackend.scala       |  9 +--
 .../engine/backend/sge/SgeBackendCall.scala   |  4 +-
 .../engine/workflow/WorkflowActor.scala       |  4 +-
 .../workflow/WorkflowManagerActor.scala       | 17 ++++--
 .../InvalidRuntimeAttributesSpec.scala        |  6 +-
 .../engine/WorkflowManagerActorSpec.scala     |  2 +-
 .../engine/backend/BackendCallSpec.scala      |  4 +-
 .../engine/backend/jes/JesBackendSpec.scala   | 45 +++++++++++----
 .../backend/local/LocalBackendSpec.scala      |  2 +-
 .../engine/db/slick/SlickDataAccessSpec.scala | 40 +++++++-------
 .../cromwell/logging/WorkflowLoggerSpec.scala |  2 +-
 src/test/scala/cromwell/util/SampleWdl.scala  |  9 ++-
 23 files changed, 193 insertions(+), 149 deletions(-)

diff --git a/build.sbt b/build.sbt
index 1b28cc2469b..8853e016660 100644
--- a/build.sbt
+++ b/build.sbt
@@ -11,7 +11,7 @@ scalaVersion := "2.11.7"
 
 val lenthallV = "0.15"
 
-val wdl4sV = "0.2-d949ac2-SNAPSHOT"
+val wdl4sV = "0.2-6f2ba49-SNAPSHOT"
 
 val sprayV = "1.3.2"
 
diff --git a/src/main/scala/cromwell/engine/CallActor.scala b/src/main/scala/cromwell/engine/CallActor.scala
index d9ffb6c008c..8f175f1ae48 100644
--- a/src/main/scala/cromwell/engine/CallActor.scala
+++ b/src/main/scala/cromwell/engine/CallActor.scala
@@ -111,7 +111,7 @@ class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backe
       // There's no special Retry/Ack handling required for CallStarted message, the WorkflowActor can always
       // handle those immediately.
       context.parent ! WorkflowActor.CallStarted(key)
-      val backendCall = backend.bindCall(workflowDescriptor, key, locallyQualifiedInputs, AbortRegistrationFunction(registerAbortFunction))
+      val backendCall = backend.bindCall(workflowDescriptor, key, locallyQualifiedInputs, Option(AbortRegistrationFunction(registerAbortFunction)))
       val executionActorName = s"CallExecutionActor-${workflowDescriptor.id}-${call.unqualifiedName}"
       context.actorOf(CallExecutionActor.props(backendCall), executionActorName) ! startMode.executionMessage
       goto(CallRunningAbortUnavailable)
diff --git a/src/main/scala/cromwell/engine/backend/Backend.scala b/src/main/scala/cromwell/engine/backend/Backend.scala
index bfe525613de..098fb8e03a9 100644
--- a/src/main/scala/cromwell/engine/backend/Backend.scala
+++ b/src/main/scala/cromwell/engine/backend/Backend.scala
@@ -2,24 +2,22 @@ package cromwell.engine.backend
 
 import akka.actor.ActorSystem
 import com.typesafe.config.Config
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s._
-import cromwell.engine.ExecutionIndex.ExecutionIndex
 import cromwell.engine._
 import cromwell.engine.backend.jes.JesBackend
 import cromwell.engine.backend.local.LocalBackend
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import cromwell.engine.backend.sge.SgeBackend
 import cromwell.engine.db.ExecutionDatabaseKey
 import cromwell.engine.io.IoInterface
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.engine.{HostInputs, CallOutputs}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.docker.SprayDockerRegistryApiClient
 import org.slf4j.LoggerFactory
+import wdl4s._
 import wdl4s.values.WdlValue
-import scala.language.postfixOps
 
 import scala.concurrent.{ExecutionContext, Future}
+import scala.language.postfixOps
 import scala.util.{Success, Try}
 
 object Backend {
@@ -113,8 +111,8 @@ trait Backend {
    */
   def bindCall(workflowDescriptor: WorkflowDescriptor,
                key: CallKey,
-               locallyQualifiedInputs: CallInputs,
-               abortRegistrationFunction: AbortRegistrationFunction): BackendCall
+               locallyQualifiedInputs: CallInputs = Map.empty[String, WdlValue],
+               abortRegistrationFunction: Option[AbortRegistrationFunction] = None): BackendCall
 
   def workflowContext(workflowOptions: WorkflowOptions, workflowId: WorkflowId, name: String): WorkflowContext
 
@@ -126,9 +124,9 @@ trait Backend {
   def prepareForRestart(restartableWorkflow: WorkflowDescriptor)(implicit ec: ExecutionContext): Future[Unit]
 
   /**
-   * Return CallStandardOutput which contains the stdout/stderr of the particular call
+   * Return CallLogs which contains the stdout/stderr of the particular call
    */
-  def stdoutStderr(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): CallLogs
+  def stdoutStderr(backendCall: BackendCall): CallLogs
 
   def backendType: BackendType
 
diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index e79f72fd460..0b4c0d7f98d 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -1,16 +1,14 @@
 package cromwell.engine.backend
 
 import akka.event.LoggingAdapter
-import cromwell.engine
-import cromwell.engine.backend.runtimeattributes.{ContinueOnReturnCodeSet, ContinueOnReturnCodeFlag, CromwellRuntimeAttributes}
+import cromwell.engine.Hashing._
+import cromwell.engine.backend.runtimeattributes.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, CromwellRuntimeAttributes}
+import cromwell.engine.workflow.CallKey
+import cromwell.engine.{CallOutputs, ExecutionEventEntry, ExecutionHash, WorkflowDescriptor}
+import cromwell.logging.WorkflowLogger
 import wdl4s._
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.values.WdlValue
-import cromwell.engine.workflow.CallKey
-import cromwell.engine.{ExecutionEventEntry, ExecutionHash, WorkflowDescriptor}
-import cromwell.engine.CallOutputs
-import cromwell.logging.WorkflowLogger
-import cromwell.engine.Hashing._
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -90,15 +88,15 @@ trait BackendCall {
   def backend: Backend
 
   /**
-   * Inputs to the call.  For example, if a call's task specifies a command like this:
-   *
-   * command {
-   *   File some_dir
-   *   ls ${some_dir}
-   * }
-   *
-   * Then locallyQualifiedInputs could be Map("some_dir" -> WdlFile("/some/path"))
-   */
+    * Inputs to the call.  For example, if a call's task specifies a command like this:
+    *
+    * File some_dir
+    * command {
+    *   ls ${some_dir}
+    * }
+    *
+    * Then locallyQualifiedInputs could be Map("some_dir" -> WdlFile("/some/path"))
+    */
   def locallyQualifiedInputs: CallInputs
 
   /**
@@ -131,6 +129,11 @@ trait BackendCall {
 
   def useCachedCall(cachedBackendCall: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] = ???
 
+  /**
+    * Return CallLogs which contains the stdout/stderr of the particular call
+    */
+  def stdoutStderr: CallLogs
+
   @throws[IllegalArgumentException]
   lazy val runtimeAttributes = CromwellRuntimeAttributes(call.task.runtimeAttributes, backend.backendType)
 
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index 20c6393be34..62ac5af00c0 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -9,10 +9,10 @@ import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.api.client.http.HttpResponseException
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
-import cromwell.engine.ExecutionIndex.{ExecutionIndex, IndexEnhancedInt}
+import cromwell.engine.ExecutionIndex.IndexEnhancedInt
 import cromwell.engine.ExecutionStatus.ExecutionStatus
 import cromwell.engine.Hashing._
-import cromwell.engine.backend.{BackendType, _}
+import cromwell.engine.backend._
 import cromwell.engine.backend.jes.JesBackend._
 import cromwell.engine.backend.jes.Run.RunStatus
 import cromwell.engine.backend.jes.authentication._
@@ -26,9 +26,9 @@ import cromwell.engine.workflow.{CallKey, WorkflowOptions}
 import cromwell.engine.{AbortRegistrationFunction, CallOutput, CallOutputs, HostInputs, _}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.{AggregatedException, TryUtil}
-import wdl4s.{CallInputs, _}
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
+import wdl4s.{Call, CallInputs, Scatter, UnsatisfiedInputsException, _}
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
@@ -71,7 +71,7 @@ object JesBackend {
   // Decoration around WorkflowDescriptor to generate bucket names and the like
   implicit class JesWorkflowDescriptor(val descriptor: WorkflowDescriptor)
     extends JesBackend(CromwellBackend.backend().actorSystem) {
-    def callDir(key: CallKey) = callGcsPath(descriptor, key.scope.unqualifiedName, key.index)
+    def callDir(key: CallKey) = callGcsPath(descriptor, key)
   }
 
   /**
@@ -146,6 +146,17 @@ object JesBackend {
     def isScatter: Boolean = execution.callFqn.contains(Scatter.FQNIdentifier)
     def executionStatus: ExecutionStatus = ExecutionStatus.withName(execution.status)
   }
+
+  def callGcsPath(descriptor: WorkflowDescriptor, callKey: CallKey): String = {
+    val shardPath = callKey.index map { i => s"/shard-$i" } getOrElse ""
+    val workflowPath = workflowGcsPath(descriptor)
+    s"$workflowPath/call-${callKey.scope.unqualifiedName}$shardPath"
+  }
+
+  def workflowGcsPath(descriptor: WorkflowDescriptor): String = {
+    val bucket = descriptor.workflowOptions.getOrElse(GcsRootOptionKey, ProductionJesConfiguration.jesConf.executionBucket).stripSuffix("/")
+    s"$bucket/${descriptor.namespace.workflow.unqualifiedName}/${descriptor.id}"
+  }
 }
 
 final case class JesJobKey(operationId: String) extends JobKey
@@ -277,14 +288,12 @@ case class JesBackend(actorSystem: ActorSystem)
       }
   }
 
-  override def stdoutStderr(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): CallLogs = {
-    JesBackendCall.stdoutStderr(callGcsPath(descriptor, callName, index))
-  }
+  override def stdoutStderr(backendCall: BackendCall): CallLogs = backendCall.stdoutStderr
 
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
                         key: CallKey,
                         locallyQualifiedInputs: CallInputs,
-                        abortRegistrationFunction: AbortRegistrationFunction): BackendCall = {
+                        abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     new JesBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
   }
 
@@ -409,7 +418,7 @@ case class JesBackend(actorSystem: ActorSystem)
          |cd $JesCromwellRoot
          |$monitoring
          |$command
-         |echo $$? > ${JesBackendCall.RcFilename}
+         |echo $$? > ${JesBackendCall.jesReturnCodeFilename(backendCall.key)}
        """.stripMargin.trim
 
     def attemptToUploadObject(priorAttempt: Option[Unit]) = authenticateAsUser(backendCall.workflowDescriptor) { _.uploadObject(backendCall.gcsExecPath, fileContent) }
@@ -524,7 +533,7 @@ case class JesBackend(actorSystem: ActorSystem)
     try {
       val backendCall = handle.backendCall
       val outputMappings = postProcess(backendCall)
-      lazy val stderrLength: BigInteger = authenticateAsUser(backendCall.workflowDescriptor) { _.objectSize(GcsPath(backendCall.stderrJesOutput.gcs)) }
+      lazy val stderrLength: BigInteger = authenticateAsUser(backendCall.workflowDescriptor) { _.objectSize(GcsPath(backendCall.jesStderrGcsPath)) }
       lazy val returnCodeContents = backendCall.downloadRcFile
       lazy val returnCode = returnCodeContents map { _.trim.toInt }
       lazy val continueOnReturnCode = backendCall.runtimeAttributes.continueOnReturnCode
@@ -669,17 +678,6 @@ case class JesBackend(actorSystem: ActorSystem)
     s"$bucket/${descriptor.id}_auth.json"
   }
 
-  def callGcsPath(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): String = {
-    val shardPath = index map { i => s"/shard-$i"} getOrElse ""
-    val workflowPath = workflowGcsPath(descriptor)
-    s"$workflowPath/call-$callName$shardPath"
-  }
-
-  def workflowGcsPath(descriptor: WorkflowDescriptor): String = {
-    val bucket = descriptor.workflowOptions.getOrElse(GcsRootOptionKey, jesConf.executionBucket).stripSuffix("/")
-    s"$bucket/${descriptor.namespace.workflow.unqualifiedName}/${descriptor.id}"
-  }
-
   def googleProject(descriptor: WorkflowDescriptor): String = {
     descriptor.workflowOptions.getOrElse(GoogleProjectOptionKey, jesConf.project)
   }
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
index 64ea4e7bcf3..18fe6686cee 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
@@ -20,26 +20,15 @@ import scala.util.{Failure, Success, Try}
 import Hashing._
 
 object JesBackendCall {
-
-  def stdoutStderr(callGcsPath: String): CallLogs = {
-    CallLogs(
-      stdout = WdlFile(s"$callGcsPath/$StdoutFilename"),
-      stderr = WdlFile(s"$callGcsPath/$StderrFilename"),
-      Option(Map("log" -> WdlFile(s"$callGcsPath/$JesLog"),
-        "stdout" -> WdlFile(s"$callGcsPath/$JesStdout"),
-        "stderr" -> WdlFile(s"$callGcsPath/$JesStderr")
-      ))
-    )
+  def jesLogBasename(key: CallKey) = {
+    val index = key.index.map(s => s"-$s").getOrElse("")
+    s"${key.scope.unqualifiedName}$index"
   }
 
-  val StdoutFilename = "job.stdout.txt"
-  val StderrFilename = "job.stderr.txt"
-  val RcFilename = "job.rc.txt"
-
-  val JesLogBasename = "jes"
-  private [jes] val JesLog = s"$JesLogBasename.log"
-  private val JesStdout = s"$JesLogBasename-stdout.log"
-  private val JesStderr = s"$JesLogBasename-stderr.log"
+  def jesLogFilename(key: CallKey) = s"${jesLogBasename(key)}.log"
+  def jesLogStdoutFilename(key: CallKey) = s"${jesLogBasename(key)}-stdout.log"
+  def jesLogStderrFilename(key: CallKey) = s"${jesLogBasename(key)}-stderr.log"
+  def jesReturnCodeFilename(key: CallKey) = s"${jesLogBasename(key)}-rc.txt"
 
   private def jesOutput(callGcsPath: String, filename: String): JesOutput =
     JesOutput(filename, s"$callGcsPath/$filename", localFilePathFromRelativePath(filename))
@@ -49,32 +38,36 @@ class JesBackendCall(val backend: JesBackend,
                      val workflowDescriptor: WorkflowDescriptor,
                      val key: CallKey,
                      val locallyQualifiedInputs: CallInputs,
-                     val callAbortRegistrationFunction: AbortRegistrationFunction)
+                     val callAbortRegistrationFunction: Option[AbortRegistrationFunction])
   extends BackendCall with ProductionJesAuthentication with LazyLogging {
 
   import JesBackend._
   import JesBackendCall._
 
-  def jesCommandLine = s"/bin/bash ${cmdInput.local} > ${stdoutJesOutput.local} 2> ${stderrJesOutput.local}"
+  def jesCommandLine = s"/bin/bash ${cmdInput.local}"
 
-  val callGcsPath = backend.callGcsPath(workflowDescriptor, call.unqualifiedName, key.index)
+  val callGcsPath = JesBackend.callGcsPath(workflowDescriptor, key)
   val callDir = GcsPath(callGcsPath)
   val gcsExecPath = GcsPath(callGcsPath + "/" + JesExecScript)
   val defaultMonitoringOutputPath = callGcsPath + "/" + JesMonitoringLogFile
 
-  private val callContext = new CallContext(callGcsPath, stdoutJesOutput.gcs, stderrJesOutput.gcs)
+  val returnCodeFilename = jesReturnCodeFilename(key)
+  val jesStdoutGcsPath = s"$callGcsPath/${jesLogStdoutFilename(key)}"
+  val jesStderrGcsPath = s"$callGcsPath/${jesLogStderrFilename(key)}"
+  val jesLogGcsPath = s"$callGcsPath/${jesLogFilename(key)}"
+  val returnCodeGcsPath = s"$callGcsPath/$returnCodeFilename"
+
+  private val callContext = new CallContext(callGcsPath, jesStdoutGcsPath, jesStderrGcsPath)
 
   val engineFunctions = new JesCallEngineFunctions(workflowDescriptor.ioManager, callContext)
 
-  lazy val stderrJesOutput = jesOutput(callGcsPath, StderrFilename)
-  lazy val stdoutJesOutput = jesOutput(callGcsPath, StdoutFilename)
-  lazy val rcJesOutput = jesOutput(callGcsPath, RcFilename)
+  lazy val rcJesOutput = jesOutput(callGcsPath, returnCodeFilename)
   lazy val cmdInput = JesInput(ExecParamName, gcsExecPath.toString, localFilePathFromRelativePath(JesExecScript))
   lazy val diskInput = JesInput(WorkingDiskParamName, LocalWorkingDiskValue, Paths.get(JesCromwellRoot))
 
-  def standardParameters = Seq(stderrJesOutput, stdoutJesOutput, rcJesOutput, diskInput)
+  def standardParameters = Seq(rcJesOutput, diskInput)
 
-  def downloadRcFile = authenticateAsUser(workflowDescriptor) { storage => Try(storage.readFile(s"$callGcsPath/$RcFilename")) }
+  def downloadRcFile = authenticateAsUser(workflowDescriptor) { storage => Try(storage.readFile(returnCodeGcsPath)) }
 
   /**
    * Determine the output directory for the files matching a particular glob.
@@ -112,4 +105,12 @@ class JesBackendCall(val backend: JesBackend,
 
   override def useCachedCall(avoidedTo: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] =
     backend.useCachedCall(avoidedTo.asInstanceOf[JesBackendCall], this)
+
+  override def stdoutStderr: CallLogs = {
+    CallLogs(
+      stdout = WdlFile(jesStdoutGcsPath),
+      stderr = WdlFile(jesStderrGcsPath),
+      Option(Map("log" -> WdlFile(jesLogGcsPath)))
+    )
+  }
 }
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesConfiguration.scala b/src/main/scala/cromwell/engine/backend/jes/JesConfiguration.scala
index 129642dcd92..f4aba0b294e 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesConfiguration.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesConfiguration.scala
@@ -4,9 +4,6 @@ import cromwell.engine.backend.jes.authentication.JesDockerCredentials
 import cromwell.engine.io.gcs.GoogleConfiguration
 import cromwell.util.DockerConfiguration
 
-/**
- * Trait for JesConfiguration
- */
 trait JesConfiguration {
   def jesConf: JesAttributes
   def googleConf: GoogleConfiguration
diff --git a/src/main/scala/cromwell/engine/backend/jes/Run.scala b/src/main/scala/cromwell/engine/backend/jes/Run.scala
index 19b45070222..33bd824421c 100644
--- a/src/main/scala/cromwell/engine/backend/jes/Run.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/Run.scala
@@ -52,7 +52,7 @@ object Run  {
       logger.info(s"Outputs:\n${stringifyMap(rpr.getOutputs.asScala.toMap)}")
 
       val logging = new Logging()
-      logging.setGcsPath(s"${pipeline.gcsPath}/${JesBackendCall.JesLog}")
+      logging.setGcsPath(s"${pipeline.gcsPath}/${JesBackendCall.jesLogFilename(pipeline.key)}")
       rpr.setLogging(logging)
 
       // Currently, some resources (specifically disk) need to be specified both at pipeline creation and pipeline run time
@@ -162,7 +162,7 @@ case class Run(runId: String, pipeline: Pipeline, logger: WorkflowLogger) {
       // If this has transitioned to a running or complete state from a state this is not running or complete,
       // register the abort function.
       if (currentStatus.isRunningOrComplete && (previousStatus.isEmpty || !previousStatus.get.isRunningOrComplete)) {
-        backendCall.callAbortRegistrationFunction.register(AbortFunction(() => abort()))
+        backendCall.callAbortRegistrationFunction.foreach(_.register(AbortFunction(() => abort())))
       }
     }
 
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
index 4683d3deda2..cc9ae71a3f2 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
@@ -5,7 +5,6 @@ import java.nio.file.{Files, Path, Paths}
 
 import akka.actor.ActorSystem
 import better.files._
-import wdl4s._
 import com.typesafe.config.ConfigFactory
 import cromwell.engine.ExecutionIndex._
 import cromwell.engine._
@@ -13,9 +12,9 @@ import cromwell.engine.backend._
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.{CallStatus, ExecutionDatabaseKey}
 import cromwell.engine.workflow.CallKey
-import cromwell.engine.backend.BackendType
 import cromwell.util.FileUtil._
 import org.slf4j.LoggerFactory
+import wdl4s._
 
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
@@ -117,10 +116,12 @@ case class LocalBackend(actorSystem: ActorSystem) extends Backend with SharedFil
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
                         key: CallKey,
                         locallyQualifiedInputs: CallInputs,
-                        abortRegistrationFunction: AbortRegistrationFunction): BackendCall = {
+                        abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     LocalBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
   }
 
+  def stdoutStderr(backendCall: BackendCall): CallLogs = sharedFileSystemStdoutStderr(backendCall)
+
   def execute(backendCall: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future({
     val logger = workflowLoggerWithCall(backendCall)
     instantiateCommand(backendCall) match {
@@ -183,7 +184,7 @@ case class LocalBackend(actorSystem: ActorSystem) extends Backend with SharedFil
     val process = argv.run(ProcessLogger(stdoutWriter writeWithNewline, stderrTailed writeWithNewline))
 
     // TODO: As currently implemented, this process.destroy() will kill the bash process but *not* its descendants. See ticket DSDEEPB-848.
-    backendCall.callAbortRegistrationFunction.register(AbortFunction(() => process.destroy()))
+    backendCall.callAbortRegistrationFunction.foreach(_.register(AbortFunction(() => process.destroy())))
     val backendCommandString = argv.map(s => "\""+s+"\"").mkString(" ")
     logger.info(s"command: $backendCommandString")
     val processReturnCode = process.exitValue() // blocks until process finishes
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
index 023640e6f05..a4f080ba5e8 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
@@ -12,7 +12,7 @@ case class LocalBackendCall(backend: LocalBackend,
                             workflowDescriptor: WorkflowDescriptor,
                             key: CallKey,
                             locallyQualifiedInputs: CallInputs,
-                            callAbortRegistrationFunction: AbortRegistrationFunction) extends BackendCall with LocalFileSystemBackendCall {
+                            callAbortRegistrationFunction: Option[AbortRegistrationFunction]) extends BackendCall with LocalFileSystemBackendCall {
   val workflowRootPath = LocalBackend.hostExecutionPath(workflowDescriptor)
   val callRootPath = LocalBackend.hostCallPath(workflowDescriptor, call.unqualifiedName, key.index)
   val dockerContainerExecutionDir = LocalBackend.containerExecutionPath(workflowDescriptor)
@@ -35,4 +35,6 @@ case class LocalBackendCall(backend: LocalBackend,
 
   override def useCachedCall(avoidedTo: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] =
     backend.useCachedCall(avoidedTo.asInstanceOf[LocalBackendCall], this)
+
+  override def stdoutStderr: CallLogs = backend.stdoutStderr(this)
 }
diff --git a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
index accf3b70cee..64c8dd5905f 100644
--- a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
+++ b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
@@ -5,27 +5,24 @@ import java.nio.file.{Files, Path, Paths}
 
 import better.files.{File => ScalaFile, _}
 import com.typesafe.config.ConfigFactory
+import cromwell.engine.Hashing._
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import wdl4s.{CallInputs, Call, TaskOutput}
-import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType}
-import wdl4s.values.{WdlValue, _}
-import cromwell.engine.ExecutionIndex.ExecutionIndex
-import cromwell.engine.backend.{CallLogs, LocalFileSystemBackendCall, _}
+import cromwell.engine.backend.{AttemptedLookupResult, CallLogs, LocalFileSystemBackendCall, _}
 import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs.{GcsPath, GoogleCloudStorage}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId}
-import cromwell.engine._
+import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId, _}
 import cromwell.util.TryUtil
 import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
+import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType}
+import wdl4s.values.{WdlValue, _}
+import wdl4s.{Call, CallInputs, TaskOutput}
 
 import scala.collection.JavaConverters._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
-import Hashing._
-import cromwell.engine.backend.AttemptedLookupResult
 
 object SharedFileSystem {
   type LocalizationStrategy = (String, Path, WorkflowDescriptor) => Try[Unit]
@@ -154,13 +151,22 @@ trait SharedFileSystem {
 
   def adjustOutputPaths(call: Call, outputs: CallOutputs): CallOutputs = outputs
 
-  def stdoutStderr(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): CallLogs = {
-    val dir = LocalBackend.hostCallPath(descriptor.namespace.workflow.unqualifiedName, descriptor.id, callName, index)
+  def sharedFileSystemStdoutStderr(backendCall: BackendCall): CallLogs = {
+    val descriptor = backendCall.workflowDescriptor
+    val key = backendCall.key
+    val dir = LocalBackend.hostCallPath(
+      descriptor.namespace.workflow.unqualifiedName,
+      descriptor.id,
+      key.scope.unqualifiedName,
+      key.index
+    )
+
     CallLogs(
       stdout = WdlFile(dir.resolve("stdout").toAbsolutePath.toString),
       stderr = WdlFile(dir.resolve("stderr").toAbsolutePath.toString)
     )
   }
+
   /**
    * Creates host execution directory.
    */
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
index 32feade866c..bc05f360623 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
@@ -4,7 +4,6 @@ import java.nio.file.Files
 
 import akka.actor.ActorSystem
 import better.files._
-import wdl4s.CallInputs
 import cromwell.engine.backend._
 import cromwell.engine.backend.local.{LocalBackend, SharedFileSystem}
 import cromwell.engine.db.DataAccess._
@@ -12,8 +11,8 @@ import cromwell.engine.db.SgeCallBackendInfo
 import cromwell.engine.workflow.CallKey
 import cromwell.engine.{AbortRegistrationFunction, _}
 import cromwell.logging.WorkflowLogger
-import cromwell.engine.backend.BackendType
 import cromwell.util.FileUtil._
+import wdl4s.CallInputs
 
 import scala.annotation.tailrec
 import scala.concurrent.{ExecutionContext, Future}
@@ -31,10 +30,12 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
                         key: CallKey,
                         locallyQualifiedInputs: CallInputs,
-                        abortRegistrationFunction: AbortRegistrationFunction): BackendCall = {
+                        abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     SgeBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
   }
 
+  def stdoutStderr(backendCall: BackendCall): CallLogs = sharedFileSystemStdoutStderr(backendCall)
+
   def execute(backendCall: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] = Future( {
     val logger = workflowLoggerWithCall(backendCall)
     instantiateCommand(backendCall) match {
@@ -166,7 +167,7 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
     val logger = workflowLoggerWithCall(backendCall)
     val abortFunction = killSgeJob(backendCall, sgeJobId)
     val waitUntilCompleteFunction = waitUntilComplete(backendCall)
-    backendCall.callAbortRegistrationFunction.register(AbortFunction(abortFunction))
+    backendCall.callAbortRegistrationFunction.foreach(_.register(AbortFunction(abortFunction)))
     val jobReturnCode = waitUntilCompleteFunction
     val continueOnReturnCode = backendCall.runtimeAttributes.continueOnReturnCode
     logger.info(s"SGE job completed (returnCode=$jobReturnCode)")
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
index 9db36b229fb..496702051a7 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
@@ -12,7 +12,7 @@ case class SgeBackendCall(backend: SgeBackend,
                           workflowDescriptor: WorkflowDescriptor,
                           key: CallKey,
                           locallyQualifiedInputs: CallInputs,
-                          callAbortRegistrationFunction: AbortRegistrationFunction) extends BackendCall with LocalFileSystemBackendCall {
+                          callAbortRegistrationFunction: Option[AbortRegistrationFunction]) extends BackendCall with LocalFileSystemBackendCall {
   val workflowRootPath = LocalBackend.hostExecutionPath(workflowDescriptor)
   val callRootPath = LocalBackend.hostCallPath(workflowDescriptor, call.unqualifiedName, key.index)
   val stdout = callRootPath.resolve("stdout")
@@ -28,4 +28,6 @@ case class SgeBackendCall(backend: SgeBackend,
 
   override def useCachedCall(avoidedTo: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle] =
     backend.useCachedCall(avoidedTo.asInstanceOf[SgeBackendCall], this)
+
+  override def stdoutStderr: CallLogs = backend.stdoutStderr(this)
 }
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 7d3fa5577a1..445464728a3 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -1033,7 +1033,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
 
   private def sendStartMessage(callKey: CallKey, callInputs: Map[String, WdlValue]) = {
     def registerAbortFunction(abortFunction: AbortFunction): Unit = {}
-    val backendCall = backend.bindCall(workflow, callKey, callInputs, AbortRegistrationFunction(registerAbortFunction))
+    val backendCall = backend.bindCall(workflow, callKey, callInputs, Option(AbortRegistrationFunction(registerAbortFunction)))
     val log = backendCall.workflowLoggerWithCall("WorkflowActor", Option(akkaLogger))
 
     def loadCachedBackendCallAndMessage(descriptor: WorkflowDescriptor, cachedExecution: Execution) = {
@@ -1043,7 +1043,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
             descriptor,
             CallKey(c, cachedExecution.index.toIndex),
             callInputs,
-            AbortRegistrationFunction(registerAbortFunction)
+            Option(AbortRegistrationFunction(registerAbortFunction))
           )
           log.info(s"Call Caching: Cache hit. Using UUID(${cachedCall.workflowDescriptor.shortId}):${cachedCall.key.tag} as results for UUID(${backendCall.workflowDescriptor.shortId}):${backendCall.key.tag}")
           self ! UseCachedCall(callKey, CallActor.UseCachedCall(cachedCall, backendCall))
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index bd49540c045..2f605800c03 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -3,13 +3,12 @@ package cromwell.engine.workflow
 import akka.actor.FSM.SubscribeTransitionCallBack
 import akka.actor.{Actor, ActorRef, Props}
 import akka.event.{Logging, LoggingReceive}
-import akka.pattern.{pipe, ask}
+import akka.pattern.{ask, pipe}
 import cromwell.engine
-import cromwell.engine.EnhancedFullyQualifiedName
-import wdl4s._
 import cromwell.engine.ExecutionIndex._
 import cromwell.engine.ExecutionStatus.ExecutionStatus
 import cromwell.engine._
+import cromwell.engine.EnhancedFullyQualifiedName
 import cromwell.engine.backend.{Backend, CallLogs, CallMetadata}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.ExecutionDatabaseKey
@@ -19,6 +18,8 @@ import cromwell.util.WriteOnceStore
 import cromwell.webservice._
 import org.joda.time.DateTime
 import spray.json._
+import wdl4s._
+import wdl4s.values.WdlFile
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -161,13 +162,17 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
   }
 
   private def callStdoutStderr(workflowId: WorkflowId, callFqn: String): Future[Any] = {
+    def callKey(descriptor: WorkflowDescriptor, callName: String, key: ExecutionDatabaseKey) =
+      CallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
+    def backendCallFromKey(descriptor: WorkflowDescriptor, callName: String, key: ExecutionDatabaseKey) =
+      backend.bindCall(descriptor, callKey(descriptor, callName, key))
     for {
         _ <- assertWorkflowExistence(workflowId)
         descriptor <- globalDataAccess.getWorkflow(workflowId)
         _ <- assertCallExistence(workflowId, callFqn)
         callName <- Future.fromTry(assertCallFqnWellFormed(descriptor, callFqn))
         callLogKeys <- getCallLogKeys(workflowId, callFqn)
-        callStandardOutput <- Future.successful(callLogKeys map { key => backend.stdoutStderr(descriptor, callName, key.index) })
+        callStandardOutput <- Future.successful(callLogKeys.map(key => backendCallFromKey(descriptor, callName, key)).map(_.stdoutStderr))
       } yield callStandardOutput
   }
 
@@ -178,7 +183,9 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
         val callsToPaths = for {
           (key, status) <- sortedMap if hasLogs(statusMap.keys)(key)
           callName = assertCallFqnWellFormed(descriptor, key.fqn).get
-          callStandardOutput = backend.stdoutStderr(descriptor, callName, key.index)
+          callKey = CallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
+          backendCall = backend.bindCall(descriptor, callKey)
+          callStandardOutput = backend.stdoutStderr(backendCall)
         } yield key.fqn -> callStandardOutput
 
         callsToPaths groupBy { _._1 } mapValues { v => v map { _._2 } }
diff --git a/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
index 23cc765278b..e669700cd9a 100644
--- a/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
+++ b/src/test/scala/cromwell/InvalidRuntimeAttributesSpec.scala
@@ -3,10 +3,9 @@ package cromwell
 import java.net.URL
 
 import akka.testkit.{EventFilter, TestActorRef}
-import cromwell.engine.ExecutionIndex._
 import cromwell.engine._
 import cromwell.engine.backend.jes.{JesAttributes, JesBackend}
-import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
+import cromwell.engine.io.gcs.{GoogleConfiguration, ServiceAccountMode}
 import cromwell.engine.workflow.WorkflowManagerActor
 import cromwell.util.SampleWdl
 import org.scalatest.BeforeAndAfterAll
@@ -31,13 +30,12 @@ class InvalidRuntimeAttributesSpec extends CromwellTestkitSpec with BeforeAndAft
           executionBucket = anyString,
           endpointUrl = anyURL) {
         }
-        override def callGcsPath(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): String = "gs://fake/path"
         override def jesUserConnection(workflow: WorkflowDescriptor) = null
         override lazy val jesCromwellInterface = null
         override lazy val googleConf = GoogleConfiguration("appName", ServiceAccountMode("accountID", "pem"), None)
       }
 
-      val workflowSources = WorkflowSourceFiles(SampleWdl.HelloWorld.wdlSource(), SampleWdl.HelloWorld.wdlJson, "{}")
+      val workflowSources = WorkflowSourceFiles(SampleWdl.HelloWorld.wdlSource(), SampleWdl.HelloWorld.wdlJson, """ {"jes_gcs_root": "gs://fake/path"} """)
       val submitMessage = WorkflowManagerActor.SubmitWorkflow(workflowSources)
 
       runWdlWithWorkflowManagerActor(
diff --git a/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala b/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
index cbd68187e45..ee75557f25a 100644
--- a/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
+++ b/src/test/scala/cromwell/engine/WorkflowManagerActorSpec.scala
@@ -78,7 +78,7 @@ class WorkflowManagerActorSpec extends CromwellTestkitSpec {
           val worldSymbolHash = worldWdlString.getHash(descriptor)
           val symbols = Map(key -> new SymbolStoreEntry(key, WdlStringType, Option(worldWdlString), worldSymbolHash))
           // FIXME? null AST
-          val task = new Task("taskName", Seq.empty[Declaration], Seq.empty[CommandPart], Seq.empty, null)
+          val task = Task.empty
           val call = new Call(None, key.scope, task, Set.empty[FullyQualifiedName], Map.empty, None)
           for {
             _ <- globalDataAccess.createWorkflow(descriptor, symbols.values, Seq(call), new LocalBackend(system))
diff --git a/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala b/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
index ace5ffdb76a..fc1b32560a2 100644
--- a/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
@@ -18,7 +18,7 @@ class BackendCallSpec extends CromwellTestkitSpec with ScalaFutures {
   val sources = SampleWdl.CallCachingHashingWdl.asWorkflowSources()
   val descriptor = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), sources)
   val call = descriptor.namespace.workflow.calls.find(_.unqualifiedName == "t").get
-  val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, AbortRegistrationFunction(_ => ()))
+  val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
 
   "BackendCall hash function" should {
     "not change very often - if it changes, make sure it is for a good reason" in {
@@ -32,7 +32,7 @@ class BackendCallSpec extends CromwellTestkitSpec with ScalaFutures {
       val sources = SampleWdl.CallCachingHashingWdl.asWorkflowSources( s"""runtime { docker: "$nameAndDigest" } """)
       val descriptor = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), sources)
       val call = descriptor.namespace.workflow.calls.find(_.unqualifiedName == "t").get
-      val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, AbortRegistrationFunction(_ => ()))
+      val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
 
       val actual = backendCall.hash.futureValue.overallHash
       val expected = "ca6ee457780b78290785f112c3a3acb4"
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index 1cef8cd918f..fa66c397568 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -2,21 +2,22 @@ package cromwell.engine.backend.jes
 
 import java.net.URL
 import java.nio.file.Paths
+import java.util.UUID
 
 import com.google.api.client.testing.http.{HttpTesting, MockHttpTransport, MockLowLevelHttpRequest, MockLowLevelHttpResponse}
 import cromwell.CromwellTestkitSpec
-import cromwell.engine.WorkflowDescriptor
 import cromwell.engine.backend.BackendType
 import cromwell.engine.backend.jes.JesBackend.{JesInput, JesOutput}
 import cromwell.engine.backend.jes.authentication._
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.util.EncryptionSpec
+import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor, WorkflowId}
+import cromwell.util.{EncryptionSpec, SampleWdl}
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 import org.specs2.mock.Mockito
 import wdl4s.types.{WdlArrayType, WdlFileType, WdlMapType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlFile, WdlMap, WdlString}
+import wdl4s.values._
 import wdl4s.{CallInputs, RuntimeAttributes}
 
 import scala.util.{Success, Try}
@@ -216,15 +217,39 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
     jesBackend.monitoringIO(backendCall) shouldBe None
   }
 
-  "JesBackendCall" should "return Jes log paths" in {
-    val stdoutstderr = JesBackendCall.stdoutStderr("gs://path/to/call")
+  "JesBackendCall" should "return JES log paths for non-scattered call" in {
+    val wd = WorkflowDescriptor(WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")), SampleWdl.HelloWorld.asWorkflowSources(
+      runtime = """ runtime {docker: "ubuntu:latest"} """,
+      workflowOptions = """ {"jes_gcs_root": "gs://path/to/gcs_root"} """
+    ))
+    val call = wd.namespace.workflow.findCallByName("hello").get
+    val backendCall = jesBackend.bindCall(wd, CallKey(call, None))
+    val stdoutstderr = backendCall.stdoutStderr
+
+    stdoutstderr.stdout shouldBe WdlFile("gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log")
+    stdoutstderr.stderr shouldBe WdlFile("gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stderr.log")
+
+    stdoutstderr.backendLogs shouldBe defined
+    val logsMap = stdoutstderr.backendLogs.get
+    logsMap should contain key "log"
+    logsMap("log") shouldBe WdlFile("gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello.log")
+  }
+
+  it should "return JES log paths for scattered call" in {
+    val wd = WorkflowDescriptor(WorkflowId(UUID.fromString("e6236763-c518-41d0-9688-432549a8bf7c")), new SampleWdl.ScatterWdl().asWorkflowSources(
+      runtime = """ runtime {docker: "ubuntu:latest"} """,
+      workflowOptions = """ {"jes_gcs_root": "gs://path/to/gcs_root"} """
+    ))
+    val call = wd.namespace.workflow.findCallByName("B").get
+    val backendCall = jesBackend.bindCall(wd, CallKey(call, Some(2)))
+    val stdoutstderr = backendCall.stdoutStderr
+
+    stdoutstderr.stdout shouldBe WdlFile("gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7c/call-B/shard-2/B-2-stdout.log")
+    stdoutstderr.stderr shouldBe WdlFile("gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7c/call-B/shard-2/B-2-stderr.log")
+
     stdoutstderr.backendLogs shouldBe defined
     val logsMap = stdoutstderr.backendLogs.get
     logsMap should contain key "log"
-    logsMap("log") shouldBe WdlFile("gs://path/to/call/jes.log")
-    logsMap should contain key "stdout"
-    logsMap("stdout") shouldBe WdlFile("gs://path/to/call/jes-stdout.log")
-    logsMap should contain key "stderr"
-    logsMap("stderr") shouldBe WdlFile("gs://path/to/call/jes-stderr.log")
+    logsMap("log") shouldBe WdlFile("gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7c/call-B/shard-2/B-2.log")
   }
 }
diff --git a/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala b/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
index a874c206d65..39aa0d52410 100644
--- a/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
@@ -39,7 +39,7 @@ class LocalBackendSpec extends CromwellTestkitSpec with Mockito {
   def testFailOnStderr(descriptor: WorkflowDescriptor, expectSuccess: Boolean): Unit = {
     val call = descriptor.namespace.workflow.calls.head
     val backend = new LocalBackend(system)
-    val backendCall = backend.bindCall(descriptor, CallKey(call, None), Map.empty[String, WdlValue], AbortRegistrationFunction(_ => ()))
+    val backendCall = backend.bindCall(descriptor, CallKey(call, None), Map.empty[String, WdlValue], abortRegistrationFunction = None)
     backendCall.execute map { _.result } map {
       case FailedExecution(e, _) => if (expectSuccess) fail("A call in a failOnStderr test which should have succeeded has failed ", e)
       case SuccessfulExecution(_, _, _, _, _) => if (!expectSuccess) fail("A call in a failOnStderr test which should have failed has succeeded")
diff --git a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
index c9ded80282c..bc7e9f061bf 100644
--- a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
+++ b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
@@ -75,14 +75,14 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
                                  ) = throw new NotImplementedError
 
     override def adjustOutputPaths(call: Call, outputs: CallOutputs): CallOutputs = throw new NotImplementedError
-    override def stdoutStderr(descriptor: WorkflowDescriptor, callName: String, index: ExecutionIndex): CallLogs = throw new NotImplementedError
+    override def stdoutStderr(backendCall: BackendCall): CallLogs = throw new NotImplementedError
     override def initializeForWorkflow(workflow: WorkflowDescriptor) = throw new NotImplementedError
     override def prepareForRestart(restartableWorkflow: WorkflowDescriptor)(implicit ec: ExecutionContext) = throw new NotImplementedError
 
     override def bindCall(workflowDescriptor: WorkflowDescriptor,
                           key: CallKey,
                           locallyQualifiedInputs: CallInputs,
-                          abortRegistrationFunction: AbortRegistrationFunction): BackendCall =
+                          abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall =
       throw new NotImplementedError
 
     override def backendType: BackendType = throw new NotImplementedError
@@ -357,7 +357,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val callFqn = "fully.qualified.name"
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
@@ -485,7 +485,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
         val workflowId = WorkflowId(UUID.randomUUID())
         val workflowInfo = WorkflowDescriptor(workflowId, testSources)
 
-        val task = new Task("taskName", Nil, Nil, Nil, null)
+        val task = Task.empty
         val call =
           if (setCallParent) {
             val workflow = new Workflow("workflow.name", Nil, Nil)
@@ -543,7 +543,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
       val callKey = CallKey(call, None)
       (for {
@@ -575,7 +575,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
       val key = new SymbolStoreKey(callFqn, symbolFqn, None, input = true)
       val entry = new SymbolStoreEntry(key, WdlStringType, Option(testWdlString), Option(testWdlStringHash))
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -607,7 +607,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowDescriptor = WorkflowDescriptor(workflowId, testSources)
       val key = new SymbolStoreKey(callFqn, symbolFqn, None, input = true)
       val entry = new SymbolStoreEntry(key, WdlArrayType(WdlStringType), Option(wdlArray), wdlArray.getHash(workflowDescriptor))
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -648,7 +648,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val symbolLqn = "symbol"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
       val workflowOutputList = Seq(ReportableSymbol("call.fully.qualified.scope.symbol", WdlArrayType(WdlStringType)))
 
@@ -678,7 +678,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val callFqn2 = "call.fully.qualified.scope$s2"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call1 = new Call(None, callFqn1, task, Set.empty[FullyQualifiedName], Map.empty, None)
       val shardIndex1 = Option(0)
       val pid1 = Option(123)
@@ -711,7 +711,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val symbolLqn = "symbol"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -760,7 +760,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       dataAccess.createWorkflow(workflowInfo, Nil, Seq(call),
@@ -772,7 +772,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       dataAccess.createWorkflow(workflowInfo, Nil, Seq(call),
@@ -788,7 +788,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
       val key = new SymbolStoreKey(callFqn, symbolFqn, None, input = true)
       val entry = new SymbolStoreEntry(key, WdlStringType, Option(testWdlString), Option(testWdlStringHash))
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -819,7 +819,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val symbolFqn = callFqn + "." + symbolLqn
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -835,7 +835,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val backendInfo = new LocalCallBackendInfo(Option(123))
 
@@ -858,7 +858,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowId2 = WorkflowId(UUID.randomUUID())
       val workflowInfo1 = WorkflowDescriptor(workflowId1, testSources)
       val workflowInfo2 = WorkflowDescriptor(workflowId2, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val backendInfo1 = new LocalCallBackendInfo(Option(123))
       val backendInfo2 = new LocalCallBackendInfo(Option(321))
@@ -887,7 +887,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       (for {
@@ -902,7 +902,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
 
       // Assert a singular `Execution` in `executions`, and that the dates of the `Execution` are defined
@@ -936,7 +936,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       // We need an execution to create an event. We need a workflow to make an execution. Le Sigh...
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val shardedCall = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val shardIndex = Some(0)
@@ -980,7 +980,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       // We need an execution to create an event. We need a workflow to make an execution. Le Sigh...
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
-      val task = new Task("taskName", Nil, Nil, Nil, null)
+      val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val backendInfo = new LocalCallBackendInfo(Option(123))
 
diff --git a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
index c7e7311b3ae..5bca95c7c1c 100644
--- a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
+++ b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
@@ -31,7 +31,7 @@ class WorkflowLoggerSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
     descriptor,
     CallKey(descriptor.namespace.workflow.calls.find(_.unqualifiedName == "x").head, None),
     Map.empty[String, WdlValue],
-    AbortRegistrationFunction(_ => ())
+    callAbortRegistrationFunction = None
   )
 
   "WorkflowLogger" should "create a valid tag" in {
diff --git a/src/test/scala/cromwell/util/SampleWdl.scala b/src/test/scala/cromwell/util/SampleWdl.scala
index 32d4223f70a..90be2ea54b4 100644
--- a/src/test/scala/cromwell/util/SampleWdl.scala
+++ b/src/test/scala/cromwell/util/SampleWdl.scala
@@ -1141,6 +1141,7 @@ object SampleWdl {
       |  command {
       |    echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nscott"
       |  }
+      |  RUNTIME
       |  output {
       |    Array[String] A_out = read_lines(stdout())
       |  }
@@ -1151,6 +1152,7 @@ object SampleWdl {
       |  command {
       |    python -c "print(len('${B_in}'))"
       |  }
+      |  RUNTIME
       |  output {
       |    Int B_out = read_int(stdout())
       |  }
@@ -1161,6 +1163,7 @@ object SampleWdl {
       |  command {
       |    python -c "print(${C_in}*100)"
       |  }
+      |  RUNTIME
       |  output {
       |    Int C_out = read_int(stdout())
       |  }
@@ -1171,6 +1174,7 @@ object SampleWdl {
       |  command {
       |    python -c "print(${sep = '+' D_in})"
       |  }
+      |  RUNTIME
       |  output {
       |    Int D_out = read_int(stdout())
       |  }
@@ -1180,6 +1184,7 @@ object SampleWdl {
       |  command {
       |    python -c "print(9)"
       |  }
+      |  RUNTIME
       |  output {
       |    Int E_out = read_int(stdout())
       |  }
@@ -1198,7 +1203,7 @@ object SampleWdl {
         |  }
         |  call D {input: D_in = B.B_out}
         |}
-      """.stripMargin
+      """.stripMargin.replaceAll("RUNTIME", runtime)
 
     override lazy val rawInputs = Map.empty[String, String]
   }
@@ -1219,7 +1224,7 @@ object SampleWdl {
         |  }
         |  call D {input: D_in = B.B_out}
         |}
-      """.stripMargin
+      """.stripMargin.replaceAll("RUNTIME", runtime)
 
     override lazy val rawInputs = Map.empty[String, String]
   }

From e02e2510c1686ef9d3723c061f9ac21bdd67f1e5 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Wed, 20 Jan 2016 17:50:47 -0500
Subject: [PATCH 19/31] Persist disallowed return codes in DB

PR comment
---
 src/main/scala/cromwell/engine/backend/jes/JesBackend.scala | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index 62ac5af00c0..c58849da1c1 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -540,7 +540,8 @@ case class JesBackend(actorSystem: ActorSystem)
 
       status match {
         case Run.Success(events) if backendCall.runtimeAttributes.failOnStderr && stderrLength.intValue > 0 =>
-          FailedExecutionHandle(new Throwable(s"${log.tag} execution failed: stderr has length $stderrLength")).future
+          // returnCode will be None if it couldn't be downloaded/parsed, which will yield a null in the DB
+          FailedExecutionHandle(new Throwable(s"${log.tag} execution failed: stderr has length $stderrLength"), returnCode.toOption).future
         case Run.Success(events) if returnCodeContents.isFailure =>
           val exception = returnCode.failed.get
           log.warn(s"${log.tag} could not download return code file, retrying: " + exception.getMessage, exception)
@@ -549,7 +550,7 @@ case class JesBackend(actorSystem: ActorSystem)
         case Run.Success(events) if returnCode.isFailure =>
           FailedExecutionHandle(new Throwable(s"${log.tag} execution failed: could not parse return code as integer: " + returnCodeContents.get)).future
         case Run.Success(events) if !continueOnReturnCode.continueFor(returnCode.get) =>
-          FailedExecutionHandle(new Throwable(s"${log.tag} execution failed: disallowed command return code: " + returnCode.get)).future
+          FailedExecutionHandle(new Throwable(s"${log.tag} execution failed: disallowed command return code: " + returnCode.get), returnCode.toOption).future
         case Run.Success(events) =>
           backendCall.hash map { h => handleSuccess(outputMappings, backendCall.workflowDescriptor, events, returnCode.get, h, handle) }
         case Run.Failed(errorCode, errorMessage) =>

From 06072ada3d7ef09b847f33d5d9b2a9224e26a04a Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Thu, 21 Jan 2016 11:40:22 -0500
Subject: [PATCH 20/31] Added a note regarding the application.conf
 `db.connectionTimeout`.

---
 README.md                           | 7 +------
 src/main/resources/application.conf | 3 +++
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 30423da0485..91a95a771a5 100644
--- a/README.md
+++ b/README.md
@@ -384,17 +384,12 @@ database {
   config = main.mysql
 
   main {
-    hsqldb {
-      db.url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
-      db.driver = "org.hsqldb.jdbcDriver"
-      driver = "slick.driver.HsqldbDriver$"
-      slick.createSchema = true
-    }
     mysql {
       db.url = "jdbc:mysql://localhost:3306/cromwell"
       db.user = "root"
       db.password = ""
       db.driver = "com.mysql.jdbc.Driver"
+      db.connectionTimeout = 5000 // NOTE: The default 1000ms is often too short for production mysql use
       driver = "slick.driver.MySQLDriver$"
     }
   }
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index e88ebf5aeea..62488393d73 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -135,6 +135,7 @@ database {
     hsqldb {
       db.url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
       db.driver = "org.hsqldb.jdbcDriver"
+      db.connectionTimeout = 1000 // NOTE: 1000ms (the default) is ok for a small hsqldb, but often too short for mysql
       driver = "slick.driver.HsqldbDriver$"
       slick.createSchema = true
     }
@@ -144,6 +145,7 @@ database {
     hsqldb {
       db.url = "jdbc:hsqldb:mem:testdb;shutdown=false;hsqldb.tx=mvcc"
       db.driver = "org.hsqldb.jdbcDriver"
+      db.connectionTimeout = 1000 // NOTE: 1000ms (the default) is ok for a small hsqldb, but often too short for mysql
       driver = "slick.driver.HsqldbDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"
@@ -156,6 +158,7 @@ database {
       db.user = "travis"
       db.password = ""
       db.driver = "com.mysql.jdbc.Driver"
+      db.connectionTimeout = 5000 // NOTE: The default 1000ms is often too short for production mysql use
       driver = "slick.driver.MySQLDriver$"
       liquibase = {
         changelog = "src/main/migrations/changelog.xml"

From eacda40bdeb462ff9d2ba72de011534d95e7ef9d Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Thu, 21 Jan 2016 11:59:33 -0500
Subject: [PATCH 21/31] Validate endpoint docs

---
 README.md                                     | 69 +++++++++++++++++++
 src/main/resources/swagger/cromwell.yaml      |  2 +-
 .../engine/workflow/ValidateActor.scala       | 21 ++++--
 .../webservice/CromwellApiService.scala       |  2 +-
 4 files changed, 87 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index 30423da0485..d5e4866c06d 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,7 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
 * [REST API](#rest-api)
   * [REST API Versions](#rest-api-versions)
   * [POST /api/workflows/:version](#post-apiworkflowsversion)
+  * [POST /api/workflows/:version/validate](#post-apiworkflowsversionvalidate)
   * [GET /api/workflows/:version/query](#get-apiworkflowsversionquery)
   * [GET /api/workflows/:version/:id/status](#get-apiworkflowsversionidstatus)
   * [GET /api/workflows/:version/:id/outputs](#get-apiworkflowsversionidoutputs)
@@ -1169,6 +1170,74 @@ Content-Disposition: form-data; name="workflowOptions"; filename="options.json"
 --f3fd038395644de596c460257626edd7--
 ```
 
+## POST /api/workflows/:version/validate
+
+This endpoint allows WDL to be validated in the context of a running Cromwell server and a given inputs file.
+
+
+Validation includes checking that the WDL is syntactically correct, that the runtime attributes are correct and (if supplied) that
+the inputs file satisfies the WDL file's input requirements.
+
+* `wdlSource` - *Required* Contains the WDL file to submit for execution.
+* `workflowInputs` - *Optional* JSON file containing the inputs.
+
+cURL:
+```
+$ curl -v "localhost:8000/api/workflows/v1/validate" -F wdlSource=@src/main/resources/3step.wdl -F workflowInputs=@test.json
+```
+
+HTTPie:
+
+```
+$ http --print=hbHB --form POST localhost:8000/api/workflows/v1 wdlSource=@src/main/resources/3step.wdl workflowInputs@inputs.json
+```
+
+Request:
+```
+POST /api/workflows/v1/validate HTTP/1.1
+Accept: */*
+Accept-Encoding: gzip, deflate
+Connection: keep-alive
+Content-Length: 463
+Content-Type: application/x-www-form-urlencoded; charset=utf-8
+Host: localhost:8000
+User-Agent: HTTPie/0.9.2
+
+wdlSource=...wdlInputs=...
+```
+
+Response (successful validation):
+```
+HTTP/1.1 200 OK
+Content-Length: 63
+Content-Type: application/json; charset=UTF-8
+Date: Thu, 21 Jan 2016 16:57:39 GMT
+Server: spray-can/1.3.2
+
+{
+    "message": "Validation succeeded.",
+    "status": "success"
+}
+```
+
+Response (failed validation example):
+```
+HTTP/1.1 400 Bad Request
+Content-Length: 159
+Content-Type: application/json; charset=UTF-8
+Date: Thu, 21 Jan 2016 16:56:32 GMT
+Server: spray-can/1.3.2
+
+{
+    "errors": [
+        "Missing required keys in runtime configuration for backend 'JES': docker"
+    ],
+    "message": "RuntimeAttribute is not valid.",
+    "status": "fail"
+}
+```
+
+
 ## GET /api/workflows/:version/query
 
 This endpoint allows for querying workflows based on the following criteria:
diff --git a/src/main/resources/swagger/cromwell.yaml b/src/main/resources/swagger/cromwell.yaml
index 8a299d3c7f4..c91796f0eee 100644
--- a/src/main/resources/swagger/cromwell.yaml
+++ b/src/main/resources/swagger/cromwell.yaml
@@ -105,7 +105,7 @@ paths:
           in: formData
         - name: workflowInputs
           description: WDL JSON
-          required: true
+          required: false
           type: file
           in: formData
       tags:
diff --git a/src/main/scala/cromwell/engine/workflow/ValidateActor.scala b/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
index 60f048ff52d..9ac9aaee544 100644
--- a/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/ValidateActor.scala
@@ -23,7 +23,7 @@ object ValidateActor {
   sealed trait ValidateActorMessage
   case object ValidateWorkflow extends ValidateActorMessage
 
-  def props(wdlSource: WdlSource, wdlJson: WdlJson): Props = {
+  def props(wdlSource: WdlSource, wdlJson: Option[WdlJson]): Props = {
     Props(new ValidateActor(wdlSource, wdlJson))
   }
 
@@ -32,7 +32,7 @@ object ValidateActor {
   }
 }
 
-class ValidateActor(wdlSource: WdlSource, wdlJson: WdlJson)
+class ValidateActor(wdlSource: WdlSource, workflowInputs: Option[WdlJson])
   extends Actor with LazyLogging {
 
   import ValidateActor.{ValidateWorkflow, tag}
@@ -45,15 +45,26 @@ class ValidateActor(wdlSource: WdlSource, wdlJson: WdlJson)
       // NOTE: self shuts down when the parent PerRequest shuts down
   }
 
+  private def validateInputs(namespaceWithWorkflow: NamespaceWithWorkflow, maybeWorkflowInputs: Option[WdlJson]): Future[Unit] = {
+    maybeWorkflowInputs match {
+      case Some(wi) =>
+        for {
+          inputs <- Future(wi.parseJson).map(_.asJsObject.fields)
+          coercedInputs <- Future.fromTry(namespaceWithWorkflow.coerceRawInputs(inputs))
+        } yield ()
+      case None => Future.successful(())
+    }
+  }
+
   private def validateWorkflow(sentBy: ActorRef): Unit = {
     logger.info(s"$tag for $sentBy")
     val futureValidation: Future[Unit] = for {
       namespaceWithWorkflow <- Future(NamespaceWithWorkflow.load(wdlSource))
-      inputs <- Future(wdlJson.parseJson).map(_.asJsObject.fields)
-      coercedInputs <- Future.fromTry(namespaceWithWorkflow.coerceRawInputs(inputs))
+      validatedInputs <- validateInputs(namespaceWithWorkflow, workflowInputs)
       runtime = namespaceWithWorkflow.workflow.calls foreach { _.toRuntimeAttributes }
-    } yield () // Validate that the future run and return `Success[Unit]` aka (), or `Failure[Exception]`
+    } yield ()
 
+    // Now validate that this Future completed successfully:
     futureValidation onComplete {
       case Success(_) =>
         logger.info(s"$tag success $sentBy")
diff --git a/src/main/scala/cromwell/webservice/CromwellApiService.scala b/src/main/scala/cromwell/webservice/CromwellApiService.scala
index 37e18cd8217..071dace6046 100644
--- a/src/main/scala/cromwell/webservice/CromwellApiService.scala
+++ b/src/main/scala/cromwell/webservice/CromwellApiService.scala
@@ -111,7 +111,7 @@ trait CromwellApiService extends HttpService with PerRequestCreator {
     path("workflows" / Segment / "validate") { version =>
       traceName("validate") {
         post {
-          formFields("wdlSource", "workflowInputs") { (wdlSource, workflowInputs) =>
+          formFields("wdlSource", "workflowInputs".?) { (wdlSource, workflowInputs) =>
             requestContext =>
               perRequest(
                 requestContext,

From 025e22b2979219a5062ea51cdaaa39e6c17a189e Mon Sep 17 00:00:00 2001
From: Scott Frazer <scott.d.frazer@gmail.com>
Date: Thu, 21 Jan 2016 14:05:15 -0500
Subject: [PATCH 22/31] Updating Cromwell based on some wdl4s changes

---
 build.sbt                                        |  2 +-
 .../cromwell/engine/backend/BackendCall.scala    |  2 +-
 .../cromwell/engine/backend/jes/JesBackend.scala |  4 ++--
 .../engine/backend/local/SharedFileSystem.scala  |  2 +-
 .../cromwell/engine/workflow/WorkflowActor.scala | 16 +++++++---------
 5 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/build.sbt b/build.sbt
index 8853e016660..9acd8382eac 100644
--- a/build.sbt
+++ b/build.sbt
@@ -11,7 +11,7 @@ scalaVersion := "2.11.7"
 
 val lenthallV = "0.15"
 
-val wdl4sV = "0.2-6f2ba49-SNAPSHOT"
+val wdl4sV = "0.2-b0ec447-SNAPSHOT"
 
 val sprayV = "1.3.2"
 
diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index 0b4c0d7f98d..f7f6329bdaa 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -160,7 +160,7 @@ trait BackendCall {
       call.task.commandTemplateString,
       orderedInputs map { case (k, v) => s"$k=${v.computeHash(workflowDescriptor.fileHasher).value}" } mkString "\n",
       orderedRuntime map { case (k, v) => s"$k=$v" } mkString "\n",
-      orderedOutputs map { o => s"${o.wdlType.toWdlString} ${o.name} = ${o.expression.toWdlString}" } mkString "\n"
+      orderedOutputs map { o => s"${o.wdlType.toWdlString} ${o.name} = ${o.requiredExpression.toWdlString}" } mkString "\n"
     ).mkString("\n---\n").md5Sum
 
     ExecutionHash(overallHash, dockerHash)
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index 62ac5af00c0..03faa70c312 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -370,7 +370,7 @@ case class JesBackend(actorSystem: ActorSystem)
   def generateJesOutputs(backendCall: BackendCall): Seq[JesOutput] = {
     val log = workflowLoggerWithCall(backendCall)
     val wdlFileOutputs = backendCall.call.task.outputs flatMap { taskOutput =>
-      taskOutput.expression.evaluateFiles(backendCall.lookupFunction(Map.empty), NoFunctions, taskOutput.wdlType) match {
+      taskOutput.requiredExpression.evaluateFiles(backendCall.lookupFunction(Map.empty), NoFunctions, taskOutput.wdlType) match {
         case Success(wdlFiles) => wdlFiles map gcsPathToLocal
         case Failure(ex) =>
           log.warn(s"Could not evaluate $taskOutput: ${ex.getMessage}")
@@ -522,7 +522,7 @@ case class JesBackend(actorSystem: ActorSystem)
     * Then, via wdlFileToGcsPath(), we attempt to find the JesOutput with .name == "out.txt".
     * If it is found, then WdlFile("gs://some_bucket/out.txt") will be returned.
     */
-    wdlValue <- taskOutput.expression.evaluate(customLookupFunction(backendCall, currentList.toLookupMap), backendCall.engineFunctions)
+    wdlValue <- taskOutput.requiredExpression.evaluate(customLookupFunction(backendCall, currentList.toLookupMap), backendCall.engineFunctions)
     coercedValue <- taskOutput.wdlType.coerceRawValue(wdlValue)
     value = wdlValueToGcsPath(generateJesOutputs(backendCall))(coercedValue)
   } yield value
diff --git a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
index 64c8dd5905f..ecf1390224e 100644
--- a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
+++ b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
@@ -144,7 +144,7 @@ trait SharedFileSystem {
   }
 
   private def outputLookup(taskOutput: TaskOutput, backendCall: LocalFileSystemBackendCall, currentList: Seq[AttemptedLookupResult]) = for {
-    expressionValue <- taskOutput.expression.evaluate(backendCall.lookupFunction(currentList.toLookupMap), backendCall.engineFunctions)
+    expressionValue <- taskOutput.requiredExpression.evaluate(backendCall.lookupFunction(currentList.toLookupMap), backendCall.engineFunctions)
     convertedValue <- outputAutoConversion(backendCall, taskOutput, expressionValue)
     pathAdjustedValue <- Success(absolutizeOutputWdlFile(convertedValue, backendCall.callRootPath))
   } yield pathAdjustedValue
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 445464728a3..4831f90500a 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -776,7 +776,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   }
 
   private def lookupDeclaration(workflow: Workflow)(name: String): Try[WdlValue] = {
-    workflow.declarations find { _.name == name } match {
+    workflow.scopedDeclarations find { _.name == name } match {
       case Some(declaration) => fetchFullyQualifiedName(declaration.fullyQualifiedName)
       case None => Failure(new WdlExpressionException(s"Could not find a declaration with name '$name'"))
     }
@@ -816,9 +816,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   }
 
   def fetchLocallyQualifiedInputs(callKey: CallKey): Try[Map[String, WdlValue]] = Try {
-    val parentWorkflow = callKey.scope.ancestry.lastOption map { _.asInstanceOf[Workflow] } getOrElse {
-      throw new WdlExpressionException("Expecting 'call' to have a 'workflow' parent.")
-    }
+    val parentWorkflow = callKey.scope.rootWorkflow
 
     def lookup(identifier: String): WdlValue = {
       /* This algorithm defines three ways to lookup an identifier in order of their precedence:
@@ -839,24 +837,24 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     val entries = fetchCallInputEntries(callKey)
     entries.map { entry =>
       // .get are used below because the exception will be captured by the Try
-      val taskInput = findTaskInput(entry.scope, entry.key.name).get
+      val declaration = findTaskDeclaration(entry.scope, entry.key.name).get
 
       val value = entry.wdlValue match {
         case Some(e: WdlExpression) => e.evaluate(lookup, NoFunctions)
         case Some(v) => Success(v)
         case _ => Failure(new WdlExpressionException("Unknown error"))
       }
-      val coercedValue = value.flatMap(x => taskInput.wdlType.coerceRawValue(x))
+      val coercedValue = value.flatMap(x => declaration.wdlType.coerceRawValue(x))
       entry.key.name -> coercedValue.get
     }.toMap
   }
 
-  private def findTaskInput(callFqn: String, inputName: String): Try[TaskInput] = {
+  private def findTaskDeclaration(callFqn: String, inputName: String): Try[Declaration] = {
     val exception = new WdlExpressionException(s"Could not find task input '$inputName' for call '$callFqn'")
     workflow.namespace.resolve(callFqn) match {
       case Some(c:Call) =>
-        c.task.inputs.find(_.name == inputName) match {
-          case Some(i) => Success(i)
+        c.task.declarations.find(_.name == inputName) match {
+          case Some(decl) => Success(decl)
           case None => Failure(exception)
         }
       case _ => Failure(exception)

From 64d3f96228c4077c924471c27be27ff257ac809b Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 22 Jan 2016 11:10:00 -0500
Subject: [PATCH 23/31] Fixed timeout error during mysql testing. Tagged all
 database specific tests as `IntegrationTest`. As mysql now has 5s to timeout,
 increased the testing timeout used to detect if mysql is available. Removed
 unused code and optimized imports.

---
 .../engine/db/slick/SlickDataAccessSpec.scala | 98 +++++++++----------
 1 file changed, 47 insertions(+), 51 deletions(-)

diff --git a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
index bc7e9f061bf..e9d341d18b1 100644
--- a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
+++ b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
@@ -3,35 +3,34 @@ package cromwell.engine.db.slick
 import java.sql.SQLException
 import java.util.UUID
 
+import cromwell.CromwellSpec.IntegrationTest
 import cromwell.CromwellTestkitSpec.TestWorkflowManagerSystem
-import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import cromwell.engine.{CallOutput, CallOutputs}
-import wdl4s._
-import wdl4s.types.{WdlArrayType, WdlStringType}
-import wdl4s.values.{WdlArray, WdlString}
-import cromwell.engine.ExecutionIndex.ExecutionIndex
-import cromwell.engine._
+import cromwell.engine.Hashing._
 import cromwell.engine.backend.local.{LocalBackend, LocalBackendCall}
-import cromwell.engine.backend.{Backend, CallLogs}
+import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
+import cromwell.engine.backend.{Backend, BackendType, CallLogs}
 import cromwell.engine.db.slick.SlickDataAccessSpec.{AllowFalse, AllowTrue}
 import cromwell.engine.db.{CallStatus, ExecutionDatabaseKey, LocalCallBackendInfo}
 import cromwell.engine.io.IoInterface
 import cromwell.engine.workflow.{CallKey, ScatterKey, WorkflowOptions}
-import cromwell.engine.backend.BackendType
+import cromwell.engine.{CallOutput, CallOutputs, _}
 import cromwell.util.SampleWdl
 import cromwell.webservice
 import cromwell.webservice.{CallCachingParameters, WorkflowQueryKey, WorkflowQueryParameters}
-import cromwell.webservice.{WorkflowQueryKey, WorkflowQueryParameters}
 import org.joda.time.DateTime
 import org.scalactic.StringNormalizations._
+import org.scalatest.PartialFunctionValues._
+import org.scalatest.concurrent.PatienceConfiguration.Timeout
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.time.SpanSugar._
 import org.scalatest.time.{Millis, Seconds, Span}
-import org.scalatest.PartialFunctionValues._
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
+import wdl4s._
+import wdl4s.types.{WdlArrayType, WdlStringType}
+import wdl4s.values.{WdlArray, WdlString}
 
 import scala.concurrent.{ExecutionContext, Future}
-import Hashing._
 
 object SlickDataAccessSpec {
   val AllowFalse = Seq(webservice.QueryParameter("allow", "false"))
@@ -102,7 +101,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
   def databaseWithConfig(path: => String, testRequired: => Boolean): Unit = {
 
     lazy val testDatabase = new TestSlickDatabase(path)
-    lazy val canConnect = testRequired || testDatabase.isValidConnection.futureValue
+    lazy val canConnect = testRequired || testDatabase.isValidConnection.futureValue(Timeout(10.seconds))
     lazy val dataAccess = testDatabase.slickDataAccess
 
     /**
@@ -126,7 +125,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
     val testWdlStringShard = WdlString("testStringValueShard")
     val testWdlStringShardHash = testWdlStringShard.computeHash
 
-    it should "(if hsqldb) have transaction isolation mvcc" in {
+    it should "(if hsqldb) have transaction isolation mvcc" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       import dataAccess.dataAccess.driver.api._
 
@@ -148,13 +147,13 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "setup via liquibase if necessary" in {
+    it should "setup via liquibase if necessary" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       if (testDatabase.useLiquibase)
         testDatabase.setupLiquibase()
     }
 
-    it should "create and retrieve the workflow for just reading" in {
+    it should "create and retrieve the workflow for just reading" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -174,7 +173,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "create and query a workflow" in {
+    it should "create and query a workflow" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowInfo = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), testSources)
       val workflow2Info = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), test2Sources)
@@ -246,7 +245,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       }
     }
 
-    it should "support call caching configuration for specified calls in a regular workflow" in {
+    it should "support call caching configuration for specified calls in a regular workflow" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowInfo = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), SampleWdl.ThreeStep.asWorkflowSources())
 
@@ -270,7 +269,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "support call caching configuration for specified calls in a scattered workflow" in {
+    it should "support call caching configuration for specified calls in a scattered workflow" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowInfo = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), SampleWdl.SimpleScatterWdl.asWorkflowSources())
 
@@ -312,7 +311,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "support call caching configuration for all calls in a regular workflow" in {
+    it should "support call caching configuration for all calls in a regular workflow" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
 
       val workflowInfo = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), SampleWdl.ThreeStep.asWorkflowSources())
@@ -330,7 +329,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "support call caching configuration for all calls in a scattered workflow" in {
+    it should "support call caching configuration for all calls in a scattered workflow" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
 
       val workflowInfo = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), SampleWdl.SimpleScatterWdl.asWorkflowSources())
@@ -353,7 +352,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
     }
 
 
-    it should "query a single execution status" in {
+    it should "query a single execution status" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -371,7 +370,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "create and retrieve 3step.wdl with a 10,000 char pattern" in {
+    it should "create and retrieve 3step.wdl with a 10,000 char pattern" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val sampleWdl = SampleWdl.ThreeStepLargeJson
@@ -392,7 +391,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail when saving a workflow twice" in {
+    it should "fail when saving a workflow twice" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -403,7 +402,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(a[SQLException])
     }
 
-    it should "fail when updating a non-existent workflow state" in {
+    it should "fail when updating a non-existent workflow state" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
 
@@ -412,7 +411,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "update and get a workflow state" in {
+    it should "update and get a workflow state" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -433,7 +432,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "get workflow state" in {
+    it should "get workflow state" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -480,7 +479,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
       val callAlias = if (useAlias) Some("call.alias") else None
 
-      it should s"get $spec" in {
+      it should s"get $spec" taggedAs IntegrationTest in {
         assume(canConnect || testRequired)
         val workflowId = WorkflowId(UUID.randomUUID())
         val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -538,7 +537,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       }
     }
 
-    it should "insert a call in execution table" in {
+    it should "insert a call in execution table" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
@@ -562,12 +561,12 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail to get an non-existent execution status" in {
+    it should "fail to get an non-existent execution status" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       dataAccess.getExecutionStatuses(WorkflowId(UUID.randomUUID())).failed.futureValue should be(a[NoSuchElementException])
     }
 
-    it should "get a symbol input" in {
+    it should "get a symbol input" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val symbolFqn = "symbol.fully.qualified.scope"
@@ -598,7 +597,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "get a symbol input that has a very long WDL value field" in {
+    it should "get a symbol input that has a very long WDL value field" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val wdlArray = new WdlArray(WdlArrayType(WdlStringType), Seq(WdlString("test"), WdlString("*" * 10000)))
       val callFqn = "call.fully.qualified.scope"
@@ -630,7 +629,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail to get inputs for a null call" in {
+    it should "fail to get inputs for a null call" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId: WorkflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -642,7 +641,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "set and get an output" in {
+    it should "set and get an output" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val symbolLqn = "symbol"
@@ -650,7 +649,6 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
       val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
-      val workflowOutputList = Seq(ReportableSymbol("call.fully.qualified.scope.symbol", WdlArrayType(WdlStringType)))
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Nil, localBackend)
@@ -672,7 +670,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "set and get shard statuses" in {
+    it should "set and get shard statuses" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn1 = "call.fully.qualified.scope$s1"
       val callFqn2 = "call.fully.qualified.scope$s2"
@@ -705,7 +703,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "set and get an output by call" in {
+    it should "set and get an output by call" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val symbolLqn = "symbol"
@@ -743,7 +741,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail to get outputs for a null call" in {
+    it should "fail to get outputs for a null call" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -755,7 +753,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "fail to create workflow for an unknown backend" in {
+    it should "fail to create workflow for an unknown backend" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
@@ -767,7 +765,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
         UnknownBackend).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "fail to create workflow for a null backend" in {
+    it should "fail to create workflow for a null backend" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
@@ -779,7 +777,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
         null).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "set and get the same symbol with IO as input then output" in {
+    it should "set and get the same symbol with IO as input then output" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val symbolLqn = "symbol"
@@ -812,7 +810,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail when setting an existing symbol output" in {
+    it should "fail when setting an existing symbol output" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val symbolLqn = "symbol"
@@ -831,7 +829,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(a[SQLException])
     }
 
-    it should "set and get a backend info" in {
+    it should "set and get a backend info" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -852,7 +850,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
     }
 
     // Queries use `.head` a lot. There was a bug that pulled the backend info by fqn, but for any workflow.
-    it should "set and get a backend info for same call on two workflows" in {
+    it should "set and get a backend info for same call on two workflows" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId1 = WorkflowId(UUID.randomUUID())
       val workflowId2 = WorkflowId(UUID.randomUUID())
@@ -883,7 +881,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "fail to set null backend info" in {
+    it should "fail to set null backend info" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val workflowId = WorkflowId(UUID.randomUUID())
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
@@ -897,7 +895,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(an[IllegalArgumentException])
     }
 
-    it should "update call start and end dates appropriately" in {
+    it should "update call start and end dates appropriately" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       val callFqn = "call.fully.qualified.scope"
       val workflowId = WorkflowId(UUID.randomUUID())
@@ -930,7 +928,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield()).futureValue
     }
 
-    it should "set and get execution events" in {
+    it should "set and get execution events" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
 
       // We need an execution to create an event. We need a workflow to make an execution. Le Sigh...
@@ -940,7 +938,6 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val shardedCall = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
       val shardIndex = Some(0)
-      val backendInfo = new LocalCallBackendInfo(Option(123))
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Seq(call), localBackend)
@@ -974,7 +971,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).futureValue
     }
 
-    it should "reject a set of execution events without a valid execution to link to" in {
+    it should "reject a set of execution events without a valid execution to link to" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
 
       // We need an execution to create an event. We need a workflow to make an execution. Le Sigh...
@@ -982,7 +979,6 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
       val task = Task.empty
       val call = new Call(None, "fully.qualified.name", task, Set.empty[FullyQualifiedName], Map.empty, None)
-      val backendInfo = new LocalCallBackendInfo(Option(123))
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Seq(call), localBackend)
@@ -992,7 +988,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       } yield ()).failed.futureValue should be(a[NoSuchElementException])
     }
 
-    it should "shutdown the database" in {
+    it should "shutdown the database" taggedAs IntegrationTest in {
       assume(canConnect || testRequired)
       dataAccess.shutdown().futureValue
     }

From 1be827b12e45be4fe8175b0f5c253ad8684e55dd Mon Sep 17 00:00:00 2001
From: Tova Nadler <tovanadler@google.com>
Date: Thu, 21 Jan 2016 12:51:38 -0500
Subject: [PATCH 24/31] Fix typo where we wrote application-default instead of
 application_default

---
 src/main/resources/application.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index 62488393d73..b0217981770 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -96,7 +96,7 @@ google {
   // cromwellAuthenticationScheme can take the values "service_account",
   // "user_account", and "application_default". The first two are described
   // in the stanzas below.
-  // Use "application-default" to use the default service account credentials.
+  // Use "application_default" to use the default service account credentials.
   // This is useful if you are running on a GCE VM and if you don't need
   // user level access. See https://developers.google.com/identity/protocols/application-default-credentials
   // for more info. No further information is needed.

From f5fcde61f89904f3c62104994ff06d4614f700da Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@gmail.com>
Date: Wed, 13 Jan 2016 17:36:17 -0500
Subject: [PATCH 25/31] Refactoring actors for FinalCalls

---
 .../cromwell/engine/CopyWorkflowOutputs.scala |  19 --
 .../cromwell/engine/WorkflowDescriptor.scala  |   2 +-
 .../cromwell/engine/backend/Backend.scala     |   6 +-
 .../cromwell/engine/backend/BackendCall.scala |   9 +-
 .../engine/backend/ExecutionResult.scala      |   4 +-
 .../engine/backend/jes/JesBackend.scala       |  11 +-
 .../engine/backend/jes/JesBackendCall.scala   |  14 +-
 .../engine/backend/jes/Pipeline.scala         |   6 +-
 .../cromwell/engine/backend/jes/Run.scala     |   4 +-
 .../engine/backend/local/LocalBackend.scala   |   6 +-
 .../backend/local/LocalBackendCall.scala      |   4 +-
 .../backend/local/SharedFileSystem.scala      |   7 +-
 .../engine/backend/sge/SgeBackend.scala       |  11 +-
 .../engine/backend/sge/SgeBackendCall.scala   |   4 +-
 .../engine/callactor/BackendCallActor.scala   |  19 ++
 .../engine/{ => callactor}/CallActor.scala    |  42 ++--
 .../engine/callactor/FinalCallActor.scala     |  13 ++
 .../BackendCallExecutionActor.scala           |  27 +++
 .../CallExecutionActor.scala                  |  82 ++++----
 .../FinalCallExecutionActor.scala             |  24 +++
 .../scala/cromwell/engine/db/DataAccess.scala |   6 +-
 .../engine/db/slick/SlickDataAccess.scala     |   7 +-
 .../finalcall/CopyWorkflowOutputsCall.scala   |  36 ++++
 .../engine/{ => finalcall}/FinalCall.scala    |  25 ++-
 .../engine/workflow/ExecutionStoreKey.scala   |  12 +-
 .../engine/workflow/WorkflowActor.scala       | 196 +++++++++---------
 .../workflow/WorkflowManagerActor.scala       |   5 +-
 .../cromwell/CopyWorkflowOutputsSpec.scala    |   2 +-
 .../engine/backend/BackendCallSpec.scala      |   6 +-
 .../engine/backend/jes/JesBackendSpec.scala   |  11 +-
 .../backend/local/LocalBackendSpec.scala      |   7 +-
 .../engine/db/slick/SlickDataAccessSpec.scala |  42 ++--
 .../cromwell/logging/WorkflowLoggerSpec.scala |   4 +-
 33 files changed, 395 insertions(+), 278 deletions(-)
 delete mode 100644 src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
 create mode 100644 src/main/scala/cromwell/engine/callactor/BackendCallActor.scala
 rename src/main/scala/cromwell/engine/{ => callactor}/CallActor.scala (84%)
 create mode 100644 src/main/scala/cromwell/engine/callactor/FinalCallActor.scala
 create mode 100644 src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
 rename src/main/scala/cromwell/engine/{ => callexecution}/CallExecutionActor.scala (55%)
 create mode 100644 src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
 create mode 100644 src/main/scala/cromwell/engine/finalcall/CopyWorkflowOutputsCall.scala
 rename src/main/scala/cromwell/engine/{ => finalcall}/FinalCall.scala (57%)

diff --git a/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala b/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
deleted file mode 100644
index 10c17040a31..00000000000
--- a/src/main/scala/cromwell/engine/CopyWorkflowOutputs.scala
+++ /dev/null
@@ -1,19 +0,0 @@
-package cromwell.engine
-
-
-import wdl4s.Workflow
-
-import scala.concurrent.{ExecutionContext, Future}
-
-object CopyWorkflowOutputs {
-  val Name: LocallyQualifiedName = "$final_call$copy_workflow_outputs"
-}
-
-/**
-  * Final call implementation that copies workflow outputs to a specified destination.
-  */
-case class CopyWorkflowOutputs(workflow: WorkflowDescriptor) extends FinalCall {
-  override def unqualifiedName = CopyWorkflowOutputs.Name
-  override def rootWorkflow: Workflow = workflow.namespace.workflow
-  override def execute(implicit ec: ExecutionContext): Future[Unit] = workflow.copyWorkflowOutputs
-}
diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index 278204e54bc..1908fc11024 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -9,7 +9,7 @@ import ch.qos.logback.core.FileAppender
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.engine.backend.jes.JesBackend
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import cromwell.engine.backend.{Backend, BackendType, CromwellBackend}
+import cromwell.engine.backend.{BackendType, Backend, CromwellBackend}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.io.gcs.{GcsFileSystem, GoogleCloudStorage}
 import cromwell.engine.io.shared.SharedFileSystemIoInterface
diff --git a/src/main/scala/cromwell/engine/backend/Backend.scala b/src/main/scala/cromwell/engine/backend/Backend.scala
index 098fb8e03a9..90d34cdc05d 100644
--- a/src/main/scala/cromwell/engine/backend/Backend.scala
+++ b/src/main/scala/cromwell/engine/backend/Backend.scala
@@ -10,6 +10,8 @@ import cromwell.engine.backend.sge.SgeBackend
 import cromwell.engine.db.ExecutionDatabaseKey
 import cromwell.engine.io.IoInterface
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
+import cromwell.engine.workflow.{BackendCallKey, WorkflowOptions}
+import cromwell.engine.{HostInputs, CallOutputs}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.docker.SprayDockerRegistryApiClient
 import org.slf4j.LoggerFactory
@@ -86,7 +88,7 @@ trait Backend {
    * Return a possibly altered copy of inputs reflecting any localization of input file paths that might have
    * been performed for this `Backend` implementation.
    */
-  def adjustInputPaths(callKey: CallKey,
+  def adjustInputPaths(callKey: BackendCallKey,
                        runtimeAttributes: CromwellRuntimeAttributes,
                        inputs: CallInputs,
                        workflowDescriptor: WorkflowDescriptor): CallInputs
@@ -110,7 +112,7 @@ trait Backend {
    * Essentially turns a Call object + CallInputs into a BackendCall
    */
   def bindCall(workflowDescriptor: WorkflowDescriptor,
-               key: CallKey,
+               key: BackendCallKey,
                locallyQualifiedInputs: CallInputs = Map.empty[String, WdlValue],
                abortRegistrationFunction: Option[AbortRegistrationFunction] = None): BackendCall
 
diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index f7f6329bdaa..9dc2cc7fbb4 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -9,6 +9,11 @@ import cromwell.logging.WorkflowLogger
 import wdl4s._
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.values.WdlValue
+import cromwell.engine.workflow.BackendCallKey
+import cromwell.engine.{ExecutionEventEntry, ExecutionHash, WorkflowDescriptor}
+import cromwell.engine.CallOutputs
+import cromwell.logging.WorkflowLogger
+import cromwell.engine.Hashing._
 
 import scala.concurrent.{ExecutionContext, Future}
 
@@ -60,7 +65,7 @@ final case class CompletedExecutionHandle(override val result: ExecutionResult)
 
 final case class SuccessfulExecutionHandle(outputs: CallOutputs, events: Seq[ExecutionEventEntry], returnCode: Int, hash: ExecutionHash, resultsClonedFrom: Option[BackendCall] = None) extends ExecutionHandle {
   override val isDone = true
-  override val result = SuccessfulExecution(outputs, events, returnCode, hash, resultsClonedFrom)
+  override val result = SuccessfulBackendCallExecution(outputs, events, returnCode, hash, resultsClonedFrom)
 }
 
 final case class FailedExecutionHandle(throwable: Throwable, returnCode: Option[Int] = None) extends ExecutionHandle {
@@ -79,7 +84,7 @@ trait BackendCall {
    * of a BackendCall object that the 'call' would be within the workflow
    */
   def workflowDescriptor: WorkflowDescriptor
-  def key: CallKey
+  def key: BackendCallKey
   def call = key.scope
 
   /**
diff --git a/src/main/scala/cromwell/engine/backend/ExecutionResult.scala b/src/main/scala/cromwell/engine/backend/ExecutionResult.scala
index 230a1bb4282..b69214d8826 100644
--- a/src/main/scala/cromwell/engine/backend/ExecutionResult.scala
+++ b/src/main/scala/cromwell/engine/backend/ExecutionResult.scala
@@ -9,10 +9,12 @@ import cromwell.engine.CallOutputs
  */
 sealed trait ExecutionResult
 
+case object SuccessfulFinalCallExecution extends ExecutionResult
+
 /**
  * A successful execution with resolved outputs.
  */
-final case class SuccessfulExecution(outputs: CallOutputs, executionEvents: Seq[ExecutionEventEntry], returnCode: Int, hash: ExecutionHash, resultsClonedFrom: Option[BackendCall] = None) extends ExecutionResult
+final case class SuccessfulBackendCallExecution(outputs: CallOutputs, executionEvents: Seq[ExecutionEventEntry], returnCode: Int, hash: ExecutionHash, resultsClonedFrom: Option[BackendCall] = None) extends ExecutionResult
 
 /**
  * A user-requested abort of the command.
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index f134079e856..ed09e8fe890 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -5,7 +5,6 @@ import java.net.SocketTimeoutException
 import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorSystem
-import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.api.client.http.HttpResponseException
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
@@ -22,7 +21,7 @@ import cromwell.engine.db.ExecutionDatabaseKey
 import cromwell.engine.db.slick.Execution
 import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs._
-import cromwell.engine.workflow.{CallKey, WorkflowOptions}
+import cromwell.engine.workflow.{BackendCallKey, WorkflowOptions}
 import cromwell.engine.{AbortRegistrationFunction, CallOutput, CallOutputs, HostInputs, _}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.{AggregatedException, TryUtil}
@@ -71,7 +70,7 @@ object JesBackend {
   // Decoration around WorkflowDescriptor to generate bucket names and the like
   implicit class JesWorkflowDescriptor(val descriptor: WorkflowDescriptor)
     extends JesBackend(CromwellBackend.backend().actorSystem) {
-    def callDir(key: CallKey) = callGcsPath(descriptor, key)
+    def callDir(key: BackendCallKey) = callGcsPath(descriptor, key)
   }
 
   /**
@@ -147,7 +146,7 @@ object JesBackend {
     def executionStatus: ExecutionStatus = ExecutionStatus.withName(execution.status)
   }
 
-  def callGcsPath(descriptor: WorkflowDescriptor, callKey: CallKey): String = {
+  def callGcsPath(descriptor: WorkflowDescriptor, callKey: BackendCallKey): String = {
     val shardPath = callKey.index map { i => s"/shard-$i" } getOrElse ""
     val workflowPath = workflowGcsPath(descriptor)
     s"$workflowPath/call-${callKey.scope.unqualifiedName}$shardPath"
@@ -184,7 +183,7 @@ case class JesBackend(actorSystem: ActorSystem)
 
   // FIXME: Add proper validation of jesConf and have it happen up front to provide fail-fast behavior (will do as a separate PR)
 
-  override def adjustInputPaths(callKey: CallKey,
+  override def adjustInputPaths(callKey: BackendCallKey,
                                 runtimeAttributes: CromwellRuntimeAttributes,
                                 inputs: CallInputs,
                                 workflowDescriptor: WorkflowDescriptor): CallInputs = inputs mapValues gcsPathToLocal
@@ -291,7 +290,7 @@ case class JesBackend(actorSystem: ActorSystem)
   override def stdoutStderr(backendCall: BackendCall): CallLogs = backendCall.stdoutStderr
 
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
-                        key: CallKey,
+                        key: BackendCallKey,
                         locallyQualifiedInputs: CallInputs,
                         abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     new JesBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
index 18fe6686cee..ad131c752b6 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
@@ -11,7 +11,7 @@ import cromwell.engine.backend.jes.Run.TerminalRunStatus
 import cromwell.engine.backend.jes.authentication.ProductionJesAuthentication
 import cromwell.engine.backend.{BackendCall, CallLogs, JobKey, _}
 import cromwell.engine.io.gcs.GcsPath
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, CallContext, WorkflowDescriptor, _}
 
 import scala.concurrent.{ExecutionContext, Future}
@@ -20,15 +20,15 @@ import scala.util.{Failure, Success, Try}
 import Hashing._
 
 object JesBackendCall {
-  def jesLogBasename(key: CallKey) = {
+  def jesLogBasename(key: BackendCallKey) = {
     val index = key.index.map(s => s"-$s").getOrElse("")
     s"${key.scope.unqualifiedName}$index"
   }
 
-  def jesLogFilename(key: CallKey) = s"${jesLogBasename(key)}.log"
-  def jesLogStdoutFilename(key: CallKey) = s"${jesLogBasename(key)}-stdout.log"
-  def jesLogStderrFilename(key: CallKey) = s"${jesLogBasename(key)}-stderr.log"
-  def jesReturnCodeFilename(key: CallKey) = s"${jesLogBasename(key)}-rc.txt"
+  def jesLogFilename(key: BackendCallKey) = s"${jesLogBasename(key)}.log"
+  def jesLogStdoutFilename(key: BackendCallKey) = s"${jesLogBasename(key)}-stdout.log"
+  def jesLogStderrFilename(key: BackendCallKey) = s"${jesLogBasename(key)}-stderr.log"
+  def jesReturnCodeFilename(key: BackendCallKey) = s"${jesLogBasename(key)}-rc.txt"
 
   private def jesOutput(callGcsPath: String, filename: String): JesOutput =
     JesOutput(filename, s"$callGcsPath/$filename", localFilePathFromRelativePath(filename))
@@ -36,7 +36,7 @@ object JesBackendCall {
 
 class JesBackendCall(val backend: JesBackend,
                      val workflowDescriptor: WorkflowDescriptor,
-                     val key: CallKey,
+                     val key: BackendCallKey,
                      val locallyQualifiedInputs: CallInputs,
                      val callAbortRegistrationFunction: Option[AbortRegistrationFunction])
   extends BackendCall with ProductionJesAuthentication with LazyLogging {
diff --git a/src/main/scala/cromwell/engine/backend/jes/Pipeline.scala b/src/main/scala/cromwell/engine/backend/jes/Pipeline.scala
index 595908b74cf..6a1926adebe 100644
--- a/src/main/scala/cromwell/engine/backend/jes/Pipeline.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/Pipeline.scala
@@ -6,7 +6,7 @@ import com.typesafe.scalalogging.LazyLogging
 import cromwell.engine.WorkflowDescriptor
 import cromwell.engine.backend.jes.JesBackend._
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.logging.WorkflowLogger
 import org.slf4j.LoggerFactory
 
@@ -15,7 +15,7 @@ import scala.collection.JavaConverters._
 object Pipeline {
   def apply(command: String,
             workflow: WorkflowDescriptor,
-            key: CallKey,
+            key: BackendCallKey,
             runtimeAttributes: CromwellRuntimeAttributes,
             jesParameters: Seq[JesParameter],
             projectId: String,
@@ -66,7 +66,7 @@ case class Pipeline private(command: String,
                             projectId: String,
                             gcsPath: String,
                             workflow: WorkflowDescriptor,
-                            key: CallKey,
+                            key: BackendCallKey,
                             jesParameters: Seq[JesParameter],
                             runtimeInfo: JesRuntimeInfo,
                             genomicsService: Genomics,
diff --git a/src/main/scala/cromwell/engine/backend/jes/Run.scala b/src/main/scala/cromwell/engine/backend/jes/Run.scala
index 1f8cf408f00..e1b49b0b509 100644
--- a/src/main/scala/cromwell/engine/backend/jes/Run.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/Run.scala
@@ -8,7 +8,7 @@ import cromwell.engine.backend.jes.JesBackend.{JesInput, JesOutput, JesParameter
 import cromwell.engine.backend.jes.Run.{Failed, Running, Success, _}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.{JesCallBackendInfo, JesId, JesStatus}
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.logging.WorkflowLogger
 import cromwell.util.google.GoogleScopes
 import org.joda.time.DateTime
@@ -157,7 +157,7 @@ case class Run(runId: String, pipeline: Pipeline, logger: WorkflowLogger) {
 
       // Update the database state:
       val newBackendInfo = JesCallBackendInfo(Option(JesId(runId)), Option(JesStatus(currentStatus.toString)))
-      globalDataAccess.updateExecutionBackendInfo(workflowId, CallKey(call, pipeline.key.index), newBackendInfo)
+      globalDataAccess.updateExecutionBackendInfo(workflowId, BackendCallKey(call, pipeline.key.index), newBackendInfo)
 
       // If this has transitioned to a running or complete state from a state that is not running or complete,
       // register the abort function.
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
index cc9ae71a3f2..9ddab24b4de 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
@@ -12,6 +12,8 @@ import cromwell.engine.backend._
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.{CallStatus, ExecutionDatabaseKey}
 import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
+import cromwell.engine.backend.BackendType
 import cromwell.util.FileUtil._
 import org.slf4j.LoggerFactory
 import wdl4s._
@@ -114,7 +116,7 @@ case class LocalBackend(actorSystem: ActorSystem) extends Backend with SharedFil
   import LocalBackend._
 
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
-                        key: CallKey,
+                        key: BackendCallKey,
                         locallyQualifiedInputs: CallInputs,
                         abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     LocalBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
@@ -210,7 +212,7 @@ case class LocalBackend(actorSystem: ActorSystem) extends Backend with SharedFil
 
       def processSuccess(rc: Int) = {
         postProcess(backendCall) match {
-          case Success(outputs) => backendCall.hash map { h => SuccessfulExecution(outputs, Seq.empty, rc, h) }
+          case Success(outputs) => backendCall.hash map { h => SuccessfulBackendCallExecution(outputs, Seq.empty, rc, h) }
           case Failure(e) =>
             val message = Option(e.getMessage) map { ": " + _ } getOrElse ""
             FailedExecution(new Throwable("Failed post processing of outputs" + message, e)).future
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
index a4f080ba5e8..d4ccaf24c50 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
@@ -3,14 +3,14 @@ package cromwell.engine.backend.local
 import better.files._
 import wdl4s.CallInputs
 import cromwell.engine.backend.{BackendCall, LocalFileSystemBackendCall, _}
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, CallContext, WorkflowDescriptor}
 
 import scala.concurrent.{ExecutionContext, Future}
 
 case class LocalBackendCall(backend: LocalBackend,
                             workflowDescriptor: WorkflowDescriptor,
-                            key: CallKey,
+                            key: BackendCallKey,
                             locallyQualifiedInputs: CallInputs,
                             callAbortRegistrationFunction: Option[AbortRegistrationFunction]) extends BackendCall with LocalFileSystemBackendCall {
   val workflowRootPath = LocalBackend.hostExecutionPath(workflowDescriptor)
diff --git a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
index ecf1390224e..8d84654bfc6 100644
--- a/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
+++ b/src/main/scala/cromwell/engine/backend/local/SharedFileSystem.scala
@@ -12,6 +12,9 @@ import cromwell.engine.io.IoInterface
 import cromwell.engine.io.gcs.{GcsPath, GoogleCloudStorage}
 import cromwell.engine.workflow.{CallKey, WorkflowOptions}
 import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId, _}
+import cromwell.engine.workflow.{BackendCallKey, WorkflowOptions}
+import cromwell.engine.{WorkflowContext, WorkflowDescriptor, WorkflowEngineFunctions, WorkflowId}
+import cromwell.engine._
 import cromwell.util.TryUtil
 import org.apache.commons.io.FileUtils
 import org.apache.commons.lang3.exception.ExceptionUtils
@@ -105,7 +108,7 @@ trait SharedFileSystem {
 
     outputs match {
       case Success(o) =>
-        cachedBackendCall.hash map { h => CompletedExecutionHandle(SuccessfulExecution(o, Seq.empty[ExecutionEventEntry], cachedBackendCall.returnCode.contentAsString.stripLineEnd.toInt, h, Option(cachedBackendCall))) }
+        cachedBackendCall.hash map { h => CompletedExecutionHandle(SuccessfulBackendCallExecution(o, Seq.empty[ExecutionEventEntry], cachedBackendCall.returnCode.contentAsString.stripLineEnd.toInt, h, Option(cachedBackendCall))) }
       case Failure(ex) => FailedExecutionHandle(ex).future
     }
   } flatten
@@ -183,7 +186,7 @@ trait SharedFileSystem {
    *    end up with this implementation and thus use it to satisfy their contract with Backend.
    *    This is yuck-tastic and I consider this a FIXME, but not for this refactor
    */
-  def adjustInputPaths(callKey: CallKey,
+  def adjustInputPaths(callKey: BackendCallKey,
                        runtimeAttributes: CromwellRuntimeAttributes,
                        inputs: CallInputs,
                        workflowDescriptor: WorkflowDescriptor): CallInputs = {
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
index bc05f360623..e69ba56e63c 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
@@ -8,7 +8,7 @@ import cromwell.engine.backend._
 import cromwell.engine.backend.local.{LocalBackend, SharedFileSystem}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.SgeCallBackendInfo
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, _}
 import cromwell.logging.WorkflowLogger
 import cromwell.util.FileUtil._
@@ -28,7 +28,7 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
   override def backendType = BackendType.SGE
 
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
-                        key: CallKey,
+                        key: BackendCallKey,
                         locallyQualifiedInputs: CallInputs,
                         abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall = {
     SgeBackendCall(this, workflowDescriptor, key, locallyQualifiedInputs, abortRegistrationFunction)
@@ -66,7 +66,8 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
   private def statusString(result: ExecutionResult): String = (result match {
       case AbortedExecution => ExecutionStatus.Aborted
       case FailedExecution(_, _) => ExecutionStatus.Failed
-      case SuccessfulExecution(_, _, _, _, _) => ExecutionStatus.Done
+      case SuccessfulBackendCallExecution(_, _, _, _, _) => ExecutionStatus.Done
+      case SuccessfulFinalCallExecution => ExecutionStatus.Done
     }).toString
 
   private def recordDatabaseFailure(logger: WorkflowLogger, status: String, rc: Int): PartialFunction[Throwable, Unit] = {
@@ -76,7 +77,7 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
 
   private def updateSgeJobTable(call: BackendCall, status: String, rc: Option[Int], sgeJobId: Option[Int]): Future[Unit] = {
     val backendInfo = SgeCallBackendInfo(sgeJobId)
-    globalDataAccess.updateExecutionBackendInfo(call.workflowDescriptor.id, CallKey(call.call, call.key.index), backendInfo)
+    globalDataAccess.updateExecutionBackendInfo(call.workflowDescriptor.id, BackendCallKey(call.call, call.key.index), backendInfo)
   }
 
   /** TODO restart isn't currently implemented for SGE, there is probably work that needs to be done here much like
@@ -183,7 +184,7 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
         FailedExecution(new Exception(message), Option(0)).future
       case (r, _) =>
         postProcess(backendCall) match {
-          case Success(callOutputs) => backendCall.hash map { h => SuccessfulExecution(callOutputs, Seq.empty, r, h) }
+          case Success(callOutputs) => backendCall.hash map { h => SuccessfulBackendCallExecution(callOutputs, Seq.empty, r, h) }
           case Failure(e) => FailedExecution(e).future
         }
     }
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
index 496702051a7..b2eed057a27 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
@@ -3,14 +3,14 @@ package cromwell.engine.backend.sge
 import wdl4s.CallInputs
 import cromwell.engine.backend.local.LocalBackend
 import cromwell.engine.backend.{BackendCall, LocalFileSystemBackendCall, _}
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor}
 
 import scala.concurrent.{ExecutionContext, Future}
 
 case class SgeBackendCall(backend: SgeBackend,
                           workflowDescriptor: WorkflowDescriptor,
-                          key: CallKey,
+                          key: BackendCallKey,
                           locallyQualifiedInputs: CallInputs,
                           callAbortRegistrationFunction: Option[AbortRegistrationFunction]) extends BackendCall with LocalFileSystemBackendCall {
   val workflowRootPath = LocalBackend.hostExecutionPath(workflowDescriptor)
diff --git a/src/main/scala/cromwell/engine/callactor/BackendCallActor.scala b/src/main/scala/cromwell/engine/callactor/BackendCallActor.scala
new file mode 100644
index 00000000000..4b1721ee9d0
--- /dev/null
+++ b/src/main/scala/cromwell/engine/callactor/BackendCallActor.scala
@@ -0,0 +1,19 @@
+package cromwell.engine.callactor
+
+import cromwell.engine.callexecution.CallExecutionActor
+import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor}
+import cromwell.engine.backend.Backend
+import cromwell.engine.workflow.BackendCallKey
+import wdl4s._
+
+case class BackendCallActor(override val key: BackendCallKey,
+                            locallyQualifiedInputs: CallInputs,
+                            backend: Backend,
+                            override val workflowDescriptor: WorkflowDescriptor) extends CallActor {
+
+  override lazy val callExecutionActor = {
+    val backendCall = backend.bindCall(workflowDescriptor, key, locallyQualifiedInputs, Option(AbortRegistrationFunction(registerAbortFunction)))
+    val executionActorName = s"CallExecutionActor-${workflowDescriptor.id}-${call.unqualifiedName}"
+    context.actorOf(CallExecutionActor.props(backendCall), executionActorName)
+  }
+}
diff --git a/src/main/scala/cromwell/engine/CallActor.scala b/src/main/scala/cromwell/engine/callactor/CallActor.scala
similarity index 84%
rename from src/main/scala/cromwell/engine/CallActor.scala
rename to src/main/scala/cromwell/engine/callactor/CallActor.scala
index 8f175f1ae48..c2f15c29f21 100644
--- a/src/main/scala/cromwell/engine/CallActor.scala
+++ b/src/main/scala/cromwell/engine/callactor/CallActor.scala
@@ -1,17 +1,19 @@
-package cromwell.engine
+package cromwell.engine.callactor
 
 import akka.actor.FSM.NullFunction
-import akka.actor.{Actor, Cancellable, LoggingFSM, Props}
+import akka.actor._
 import akka.event.Logging
 import com.google.api.client.util.ExponentialBackOff
-import wdl4s._
-import wdl4s.values.WdlValue
-import cromwell.engine.CallActor.{CallActorData, CallActorState}
-import cromwell.engine.CallExecutionActor.CallExecutionActorMessage
+import cromwell.engine._
 import cromwell.engine.backend._
-import cromwell.engine.workflow.{CallKey, WorkflowActor}
+import cromwell.engine.callactor.CallActor.{CallActorState, CallActorData}
+import cromwell.engine.callexecution.CallExecutionActor
+import cromwell.engine.callexecution.CallExecutionActor.CallExecutionActorMessage
+import cromwell.engine.workflow.{FinalCallKey, BackendCallKey, CallKey, WorkflowActor}
 import cromwell.instrumentation.Instrumentation.Monitor
 import cromwell.logging.WorkflowLogger
+import wdl4s.{CallInputs, Scope}
+import wdl4s.values.WdlValue
 
 import scala.concurrent.ExecutionContext
 import scala.concurrent.duration._
@@ -27,11 +29,11 @@ object CallActor {
   case object Start extends StartMode { override val executionMessage = CallExecutionActor.Execute }
   final case class Resume(jobKey: JobKey) extends StartMode { override val executionMessage = CallExecutionActor.Resume(jobKey) }
   final case class UseCachedCall(cachedBackendCall: BackendCall, backendCall: BackendCall) extends StartMode {
-    override val executionMessage = CallExecutionActor.UseCachedCall(cachedBackendCall, backendCall)
+    override val executionMessage = CallExecutionActor.UseCachedCall(cachedBackendCall)
   }
   final case class RegisterCallAbortFunction(abortFunction: AbortFunction) extends CallActorMessage
   case object AbortCall extends CallActorMessage
-  final case class ExecutionFinished(call: Call, executionResult: ExecutionResult) extends CallActorMessage
+  final case class ExecutionFinished(call: Scope, executionResult: ExecutionResult) extends CallActorMessage
 
   sealed trait CallActorState
   case object CallNotStarted extends CallActorState
@@ -72,13 +74,16 @@ object CallActor {
 
   val CallCounter = Monitor.minMaxCounter("calls-running")
 
-  def props(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backend, workflowDescriptor: WorkflowDescriptor): Props =
-    Props(new CallActor(key, locallyQualifiedInputs, backend, workflowDescriptor))
+  def props(key: BackendCallKey, locallyQualifiedInputs: CallInputs, backend: Backend, workflowDescriptor: WorkflowDescriptor): Props =
+    Props(new BackendCallActor(key, locallyQualifiedInputs, backend, workflowDescriptor))
+  def props(key: FinalCallKey) = Props(new FinalCallActor(key))
 }
 
 /** Actor to manage the execution of a single call. */
-class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backend, workflowDescriptor: WorkflowDescriptor)
-  extends LoggingFSM[CallActorState, CallActorData] with CromwellActor {
+trait CallActor extends LoggingFSM[CallActorState, CallActorData] with CromwellActor {
+  def key: CallKey
+  def workflowDescriptor: WorkflowDescriptor
+  protected def callExecutionActor: ActorRef
 
   import CallActor._
 
@@ -111,9 +116,7 @@ class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backe
       // There's no special Retry/Ack handling required for CallStarted message, the WorkflowActor can always
       // handle those immediately.
       context.parent ! WorkflowActor.CallStarted(key)
-      val backendCall = backend.bindCall(workflowDescriptor, key, locallyQualifiedInputs, Option(AbortRegistrationFunction(registerAbortFunction)))
-      val executionActorName = s"CallExecutionActor-${workflowDescriptor.id}-${call.unqualifiedName}"
-      context.actorOf(CallExecutionActor.props(backendCall), executionActorName) ! startMode.executionMessage
+      callExecutionActor ! startMode.executionMessage
       goto(CallRunningAbortUnavailable)
     case Event(AbortCall, _) => handleFinished(call, AbortedExecution)
   }
@@ -179,7 +182,7 @@ class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backe
     }
   }
 
-  private def handleFinished(call: Call, executionResult: ExecutionResult): State = {
+  private def handleFinished(call: Scope, executionResult: ExecutionResult): State = {
 
     def createBackoff: Option[ExponentialBackOff] = Option(
       new ExponentialBackOff.Builder()
@@ -191,8 +194,9 @@ class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backe
     )
 
     val message = executionResult match {
-      case SuccessfulExecution(outputs, executionEvents, returnCode, hash, resultsClonedFrom) =>
+      case SuccessfulBackendCallExecution(outputs, executionEvents, returnCode, hash, resultsClonedFrom) =>
         WorkflowActor.CallCompleted(key, outputs, executionEvents, returnCode, if (workflowDescriptor.writeToCache) Option(hash) else None, resultsClonedFrom)
+      case SuccessfulFinalCallExecution => WorkflowActor.CallCompleted(key, Map.empty, Seq.empty, 0, None, None)
       case AbortedExecution => WorkflowActor.CallAborted(key)
       case FailedExecution(e, returnCode) =>
         logger.error("Failing call: " + e.getMessage, e)
@@ -212,7 +216,7 @@ class CallActor(key: CallKey, locallyQualifiedInputs: CallInputs, backend: Backe
     context.stop(self)
   }
 
-  private def registerAbortFunction(abortFunction: AbortFunction): Unit = {
+  protected def registerAbortFunction(abortFunction: AbortFunction): Unit = {
     self ! CallActor.RegisterCallAbortFunction(abortFunction)
   }
 }
diff --git a/src/main/scala/cromwell/engine/callactor/FinalCallActor.scala b/src/main/scala/cromwell/engine/callactor/FinalCallActor.scala
new file mode 100644
index 00000000000..ad696ad7a0c
--- /dev/null
+++ b/src/main/scala/cromwell/engine/callactor/FinalCallActor.scala
@@ -0,0 +1,13 @@
+package cromwell.engine.callactor
+
+import akka.actor.ActorRef
+import cromwell.engine.WorkflowDescriptor
+import cromwell.engine.callexecution.CallExecutionActor
+import cromwell.engine.workflow.FinalCallKey
+
+case class FinalCallActor(override val key: FinalCallKey) extends CallActor {
+  override protected lazy val callExecutionActor: ActorRef = context.actorOf(CallExecutionActor.props(key.scope))
+
+  // Marked as lazy so that it can be called in the superclass constructor:
+  override lazy val workflowDescriptor: WorkflowDescriptor = key.scope.workflow
+}
diff --git a/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
new file mode 100644
index 00000000000..d21eeea3be3
--- /dev/null
+++ b/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
@@ -0,0 +1,27 @@
+package cromwell.engine.callexecution
+
+import cromwell.engine.backend._
+import cromwell.logging.WorkflowLogger
+
+import scala.language.postfixOps
+import CallExecutionActor._
+
+/**
+  * Actor to manage the execution of a single backend call.
+  * */
+class BackendCallExecutionActor(backendCall: BackendCall) extends CallExecutionActor {
+  override val logger = WorkflowLogger(
+    this.getClass.getSimpleName,
+    backendCall.workflowDescriptor,
+    akkaLogger = Option(akkaLogger),
+    callTag = Option(backendCall.key.tag)
+  )
+
+  override val call = backendCall.call
+  override def poll(handle: ExecutionHandle) = backendCall.poll(handle)
+  override def execute(mode: ExecutionMode) = mode match {
+    case Execute => backendCall.execute
+    case Resume(jobKey) => backendCall.resume(jobKey)
+    case UseCachedCall(cachedBackendCall) => backendCall.useCachedCall(cachedBackendCall)
+  }
+}
diff --git a/src/main/scala/cromwell/engine/CallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
similarity index 55%
rename from src/main/scala/cromwell/engine/CallExecutionActor.scala
rename to src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
index e53c45cf39c..ace0968695d 100644
--- a/src/main/scala/cromwell/engine/CallExecutionActor.scala
+++ b/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
@@ -1,15 +1,18 @@
-package cromwell.engine
+package cromwell.engine.callexecution
 
 import akka.actor.{Actor, Props}
-import akka.event.{Logging, LoggingReceive}
-import com.google.api.client.googleapis.json.GoogleJsonResponseException
+import akka.event.{LoggingAdapter, LoggingReceive, Logging}
 import com.google.api.client.util.ExponentialBackOff
-import cromwell.engine.backend._
+import cromwell.engine.callactor.CallActor
+import cromwell.engine.finalcall.FinalCall
+import cromwell.engine.{CromwellFatalException, CromwellActor}
+import cromwell.engine.backend.{FailedExecutionHandle, BackendCall, JobKey, ExecutionHandle}
+import cromwell.engine.callexecution.CallExecutionActor.{ExecutionMode, PollResponseReceived, IssuePollRequest, Finish}
 import cromwell.logging.WorkflowLogger
-
-import scala.concurrent.duration._
-import scala.concurrent.{ExecutionContext, Future}
+import wdl4s.Scope
+import scala.concurrent.Future
 import scala.language.postfixOps
+import scala.concurrent.duration._
 import scala.util.{Failure, Success}
 
 object CallExecutionActor {
@@ -18,43 +21,27 @@ object CallExecutionActor {
   final case class PollResponseReceived(executionHandle: ExecutionHandle) extends CallExecutionActorMessage
   final case class Finish(executionHandle: ExecutionHandle) extends CallExecutionActorMessage
 
-  sealed trait ExecutionMode extends CallExecutionActorMessage {
-    def execute(backendCall: BackendCall)(implicit ec: ExecutionContext): Future[ExecutionHandle]
-  }
-
-  case object Execute extends ExecutionMode {
-    override def execute(backendCall: BackendCall)(implicit ec: ExecutionContext) = backendCall.execute
-  }
+  sealed trait ExecutionMode extends CallExecutionActorMessage
 
-  final case class Resume(jobKey: JobKey) extends ExecutionMode {
-    override def execute(backendCall: BackendCall)(implicit ec: ExecutionContext) = backendCall.resume(jobKey)
-  }
+  case object Execute extends ExecutionMode
+  final case class Resume(jobKey: JobKey) extends ExecutionMode
+  final case class UseCachedCall(cachedBackendCall: BackendCall) extends ExecutionMode
 
-  final case class UseCachedCall(cachedBackendCall: BackendCall, backendCall: BackendCall) extends ExecutionMode {
-    override def execute(backendCall: BackendCall)(implicit ec: ExecutionContext) = backendCall.useCachedCall(cachedBackendCall)
-  }
-
-  def props(backendCall: BackendCall): Props = Props(new CallExecutionActor(backendCall))
+  def props(backendCall: BackendCall): Props = Props(new BackendCallExecutionActor(backendCall))
+  def props(finalCall: FinalCall): Props = Props(new FinalCallExecutionActor(finalCall))
 }
 
-/** Actor to manage the execution of a single call. */
-class CallExecutionActor(backendCall: BackendCall) extends Actor with CromwellActor {
-  import CallExecutionActor._
+trait CallExecutionActor extends Actor with CromwellActor {
 
   val akkaLogger = Logging(context.system, classOf[CallExecutionActor])
-  val logger = WorkflowLogger(
-    "CallExecutionActor",
-    backendCall.workflowDescriptor,
-    akkaLogger = Option(akkaLogger),
-    callTag = Option(backendCall.key.tag)
-  )
+  def logger: WorkflowLogger
 
   implicit val ec = context.system.dispatcher
 
   /**
-   * Schedule work according to the schedule of the `backoff`.
-   */
-  private def scheduleWork(work: => Unit): Unit = {
+    * Schedule work according to the schedule of the `backoff`.
+    */
+  protected def scheduleWork(work: => Unit): Unit = {
     val interval = backoff.nextBackOffMillis().millis
     context.system.scheduler.scheduleOnce(interval) {
       work
@@ -69,9 +56,9 @@ class CallExecutionActor(backendCall: BackendCall) extends Actor with CromwellAc
     .build()
 
   /**
-   * If the `work` `Future` completes successfully, perform the `onSuccess` work, otherwise schedule
-   * the execution of the `onFailure` work using an exponential backoff.
-   */
+    * If the `work` `Future` completes successfully, perform the `onSuccess` work, otherwise schedule
+    * the execution of the `onFailure` work using an exponential backoff.
+    */
   def withRetry(work: Future[ExecutionHandle], onSuccess: ExecutionHandle => Unit, onFailure: => Unit): Unit = {
     work onComplete {
       case Success(s) => onSuccess(s)
@@ -88,22 +75,35 @@ class CallExecutionActor(backendCall: BackendCall) extends Actor with CromwellAc
     }
   }
 
+  /**
+    * Update the ExecutionHandle
+    */
+  def poll(handle: ExecutionHandle): Future[ExecutionHandle]
+
+  /**
+    * Start the execution. Once the Future resolves, the ExecutionHandle can be used to poll
+    * the state of the execution.
+    */
+  def execute(mode: ExecutionMode): Future[ExecutionHandle]
+  def call: Scope
+
   override def receive = LoggingReceive {
     case mode: ExecutionMode =>
-      withRetry(mode.execute(backendCall),
+      withRetry(execute(mode),
         onSuccess = self ! IssuePollRequest(_),
         onFailure = self ! mode
       )
+
     case IssuePollRequest(handle) =>
-      withRetry(backendCall.poll(handle),
+      withRetry(poll(handle),
         onSuccess = self ! PollResponseReceived(_),
         onFailure = self ! IssuePollRequest(handle)
       )
     case PollResponseReceived(handle) if handle.isDone => self ! Finish(handle)
     case PollResponseReceived(handle) => scheduleWork(self ! IssuePollRequest(handle))
     case Finish(handle) =>
-      context.parent ! CallActor.ExecutionFinished(backendCall.call, handle.result)
+      context.parent ! CallActor.ExecutionFinished(call, handle.result)
       context.stop(self)
     case badMessage => logger.error(s"Unexpected message $badMessage.")
   }
-}
+}
\ No newline at end of file
diff --git a/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
new file mode 100644
index 00000000000..0decd8118c4
--- /dev/null
+++ b/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
@@ -0,0 +1,24 @@
+package cromwell.engine.callexecution
+
+import cromwell.engine.backend.ExecutionHandle
+import cromwell.engine.callexecution.CallExecutionActor._
+import cromwell.engine.finalcall.FinalCall
+import cromwell.logging.WorkflowLogger
+
+import scala.concurrent.Future
+
+case class FinalCallExecutionActor(override val call: FinalCall) extends CallExecutionActor {
+
+  override val logger = WorkflowLogger(
+    this.getClass.getSimpleName,
+    call.workflow,
+    akkaLogger = Option(akkaLogger),
+    callTag = Option(call.unqualifiedName)
+  )
+
+  override def poll(handle: ExecutionHandle) = call.poll(ec, handle)
+  override def execute(mode: ExecutionMode) = mode match {
+    case UseCachedCall(cachedBackendCall) => Future.failed(new UnsupportedOperationException("Cannot use cached results for a FinalCall"))
+    case _ => call.execute
+  }
+}
diff --git a/src/main/scala/cromwell/engine/db/DataAccess.scala b/src/main/scala/cromwell/engine/db/DataAccess.scala
index 0ae8e3d3459..384e11ef2aa 100644
--- a/src/main/scala/cromwell/engine/db/DataAccess.scala
+++ b/src/main/scala/cromwell/engine/db/DataAccess.scala
@@ -6,7 +6,7 @@ import cromwell.engine.ExecutionStatus.ExecutionStatus
 import cromwell.engine._
 import cromwell.engine.backend.{Backend, JobKey}
 import cromwell.engine.db.slick._
-import cromwell.engine.workflow.{CallKey, ExecutionStoreKey, OutputKey}
+import cromwell.engine.workflow.{BackendCallKey, ExecutionStoreKey, OutputKey}
 import cromwell.webservice.{CallCachingParameters, WorkflowQueryParameters, WorkflowQueryResponse}
 
 import scala.concurrent.Future
@@ -35,7 +35,7 @@ trait DataAccess {
 
   def getExecutionBackendInfo(workflowId: WorkflowId, call: Call): Future[CallBackendInfo]
 
-  def updateExecutionBackendInfo(workflowId: WorkflowId, callKey: CallKey, backendInfo: CallBackendInfo): Future[Unit]
+  def updateExecutionBackendInfo(workflowId: WorkflowId, callKey: BackendCallKey, backendInfo: CallBackendInfo): Future[Unit]
 
   def updateWorkflowState(workflowId: WorkflowId, workflowState: WorkflowState): Future[Unit]
 
@@ -60,7 +60,7 @@ trait DataAccess {
   def setOutputs(workflowId: WorkflowId, key: OutputKey, callOutputs: WorkflowOutputs, workflowOutputFqns: Seq[ReportableSymbol]): Future[Unit]
 
   /** Updates the existing input symbols to replace expressions with real values **/
-  def updateCallInputs(workflowId: WorkflowId, key: CallKey, callInputs: CallInputs): Future[Int]
+  def updateCallInputs(workflowId: WorkflowId, key: BackendCallKey, callInputs: CallInputs): Future[Int]
 
   def setExecutionEvents(workflowId: WorkflowId, callFqn: String, shardIndex: Option[Int], events: Seq[ExecutionEventEntry]): Future[Unit]
 
diff --git a/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala b/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
index ed3420ead5f..d3009fe5d07 100644
--- a/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
+++ b/src/main/scala/cromwell/engine/db/slick/SlickDataAccess.scala
@@ -7,6 +7,7 @@ import javax.sql.rowset.serial.SerialClob
 import _root_.slick.backend.DatabaseConfig
 import _root_.slick.driver.JdbcProfile
 import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
+import cromwell.engine.finalcall.FinalCall
 import wdl4s._
 import wdl4s.types.{WdlPrimitiveType, WdlType}
 import wdl4s.values.WdlValue
@@ -185,7 +186,7 @@ class SlickDataAccess(databaseConfig: Config) extends DataAccess {
                               backend: Backend): Future[Unit] = {
 
     val scopeKeys: Traversable[ExecutionStoreKey] = scopes collect {
-      case call: Call => CallKey(call, None)
+      case call: Call => BackendCallKey(call, None)
       case scatter: Scatter => ScatterKey(scatter, None)
       case finalCall: FinalCall => FinalCallKey(finalCall)
     }
@@ -445,7 +446,7 @@ class SlickDataAccess(databaseConfig: Config) extends DataAccess {
   }
 
   override def updateExecutionBackendInfo(workflowId: WorkflowId,
-                                          callKey: CallKey,
+                                          callKey: BackendCallKey,
                                           backendInfo: CallBackendInfo): Future[Unit] = {
     require(backendInfo != null, "backend info is null")
 
@@ -561,7 +562,7 @@ class SlickDataAccess(databaseConfig: Config) extends DataAccess {
     * Updates the existing input symbols to replace expressions with real values.
     * @return The number of rows updated - as a Future.
     */
-  override def updateCallInputs(workflowId: WorkflowId, key: CallKey, callInputs: CallInputs): Future[Int] = {
+  override def updateCallInputs(workflowId: WorkflowId, key: BackendCallKey, callInputs: CallInputs): Future[Int] = {
     type ProjectionFunction = SlickDataAccess.this.dataAccess.Symbols => (Rep[String], Rep[Option[Clob]])
     val projectionFn: ProjectionFunction = (s: SlickDataAccess.this.dataAccess.Symbols) => (s.wdlType, s.wdlValue)
 
diff --git a/src/main/scala/cromwell/engine/finalcall/CopyWorkflowOutputsCall.scala b/src/main/scala/cromwell/engine/finalcall/CopyWorkflowOutputsCall.scala
new file mode 100644
index 00000000000..fec815c9b54
--- /dev/null
+++ b/src/main/scala/cromwell/engine/finalcall/CopyWorkflowOutputsCall.scala
@@ -0,0 +1,36 @@
+package cromwell.engine.finalcall
+
+import cromwell.engine._
+import cromwell.engine.backend.{SuccessfulFinalCallExecution, ExecutionResult, ExecutionHandle}
+import cromwell.engine.workflow.{FinalCallKey, ExecutionStoreKey}
+import wdl4s.Workflow
+
+import scala.concurrent.{ExecutionContext, Future}
+
+object CopyWorkflowOutputsCall {
+  val UnqualifiedName: LocallyQualifiedName = "$final_call$copy_workflow_outputs"
+  def name(workflowName: String): LocallyQualifiedName = workflowName + ".$final_call$copy_workflow_outputs"
+
+  def isCopyWorkflowOutputsCallFqn(fqn: String) = fqn.endsWith(UnqualifiedName)
+  def storeKey(workflow: WorkflowDescriptor): ExecutionStoreKey = FinalCallKey(CopyWorkflowOutputsCall(workflow))
+}
+
+/**
+  * Final call implementation that copies workflow outputs to a specified destination.
+  */
+case class CopyWorkflowOutputsCall(override val workflow: WorkflowDescriptor) extends FinalCall {
+  override def unqualifiedName = CopyWorkflowOutputsCall.name(workflow.name)
+  override def rootWorkflow: Workflow = workflow.namespace.workflow
+  override def execute(implicit ec: ExecutionContext): Future[ExecutionHandle] = workflow.copyWorkflowOutputs map {
+    _ => CopyWorkflowOutputsHandle
+  }
+  override def poll(implicit ec: ExecutionContext, current: ExecutionHandle): Future[ExecutionHandle] = current match {
+    case CopyWorkflowOutputsHandle => Future.successful(CopyWorkflowOutputsHandle)
+    case _ => Future.failed(new IllegalStateException("CopyWorkflowOutputsCall can only be polled with a CopyWorkflowOutputsHandle."))
+  }
+}
+
+case object CopyWorkflowOutputsHandle extends ExecutionHandle {
+  override val isDone = true
+  override val result = SuccessfulFinalCallExecution
+}
diff --git a/src/main/scala/cromwell/engine/FinalCall.scala b/src/main/scala/cromwell/engine/finalcall/FinalCall.scala
similarity index 57%
rename from src/main/scala/cromwell/engine/FinalCall.scala
rename to src/main/scala/cromwell/engine/finalcall/FinalCall.scala
index 14566e24156..38aa80ff8f7 100644
--- a/src/main/scala/cromwell/engine/FinalCall.scala
+++ b/src/main/scala/cromwell/engine/finalcall/FinalCall.scala
@@ -1,27 +1,30 @@
-package cromwell.engine
+package cromwell.engine.finalcall
 
-import cromwell.engine.workflow.{ExecutionStoreKey, FinalCallKey}
+import cromwell.engine._
+import cromwell.engine.backend.ExecutionHandle
+import cromwell.engine.workflow.ExecutionStoreKey
 import wdl4s.Scope
 
 import scala.concurrent.{ExecutionContext, Future}
 
 object FinalCall {
+
   implicit class FinalCallString(val fqn: FullyQualifiedName) extends AnyVal {
     /** Does this FQN conform to a final call? */
-    def isFinalCall = fqn startsWith "$final_call"
-    /** Convert this FQN to a `FinalCall` or throw if it can't be converted. */
-    def toStoreKey(workflow: WorkflowDescriptor): ExecutionStoreKey = {
-      fqn match {
-        case CopyWorkflowOutputs.Name => FinalCallKey(CopyWorkflowOutputs(workflow))
-        case _ => throw new IllegalArgumentException(s"Unrecognized call key $fqn")
-      }
+    def isFinalCall = fqn contains "$final_call"
+
+    def storeKey(workflow: WorkflowDescriptor): ExecutionStoreKey = {
+      if (CopyWorkflowOutputsCall.isCopyWorkflowOutputsCallFqn(fqn)) { CopyWorkflowOutputsCall.storeKey(workflow) }
+      else throw new IllegalArgumentException(s"Cannot convert $fqn into a FinalCall store key for $workflow")
     }
   }
 }
 
 /** Scope representing a "final call" that is inserted after all other workflow executions. */
 trait FinalCall extends Scope {
-  def execute(implicit ec: ExecutionContext): Future[Unit]
+  def workflow: WorkflowDescriptor
+  def poll(implicit ec: ExecutionContext, executionHandle: ExecutionHandle): Future[ExecutionHandle]
+  def execute(implicit ec: ExecutionContext): Future[ExecutionHandle]
 
   def prerequisiteCallNames: Set[LocallyQualifiedName] = throw new UnsupportedOperationException("prerequisiteCallNames not supported for FinalCalls")
 
@@ -33,5 +36,5 @@ trait FinalCall extends Scope {
     */
   def prerequisiteScopes: Set[Scope] = rootWorkflow.children.toSet
 
-  val parent: Option[Scope] = None
+  val parent: Option[Scope] = Option(this.rootWorkflow)
 }
diff --git a/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala b/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
index 9363d170717..a023bc4f8cc 100644
--- a/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
+++ b/src/main/scala/cromwell/engine/workflow/ExecutionStoreKey.scala
@@ -1,7 +1,8 @@
 package cromwell.engine.workflow
 
+import cromwell.engine.finalcall.FinalCall
 import wdl4s._
-import cromwell.engine.{FinalCall, ExecutionStatus}
+import cromwell.engine.ExecutionStatus
 import cromwell.engine.workflow.WorkflowActor.ExecutionStore
 
 import scala.language.postfixOps
@@ -17,7 +18,8 @@ sealed trait ExecutionStoreKey {
 
 trait OutputKey extends ExecutionStoreKey
 
-case class CallKey(scope: Call, index: Option[Int]) extends OutputKey
+trait CallKey extends OutputKey
+case class BackendCallKey(scope: Call, index: Option[Int]) extends CallKey
 case class CollectorKey(scope: Call) extends OutputKey {
   override val index: Option[Int] = None
 }
@@ -37,7 +39,7 @@ case class ScatterKey(scope: Scatter, index: Option[Int]) extends ExecutionStore
   private def explode(scope: Scope, count: Int): Seq[ExecutionStoreKey] = {
     scope match {
       case call: Call =>
-        val shards = (0 until count) map { i => CallKey(call, Option(i)) }
+        val shards = (0 until count) map { i => BackendCallKey(call, Option(i)) }
         shards :+ CollectorKey(call)
       case scatter: Scatter =>
         throw new UnsupportedOperationException("Nested Scatters are not supported (yet).")
@@ -47,6 +49,6 @@ case class ScatterKey(scope: Scatter, index: Option[Int]) extends ExecutionStore
   }
 }
 
-case class FinalCallKey(scope: FinalCall) extends OutputKey {
-  override def index: Option[Int] = None
+case class FinalCallKey(scope: FinalCall) extends CallKey {
+  override val index = None
 }
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 4831f90500a..625962abd21 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -5,11 +5,13 @@ import java.sql.SQLException
 import akka.actor.{ActorRef, FSM, LoggingFSM, Props}
 import akka.event.Logging
 import akka.pattern.pipe
+import cromwell.engine.callactor.CallActor
+import cromwell.engine.finalcall.{CopyWorkflowOutputsCall, FinalCall}
 import wdl4s._
 import wdl4s.expression.NoFunctions
 import wdl4s.types.WdlArrayType
 import wdl4s.values.{WdlArray, WdlCallOutputsObject, WdlValue}
-import cromwell.engine.CallActor.CallActorMessage
+import CallActor.CallActorMessage
 import cromwell.engine.ExecutionIndex._
 import cromwell.engine.ExecutionStatus.ExecutionStatus
 import cromwell.engine.Hashing._
@@ -82,7 +84,7 @@ object WorkflowActor {
   }
 
   /** This signifies using an existing previously run call to fulfill the results of the callKey. */
-  final case class UseCachedCall(override val callKey: CallKey,
+  final case class UseCachedCall(override val callKey: BackendCallKey,
                                  override val startMode: CallActor.UseCachedCall) extends CallStartMessage {
 
     // Nothing to do here, startRunnableCalls will have already done this work.
@@ -135,9 +137,9 @@ object WorkflowActor {
 
     override def start(actor: WorkflowActor) = {
 
-      def filterResumableCallKeys(resumableExecutionsAndJobIds: Map[ExecutionDatabaseKey, JobKey]): Traversable[CallKey] = {
+      def filterResumableCallKeys(resumableExecutionsAndJobIds: Map[ExecutionDatabaseKey, JobKey]): Traversable[BackendCallKey] = {
         actor.executionStore.keys.collect {
-          case callKey: CallKey if resumableExecutionsAndJobIds.contains(callKey.toDatabaseKey) => callKey }
+          case callKey: BackendCallKey if resumableExecutionsAndJobIds.contains(callKey.toDatabaseKey) => callKey }
       }
 
       val resumptionWork = for {
@@ -208,7 +210,7 @@ object WorkflowActor {
 
   def isTerminal(status: ExecutionStatus): Boolean = TerminalStates contains status
   def isDone(entry: ExecutionStoreEntry): Boolean = entry._2 == ExecutionStatus.Done
-  def isShard(key: CallKey): Boolean = key.index.isDefined
+  def isShard(key: BackendCallKey): Boolean = key.index.isDefined
 
   val WorkflowCounter = Monitor.minMaxCounter("workflows-running")
   val WorkflowDurationTimer = Monitor.histogram("workflow-duration")
@@ -223,8 +225,8 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   def createWorkflow(inputs: HostInputs): Future[Unit] = {
     val symbolStoreEntries = buildSymbolStoreEntries(workflow, inputs)
     symbolCache = symbolStoreEntries.groupBy(entry => SymbolCacheKey(entry.scope, entry.isInput))
-    // Currently assumes there is at most one possible final call, a `CopyWorkflowOutputs`.
-    val finalCall = workflow.workflowOutputsPath.toOption map { _ => CopyWorkflowOutputs(workflow) }
+    // Currently assumes there is at most one possible final call, a `CopyWorkflowOutputsCall`.
+    val finalCall = workflow.workflowOutputsPath.toOption map { _ => CopyWorkflowOutputsCall(workflow) }
     globalDataAccess.createWorkflow(
       workflow, symbolStoreEntries, workflow.namespace.workflow.children ++ finalCall, backend)
   }
@@ -243,7 +245,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
    * Try to generate output for a collector call, by collecting outputs for all of its shards.
    * It's fail-fast on shard output retrieval
    */
-  private def generateCollectorOutput(collector: CollectorKey, shards: Iterable[CallKey]): Try[CallOutputs] = Try {
+  private def generateCollectorOutput(collector: CollectorKey, shards: Iterable[BackendCallKey]): Try[CallOutputs] = Try {
     val shardsOutputs = shards.toSeq sortBy { _.index.fromIndex } map { e =>
       fetchCallOutputEntries(e) map { _.outputs } getOrElse(throw new RuntimeException(s"Could not retrieve output for shard ${e.scope} #${e.index}"))
     }
@@ -255,7 +257,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   }
 
   private def findShardEntries(key: CollectorKey): Iterable[ExecutionStoreEntry] = executionStore collect {
-    case (k: CallKey, v) if k.scope == key.scope && isShard(k) => (k, v)
+    case (k: BackendCallKey, v) if k.scope == key.scope && isShard(k) => (k, v)
   }
 
   /** Messages self to perform the requested transition once all persistence is complete. */
@@ -410,31 +412,11 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     }
   }
 
-  private def handleFinalCallStarted(finalCallKey: FinalCallKey): State = {
-    executionStore += finalCallKey -> ExecutionStatus.Running
-    val finalCallWork = for {
-      _ <- persistStatus(finalCallKey, ExecutionStatus.Running)
-      _ <- finalCallKey.scope.execute
-      _ = self ! CallCompleted(finalCallKey, callOutputs = Map.empty, executionEvents = Seq.empty, returnCode = 0, hash = None, resultsClonedFrom = None)
-    } yield ()
-
-    finalCallWork onFailure {
-      case t =>
-        log.error("Final call work failed", t)
-        scheduleTransition(WorkflowFailed)
-    }
-
-    val updatedData = stateData.addPersisting(finalCallKey, ExecutionStatus.Running)
-    stay() using updatedData
-  }
-
   when(WorkflowRunning) {
     case Event(StartRunnableCalls, data) =>
       val updatedData = startRunnableCalls(data)
       if (isWorkflowDone) scheduleTransition(WorkflowSucceeded)
       stay() using updatedData
-    case Event(CallStarted(finalCallKey: FinalCallKey), _) =>
-      handleFinalCallStarted(finalCallKey)
     case Event(CallStarted(callKey), data) if !data.isPending(callKey) =>
       executionStore += callKey -> ExecutionStatus.Running
       persistStatus(callKey, ExecutionStatus.Running)
@@ -443,12 +425,6 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     case Event(message: CallStarted, _) =>
       resendDueToPendingExecutionWrites(message)
       stay()
-    case Event(message @ CallCompleted(finalCallKey: FinalCallKey, _, _, _, _, _), data) if !data.isPersistedRunning(finalCallKey) =>
-      // Unlike "real" calls which are managed by a CallActor, the final call completion message is triggered
-      // internally and there isn't an automatic retry if the message isn't processed.  So if the message can't be
-      // processed due to pending writes, schedule it to be resent here.
-      resendDueToPendingExecutionWrites(message)
-      stay()
     case Event(message @ CallCompleted(callKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if data.isPersistedRunning(callKey) =>
       handleCallCompleted(callKey, outputs, executionEvents, returnCode, message, hash, resultsClonedFrom, data)
     case Event(message @ CallCompleted(collectorKey: CollectorKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if !data.isPending(collectorKey) =>
@@ -473,25 +449,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       stay() using updatedData
     case Event(message: CallStartMessage, data) if !data.isPending(message.callKey) =>
       val updatedData = message.handleStatusPersist(this, data)
-      val callKey = message.callKey
-      fetchLocallyQualifiedInputs(callKey) match {
-        case Success(callInputs) =>
-          // TODO: The block of code is only run on non-shards because of issues with DSDEEPB-2490
-          if (!callKey.index.isShard) {
-            val updateDbCallInputs = globalDataAccess.updateCallInputs(workflow.id, callKey, callInputs)
-            updateDbCallInputs onComplete {
-              case Success(i) =>
-                logger.debug(s"$i call input expression(s) updated in database.")
-                startActor(callKey, callInputs, message.startMode)
-              case Failure(e) => self ! AsyncFailure(new SQLException(s"Failed to update symbol inputs for ${callKey.scope.fullyQualifiedName}.${callKey.tag}.${callKey.index}", e))
-            }
-          } else {
-            startActor(callKey, callInputs, message.startMode)
-          }
-        case Failure(t) =>
-          logger.error(s"Failed to fetch locally qualified inputs for call ${callKey.tag}", t)
-          scheduleTransition(WorkflowFailed)
-      }
+      startCallWithMessage(message)
       stay() using updatedData
     case Event(message: CallStartMessage, _) =>
       resendDueToPendingExecutionWrites(message)
@@ -506,6 +464,34 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
       stay using finalData
   }
 
+  private def startBackendCallWithMessage(message: CallStartMessage, backendCallKey: BackendCallKey, callInputs: Map[String, WdlValue]) = {
+      // TODO: The block of code is only run on non-shards because of issues with DSDEEPB-2490
+      if (!backendCallKey.index.isShard) {
+        val updateDbCallInputs = globalDataAccess.updateCallInputs(workflow.id, backendCallKey, callInputs)
+        updateDbCallInputs onComplete {
+          case Success(i) =>
+            logger.debug(s"$i call input expression(s) updated in database.")
+            startActor(backendCallKey, callInputs, message.startMode)
+          case Failure(e) => self ! AsyncFailure(new SQLException(s"Failed to update symbol inputs for ${backendCallKey.scope.fullyQualifiedName}.${backendCallKey.tag}.${backendCallKey.index}", e))
+        }
+      } else {
+        startActor(backendCallKey, callInputs, message.startMode)
+      }
+  }
+
+  private def startCallWithMessage(message: CallStartMessage) = {
+    val callKey = message.callKey
+    callKey match {
+      case backendCallKey: BackendCallKey => fetchLocallyQualifiedInputs(backendCallKey) match {
+        case Success(callInputs) => startBackendCallWithMessage(message, backendCallKey, callInputs)
+        case Failure(t) =>
+          logger.error(s"Failed to fetch locally qualified inputs for call ${backendCallKey.tag}", t)
+          scheduleTransition(WorkflowFailed)
+      }
+      case finalCallKey: FinalCallKey => startActor(finalCallKey, Map.empty, message.startMode)
+    }
+  }
+
   when(WorkflowSucceeded) { FSM.NullFunction }
 
   when(WorkflowFailed) { FSM.NullFunction }
@@ -518,12 +504,6 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   }
 
   when(WorkflowAborting) {
-    case Event(message @ CallCompleted(finalCallKey: FinalCallKey, _, _, _, _, _), data) if !data.isPersistedRunning(finalCallKey) =>
-      // Unlike "real" calls which are managed by a CallActor, the final call completion message is triggered
-      // internally and there isn't an automatic retry if the message isn't processed.  So if the message can't be
-      // processed due to pending writes, schedule it to be resent here.
-      resendDueToPendingExecutionWrites(message)
-      stay()
     case Event(message @ CallCompleted(callKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if data.isPersistedRunning(callKey) =>
       handleCallCompleted(callKey, outputs, executionEvents, returnCode, message, hash, resultsClonedFrom, data)
     case Event(message @ CallCompleted(collectorKey: CollectorKey, outputs, executionEvents, returnCode, hash, resultsClonedFrom), data) if !data.isPending(collectorKey) =>
@@ -666,7 +646,10 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     }
 
     val callActorName = s"CallActor-${workflow.id}-${callKey.tag}"
-    val callActorProps = CallActor.props(callKey, locallyQualifiedInputs, backend, workflow)
+    val callActorProps = callKey match {
+      case backendCallKey: BackendCallKey => CallActor.props(backendCallKey, locallyQualifiedInputs, backend, workflow)
+      case finalCallKey: FinalCallKey => CallActor.props(finalCallKey)
+    }
     val callActor = context.actorOf(callActorProps, callActorName)
     callActor ! callActorMessage
     logger.info(s"created call actor for ${callKey.tag}.")
@@ -722,15 +705,15 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
 
     val runnableEntries = executionStore filter isRunnable
 
-    val runnableCalls = runnableEntries collect { case(k: CallKey, v) => k.scope }
+    val runnableCalls = runnableEntries collect { case(k: BackendCallKey, v) => k.scope }
     if (runnableCalls.nonEmpty)
       logger.info(s"starting calls: " + runnableCalls.map(_.fullyQualifiedName).toSeq.sorted.mkString(", "))
 
     val entries: Traversable[Try[ExecutionStartResult]] = runnableEntries map {
       case (k: ScatterKey, _) => processRunnableScatter(k)
       case (k: CollectorKey, _) => processRunnableCollector(k)
-      case (k: CallKey, _) => processRunnableCall(k)
-      case (k: FinalCallKey, _) => processRunnableFinalCall(k)
+      case (k: BackendCallKey, _) => processRunnableCall(k)
+      case (k: FinalCallKey, _) => processRunnableCall(k)
       case (k, v) =>
         val message = s"Unknown entry in execution store:\nKEY: $k\nVALUE:$v"
         logger.error(message)
@@ -782,7 +765,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     }
   }
 
-  private def lookupScatterVariable(callKey: CallKey, workflow: Workflow)(name: String): Try[WdlValue] = {
+  private def lookupScatterVariable(callKey: BackendCallKey, workflow: Workflow)(name: String): Try[WdlValue] = {
     val scatterBlock = callKey.scope.ancestry collect { case s: Scatter => s } find { _.item == name }
     val scatterCollection = scatterBlock map { s =>
       s.collection.evaluate(scatterCollectionLookupFunction(workflow, callKey), NoFunctions) match {
@@ -815,7 +798,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     }).headOption
   }
 
-  def fetchLocallyQualifiedInputs(callKey: CallKey): Try[Map[String, WdlValue]] = Try {
+  def fetchLocallyQualifiedInputs(callKey: BackendCallKey): Try[Map[String, WdlValue]] = Try {
     val parentWorkflow = callKey.scope.rootWorkflow
 
     def lookup(identifier: String): WdlValue = {
@@ -909,18 +892,18 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   private def createCaches: Future[(ExecutionStore, SymbolCache)] = {
 
     def isInScatterBlock(c: Call) = c.ancestry.exists(_.isInstanceOf[Scatter])
-    import FinalCall._
+    import FinalCall.FinalCallString
 
     val futureExecutionCache = globalDataAccess.getExecutionStatuses(workflow.id) map { statuses =>
       statuses map { case (k, v) =>
         val key: ExecutionStoreKey = if (k.fqn.isFinalCall) {
           // Final calls are not part of the workflow namespace, handle these differently from other keys.
-          k.fqn.toStoreKey(workflow)
+          k.fqn.storeKey(workflow)
         } else {
           (workflow.namespace.resolve(k.fqn), k.index) match {
-            case (Some(c: Call), Some(i)) => CallKey(c, Option(i))
+            case (Some(c: Call), Some(i)) => BackendCallKey(c, Option(i))
             case (Some(c: Call), None) if isInScatterBlock(c) => CollectorKey(c)
-            case (Some(c: Call), None) => CallKey(c, None)
+            case (Some(c: Call), None) => BackendCallKey(c, None)
             case (Some(s: Scatter), None) => ScatterKey(s, None)
             case _ => throw new UnsupportedOperationException(s"Execution entry invalid: $k -> $v")
           }
@@ -1010,7 +993,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
   private def processRunnableCollector(collector: CollectorKey): Try[ExecutionStartResult] = {
     executionStore += collector -> ExecutionStatus.Starting
     persistStatus(collector, ExecutionStatus.Starting)
-    val shards: Iterable[CallKey] = findShardEntries(collector) collect { case (k: CallKey, _) => k }
+    val shards: Iterable[BackendCallKey] = findShardEntries(collector) collect { case (k: BackendCallKey, _) => k }
 
     generateCollectorOutput(collector, shards) match {
       case Failure(e) =>
@@ -1023,14 +1006,9 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     Success(ExecutionStartResult(Set(StartEntry(collector, ExecutionStatus.Starting))))
   }
 
-  private def processRunnableFinalCall(finalCallKey: FinalCallKey): Try[ExecutionStartResult] = {
-    executionStore += finalCallKey -> ExecutionStatus.Starting
-    persistStatus(finalCallKey, ExecutionStatus.Starting) map { _ => self ! CallStarted(finalCallKey) }
-    Success(ExecutionStartResult(Set(StartEntry(finalCallKey, ExecutionStatus.Starting))))
-  }
-
-  private def sendStartMessage(callKey: CallKey, callInputs: Map[String, WdlValue]) = {
+  private def sendStartMessage(callKey: BackendCallKey, callInputs: Map[String, WdlValue]) = {
     def registerAbortFunction(abortFunction: AbortFunction): Unit = {}
+
     val backendCall = backend.bindCall(workflow, callKey, callInputs, Option(AbortRegistrationFunction(registerAbortFunction)))
     val log = backendCall.workflowLoggerWithCall("WorkflowActor", Option(akkaLogger))
 
@@ -1039,7 +1017,7 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
         case Some(c: Call) =>
           val cachedCall = backend.bindCall(
             descriptor,
-            CallKey(c, cachedExecution.index.toIndex),
+            BackendCallKey(c, cachedExecution.index.toIndex),
             callInputs,
             Option(AbortRegistrationFunction(registerAbortFunction))
           )
@@ -1059,28 +1037,35 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
         self ! InitialStartCall(callKey, CallActor.Start)
     }
 
+    def startCachedCall(cachedExecution: Execution) = {
+      globalDataAccess.getWorkflow(cachedExecution.workflowExecutionId) onComplete { cachedDescriptor =>
+        loadCachedCallOrInitiateCall(cachedDescriptor, cachedExecution)
+      }
+    }
+
+    def checkCacheAndStartCall = {
+      log.debug(s"Call caching 'readFromCache' is turned on. Checking cache before starting call")
+      backendCall.hash map { hash =>
+        globalDataAccess.getExecutionsWithResuableResultsByHash(hash.overallHash) onComplete {
+          case Success(executions) if executions.nonEmpty => startCachedCall(executions.head)
+          case Success(_) =>
+            log.info(s"Call Caching: cache miss")
+            self ! InitialStartCall(callKey, CallActor.Start)
+          case Failure(ex) =>
+            log.error(s"Call Caching: Failed to look up executions that matched hash '$hash'. Falling back to normal execution", ex)
+            self ! InitialStartCall(callKey, CallActor.Start)
+        }
+      } recover { case e =>
+        log.error(s"Failed to calculate hash for call '${backendCall.key.tag}'.", e)
+        scheduleTransition(WorkflowFailed)
+      }
+    }
+
     def startCall = {
       if (backendCall.workflowDescriptor.readFromCache) {
-        backendCall.hash map { hash =>
-          globalDataAccess.getExecutionsWithResuableResultsByHash(hash.overallHash) onComplete {
-            case Success(executions) if executions.nonEmpty =>
-              val cachedExecution = executions.head
-              globalDataAccess.getWorkflow(cachedExecution.workflowExecutionId) onComplete { cachedDescriptor =>
-                loadCachedCallOrInitiateCall(cachedDescriptor, cachedExecution)
-              }
-            case Success(_) =>
-              log.info(s"Call Caching: cache miss")
-              self ! InitialStartCall(callKey, CallActor.Start)
-            case Failure(ex) =>
-              log.error(s"Call Caching: Failed to look up executions that matched hash '$hash'. Falling back to normal execution", ex)
-              self ! InitialStartCall(callKey, CallActor.Start)
-          }
-        } recover { case e =>
-          log.error(s"Failed to calculate hash for call '${backendCall.key.tag}'.", e)
-          scheduleTransition(WorkflowFailed)
-        }
+        checkCacheAndStartCall
       } else {
-        log.info(s"Call caching 'readFromCache' is turned off, starting call")
+        log.debug(s"Call caching 'readFromCache' is turned off, starting call")
         self ! InitialStartCall(callKey, CallActor.Start)
       }
     }
@@ -1098,11 +1083,16 @@ case class WorkflowActor(workflow: WorkflowDescriptor, backend: Backend)
     executionStore += callKey -> ExecutionStatus.Starting
     persistStatus(callKey, ExecutionStatus.Starting)
 
-    fetchLocallyQualifiedInputs(callKey) match {
-      case Success(callInputs) => sendStartMessage(callKey, callInputs)
-      case Failure(t) =>
-        logger.error(s"Failed to fetch locally qualified inputs for call ${callKey.tag}", t)
-        scheduleTransition(WorkflowFailed)
+    callKey match {
+      case backendCallKey: BackendCallKey =>
+        fetchLocallyQualifiedInputs(backendCallKey) match {
+          case Success(callInputs) => sendStartMessage(backendCallKey, callInputs)
+          case Failure(t) =>
+            logger.error(s"Failed to fetch locally qualified inputs for call ${callKey.tag}", t)
+            scheduleTransition(WorkflowFailed)
+        }
+      case finalCallKey: FinalCallKey =>
+        self ! InitialStartCall(finalCallKey, CallActor.Start)
     }
     Success(ExecutionStartResult(Set(StartEntry(callKey, ExecutionStatus.Starting))))
   }
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 2f605800c03..6b5e6dd0380 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -19,7 +19,6 @@ import cromwell.webservice._
 import org.joda.time.DateTime
 import spray.json._
 import wdl4s._
-import wdl4s.values.WdlFile
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -163,7 +162,7 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
 
   private def callStdoutStderr(workflowId: WorkflowId, callFqn: String): Future[Any] = {
     def callKey(descriptor: WorkflowDescriptor, callName: String, key: ExecutionDatabaseKey) =
-      CallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
+      BackendCallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
     def backendCallFromKey(descriptor: WorkflowDescriptor, callName: String, key: ExecutionDatabaseKey) =
       backend.bindCall(descriptor, callKey(descriptor, callName, key))
     for {
@@ -183,7 +182,7 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
         val callsToPaths = for {
           (key, status) <- sortedMap if hasLogs(statusMap.keys)(key)
           callName = assertCallFqnWellFormed(descriptor, key.fqn).get
-          callKey = CallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
+          callKey = BackendCallKey(descriptor.namespace.workflow.findCallByName(callName).get, key.index)
           backendCall = backend.bindCall(descriptor, callKey)
           callStandardOutput = backend.stdoutStderr(backendCall)
         } yield key.fqn -> callStandardOutput
diff --git a/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala b/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
index a754baf213f..f69dff8e870 100644
--- a/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
+++ b/src/test/scala/cromwell/CopyWorkflowOutputsSpec.scala
@@ -12,7 +12,7 @@ import scala.language.postfixOps
 
 class CopyWorkflowOutputsSpec extends CromwellTestkitSpec {
 
-  "CopyWorkflowOutputs" should {
+  "CopyWorkflowOutputsCall" should {
     "copy workflow outputs" in {
       val workflowOutputsPath = "copy-workflow-outputs"
 
diff --git a/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala b/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
index fc1b32560a2..8b2d310ae52 100644
--- a/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/BackendCallSpec.scala
@@ -4,7 +4,7 @@ import java.util.UUID
 
 import cromwell.CromwellTestkitSpec
 import cromwell.engine.backend.local.LocalBackend
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor, WorkflowId}
 import cromwell.util.SampleWdl
 import org.scalatest.concurrent.ScalaFutures
@@ -18,7 +18,7 @@ class BackendCallSpec extends CromwellTestkitSpec with ScalaFutures {
   val sources = SampleWdl.CallCachingHashingWdl.asWorkflowSources()
   val descriptor = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), sources)
   val call = descriptor.namespace.workflow.calls.find(_.unqualifiedName == "t").get
-  val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
+  val backendCall = backend.bindCall(descriptor, BackendCallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
 
   "BackendCall hash function" should {
     "not change very often - if it changes, make sure it is for a good reason" in {
@@ -32,7 +32,7 @@ class BackendCallSpec extends CromwellTestkitSpec with ScalaFutures {
       val sources = SampleWdl.CallCachingHashingWdl.asWorkflowSources( s"""runtime { docker: "$nameAndDigest" } """)
       val descriptor = WorkflowDescriptor(WorkflowId(UUID.randomUUID()), sources)
       val call = descriptor.namespace.workflow.calls.find(_.unqualifiedName == "t").get
-      val backendCall = backend.bindCall(descriptor, CallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
+      val backendCall = backend.bindCall(descriptor, BackendCallKey(call, None), descriptor.actualInputs, abortRegistrationFunction = None)
 
       val actual = backendCall.hash.futureValue.overallHash
       val expected = "ca6ee457780b78290785f112c3a3acb4"
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index fa66c397568..db4354cd6ac 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -11,8 +11,9 @@ import cromwell.engine.backend.jes.JesBackend.{JesInput, JesOutput}
 import cromwell.engine.backend.jes.authentication._
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
 import cromwell.engine.io.gcs.{GoogleConfiguration, Refresh, ServiceAccountMode, SimpleClientSecrets}
-import cromwell.engine.workflow.{CallKey, WorkflowOptions}
-import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor, WorkflowId}
+import cromwell.engine.workflow.BackendCallKey
+import cromwell.engine.workflow.WorkflowOptions
+import cromwell.engine.{WorkflowDescriptor, WorkflowId}
 import cromwell.util.{EncryptionSpec, SampleWdl}
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 import org.specs2.mock.Mockito
@@ -59,7 +60,7 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
   }
 
   "adjustInputPaths" should "map GCS paths and *only* GCS paths to local" in {
-    val ignoredCall = mock[CallKey]
+    val ignoredCall = mock[BackendCallKey]
     val stringKey = "abc"
     val stringVal = WdlString("abc")
     val localFileKey = "lf"
@@ -223,7 +224,7 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
       workflowOptions = """ {"jes_gcs_root": "gs://path/to/gcs_root"} """
     ))
     val call = wd.namespace.workflow.findCallByName("hello").get
-    val backendCall = jesBackend.bindCall(wd, CallKey(call, None))
+    val backendCall = jesBackend.bindCall(wd, BackendCallKey(call, None))
     val stdoutstderr = backendCall.stdoutStderr
 
     stdoutstderr.stdout shouldBe WdlFile("gs://path/to/gcs_root/hello/e6236763-c518-41d0-9688-432549a8bf7c/call-hello/hello-stdout.log")
@@ -241,7 +242,7 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
       workflowOptions = """ {"jes_gcs_root": "gs://path/to/gcs_root"} """
     ))
     val call = wd.namespace.workflow.findCallByName("B").get
-    val backendCall = jesBackend.bindCall(wd, CallKey(call, Some(2)))
+    val backendCall = jesBackend.bindCall(wd, BackendCallKey(call, Some(2)))
     val stdoutstderr = backendCall.stdoutStderr
 
     stdoutstderr.stdout shouldBe WdlFile("gs://path/to/gcs_root/w/e6236763-c518-41d0-9688-432549a8bf7c/call-B/shard-2/B-2-stdout.log")
diff --git a/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala b/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
index 39aa0d52410..8c5c39be3ed 100644
--- a/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/local/LocalBackendSpec.scala
@@ -6,7 +6,7 @@ import cromwell.CromwellTestkitSpec
 import wdl4s.WdlSource
 import wdl4s.values.WdlValue
 import cromwell.engine.backend._
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor}
 import cromwell.util.SampleWdl
 import org.specs2.mock.Mockito
@@ -39,10 +39,11 @@ class LocalBackendSpec extends CromwellTestkitSpec with Mockito {
   def testFailOnStderr(descriptor: WorkflowDescriptor, expectSuccess: Boolean): Unit = {
     val call = descriptor.namespace.workflow.calls.head
     val backend = new LocalBackend(system)
-    val backendCall = backend.bindCall(descriptor, CallKey(call, None), Map.empty[String, WdlValue], abortRegistrationFunction = None)
+    val backendCall = backend.bindCall(descriptor, BackendCallKey(call, None), Map.empty[String, WdlValue], abortRegistrationFunction = None)
     backendCall.execute map { _.result } map {
       case FailedExecution(e, _) => if (expectSuccess) fail("A call in a failOnStderr test which should have succeeded has failed ", e)
-      case SuccessfulExecution(_, _, _, _, _) => if (!expectSuccess) fail("A call in a failOnStderr test which should have failed has succeeded")
+      case SuccessfulBackendCallExecution(_, _, _, _, _) => if (!expectSuccess) fail("A call in a failOnStderr test which should have failed has succeeded")
+      case SuccessfulFinalCallExecution => fail("A FinalCall shouldn't have run in this failOnStderr")
       case AbortedExecution => fail("Not expecting this at all")
     }
   }
diff --git a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
index e9d341d18b1..73f11d24fc8 100644
--- a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
+++ b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
@@ -14,6 +14,8 @@ import cromwell.engine.db.{CallStatus, ExecutionDatabaseKey, LocalCallBackendInf
 import cromwell.engine.io.IoInterface
 import cromwell.engine.workflow.{CallKey, ScatterKey, WorkflowOptions}
 import cromwell.engine.{CallOutput, CallOutputs, _}
+import cromwell.engine.workflow.{BackendCallKey, ScatterKey, WorkflowOptions}
+import cromwell.engine.backend.BackendType
 import cromwell.util.SampleWdl
 import cromwell.webservice
 import cromwell.webservice.{CallCachingParameters, WorkflowQueryKey, WorkflowQueryParameters}
@@ -67,7 +69,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
     override val actorSystem = workflowManagerSystem.actorSystem
 
-    override def adjustInputPaths(callKey: CallKey,
+    override def adjustInputPaths(callKey: BackendCallKey,
                                   runtimeAttributes: CromwellRuntimeAttributes,
                                   inputs: CallInputs,
                                   workflowDescriptor: WorkflowDescriptor
@@ -79,7 +81,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
     override def prepareForRestart(restartableWorkflow: WorkflowDescriptor)(implicit ec: ExecutionContext) = throw new NotImplementedError
 
     override def bindCall(workflowDescriptor: WorkflowDescriptor,
-                          key: CallKey,
+                          key: BackendCallKey,
                           locallyQualifiedInputs: CallInputs,
                           abortRegistrationFunction: Option[AbortRegistrationFunction]): BackendCall =
       throw new NotImplementedError
@@ -516,7 +518,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
             status.executionStatus should be(if (updateStatus) ExecutionStatus.Running else ExecutionStatus.NotStarted)
             status.returnCode should be(None)
           }
-          _ <- dataAccess.insertCalls(workflowId, Seq(CallKey(call, Option(0))), localBackend)
+          _ <- dataAccess.insertCalls(workflowId, Seq(BackendCallKey(call, Option(0))), localBackend)
           _ <- dataAccess.setStatus(workflowId, Seq(shardKey), CallStatus(ExecutionStatus.Done, Option(0), None, None))
           _ <- dataAccess.getExecutionStatuses(workflowId) map { result =>
             result.size should be(2)
@@ -544,7 +546,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       val workflowInfo = WorkflowDescriptor(workflowId, testSources)
       val task = Task.empty
       val call = new Call(None, callFqn, task, Set.empty[FullyQualifiedName], Map.empty, None)
-      val callKey = CallKey(call, None)
+      val callKey = BackendCallKey(call, None)
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Nil, localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
@@ -653,8 +655,8 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Nil, localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, Option(0)), Map(symbolLqn -> CallOutput(testWdlStringShard, Option(testWdlStringShardHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, Option(0)), Map(symbolLqn -> CallOutput(testWdlStringShard, Option(testWdlStringShardHash))), Seq.empty)
         _ <- dataAccess.getOutputs(workflowId, ExecutionDatabaseKey(call.fullyQualifiedName, None)) map { results =>
           results.size should be(1) //getOutputs on a workflowId does NOT return shards outputs
 
@@ -688,10 +690,10 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Nil, localBackend)
-        _ <- dataAccess.insertCalls(workflowId, Seq(CallKey(call1, shardIndex1)), localBackend)
-        _ <- dataAccess.insertCalls(workflowId, Seq(CallKey(call2, shardIndex2)), localBackend)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId, CallKey(call1, shardIndex1), backendInfo1)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId, CallKey(call2, shardIndex2), backendInfo2)
+        _ <- dataAccess.insertCalls(workflowId, Seq(BackendCallKey(call1, shardIndex1)), localBackend)
+        _ <- dataAccess.insertCalls(workflowId, Seq(BackendCallKey(call2, shardIndex2)), localBackend)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId, BackendCallKey(call1, shardIndex1), backendInfo1)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId, BackendCallKey(call2, shardIndex2), backendInfo2)
         _ <- dataAccess.getExecutionBackendInfo(workflowId, call1) map {
           case LocalCallBackendInfo(processId: Option[Int]) => assertResult(pid1) { processId }
           case _ => fail("Unexpected backend info type returned")
@@ -715,8 +717,8 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Nil, localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, Option(0)), Map(symbolLqn -> CallOutput(testWdlStringShard, Option(testWdlStringShardHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, Option(0)), Map(symbolLqn -> CallOutput(testWdlStringShard, Option(testWdlStringShardHash))), Seq.empty)
         callOutput <- dataAccess.getOutputs(workflowId, ExecutionDatabaseKey(call.fullyQualifiedName, None)) map { results =>
           results.head.key.index should be(None)
           results.head.wdlValue.get should be(testWdlString)
@@ -792,7 +794,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Seq(entry), Nil, localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, None), Map(symbolLqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
         _ <- dataAccess.getOutputs(workflowId, ExecutionDatabaseKey(call.fullyQualifiedName, None)) map { results =>
           results.size should be(1)
           val resultSymbol = results.head
@@ -823,9 +825,9 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Seq(), Nil, localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, None), Map(symbolFqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, None), Map(symbolFqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
         // Second attempt should fail
-        _ <- dataAccess.setOutputs(workflowId, CallKey(call, None), Map(symbolFqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
+        _ <- dataAccess.setOutputs(workflowId, BackendCallKey(call, None), Map(symbolFqn -> CallOutput(testWdlString, Option(testWdlStringHash))), Seq.empty)
       } yield ()).failed.futureValue should be(a[SQLException])
     }
 
@@ -839,7 +841,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Seq(call), localBackend)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId, CallKey(call, None), backendInfo)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId, BackendCallKey(call, None), backendInfo)
         _ <- dataAccess.getExecutionBackendInfo(workflowId, call) map { insertResultCall =>
           insertResultCall should be(a[LocalCallBackendInfo])
           val insertResultLocalCall = insertResultCall.asInstanceOf[LocalCallBackendInfo]
@@ -864,8 +866,8 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo1, Nil, Seq(call), localBackend)
         _ <- dataAccess.createWorkflow(workflowInfo2, Nil, Seq(call), localBackend)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId1, CallKey(call, None), backendInfo1)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId2, CallKey(call, None), backendInfo2)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId1, BackendCallKey(call, None), backendInfo1)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId2, BackendCallKey(call, None), backendInfo2)
         _ <- dataAccess.getExecutionBackendInfo(workflowId1, call) map { insertResultCall =>
           insertResultCall should be(a[LocalCallBackendInfo])
           val insertResultLocalCall = insertResultCall.asInstanceOf[LocalCallBackendInfo]
@@ -891,7 +893,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Seq(call), localBackend)
         _ <- dataAccess.updateWorkflowState(workflowId, WorkflowRunning)
-        _ <- dataAccess.updateExecutionBackendInfo(workflowId, CallKey(call, None), null)
+        _ <- dataAccess.updateExecutionBackendInfo(workflowId, BackendCallKey(call, None), null)
       } yield ()).failed.futureValue should be(an[IllegalArgumentException])
     }
 
@@ -941,7 +943,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
       (for {
         _ <- dataAccess.createWorkflow(workflowInfo, Nil, Seq(call), localBackend)
-        _ <- dataAccess.insertCalls(workflowId, Seq(CallKey(shardedCall, shardIndex)), localBackend)
+        _ <- dataAccess.insertCalls(workflowId, Seq(BackendCallKey(shardedCall, shardIndex)), localBackend)
 
         now = DateTime.now
         mainEventSeq = Seq(
diff --git a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
index 5bca95c7c1c..ba709546f9c 100644
--- a/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
+++ b/src/test/scala/cromwell/logging/WorkflowLoggerSpec.scala
@@ -5,7 +5,7 @@ import java.util.UUID
 import cromwell.CromwellTestkitSpec
 import wdl4s.values.WdlValue
 import cromwell.engine.backend.local.{LocalBackend, LocalBackendCall}
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor, WorkflowId, WorkflowSourceFiles}
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
 
@@ -29,7 +29,7 @@ class WorkflowLoggerSpec extends FlatSpec with Matchers with BeforeAndAfterAll {
   val backendCall = LocalBackendCall(
     backend,
     descriptor,
-    CallKey(descriptor.namespace.workflow.calls.find(_.unqualifiedName == "x").head, None),
+    BackendCallKey(descriptor.namespace.workflow.calls.find(_.unqualifiedName == "x").head, None),
     Map.empty[String, WdlValue],
     callAbortRegistrationFunction = None
   )

From ff546dd7cff2dad9e2594295c4ce2fc5aad7d29b Mon Sep 17 00:00:00 2001
From: geoffjentry <geoffjentry@gmail.com>
Date: Fri, 22 Jan 2016 13:43:01 -0500
Subject: [PATCH 26/31] Add deprecation notices to WDL specific command line
 functionality

---
 README.md                          | 119 ++++++++++++++++-------------
 src/main/scala/cromwell/Main.scala |  18 +++++
 2 files changed, 83 insertions(+), 54 deletions(-)

diff --git a/README.md b/README.md
index 582f83d6e3c..53a53907843 100644
--- a/README.md
+++ b/README.md
@@ -17,12 +17,12 @@ A [Workflow Management System](https://en.wikipedia.org/wiki/Workflow_management
 * [Building](#building)
 * [Installing](#installing)
 * [Command Line Usage](#command-line-usage)
-  * [validate](#validate)
-  * [inputs](#inputs)
   * [run](#run)
+  * [server](#server)
   * [parse](#parse)
+  * [validate](#validate)
+  * [inputs](#inputs)
   * [highlight](#highlight)
-  * [server](#server)
 * [Getting Started with WDL](#getting-started-with-wdl)
 * [Configuring Cromwell](#configuring-cromwell)
   * [Database](#database)
@@ -126,76 +126,36 @@ server
 
 parse <WDL file>
 
+  This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+  
   Compares a WDL file against the grammar and prints out an
   abstract syntax tree if it is valid, and a syntax error
   otherwise.  Note that higher-level AST checks are not done
   via this sub-command and the 'validate' subcommand should
   be used for full validation
 
-validate <WDL file>
-
+validate <WDL file>  
   Performs full validation of the WDL file including syntax
   and semantic checking
 
 inputs <WDL file>
 
+  This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+  
   Print a JSON skeleton file of the inputs needed for this
   workflow.  Fill in the values in this JSON document and
   pass it in to the 'run' subcommand.
 
 highlight <WDL file> <html|console>
 
+  This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+  
   Reformats and colorizes/tags a WDL file. The second
   parameter is the output type.  "html" will output the WDL
   file with <span> tags around elements.  "console" mode
   will output colorized text to the terminal
 ```
 
-## validate
-
-Given a WDL file, this runs the full syntax checker over the file and resolves imports in the process.  If any syntax errors are found, they are printed out.  Otherwise the program exits.
-
-Error if a `call` references a task that doesn't exist:
-
-```
-$ java -jar cromwell.jar validate 2.wdl
-ERROR: Call references a task (BADps) that doesn't exist (line 22, col 8)
-
-  call BADps
-       ^
-
-```
-
-Error if namespace and task have the same name:
-
-```
-$ java -jar cromwell.jar validate 5.wdl
-ERROR: Task and namespace have the same name:
-
-Task defined here (line 3, col 6):
-
-task ps {
-     ^
-
-Import statement defined here (line 1, col 20):
-
-import "ps.wdl" as ps
-                   ^
-```
-
-## inputs
-
-Examine a WDL file with one workflow in it, compute all the inputs needed for that workflow and output a JSON template that the user can fill in with values.  The keys in this document should remain unchanged.  The values tell you what type the parameter is expecting.  For example, if the value were `Array[String]`, then it's expecting a JSON array of JSON strings, like this: `["string1", "string2", "string3"]`
-
-```
-$ java -jar cromwell.jar inputs 3step.wdl
-{
-  "three_step.cgrep.pattern": "String"
-}
-```
-
-This inputs document is used as input to the `run` subcommand.
-
 ## run
 
 Given a WDL file and a JSON inputs file (see `inputs` subcommand), Run the workflow and print the outputs:
@@ -271,8 +231,14 @@ $ cat my_wf.metadata.json
 }
 ```
 
+## server
+
+Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section.
+
 ## parse
 
+This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+
 Given a WDL file input, this does grammar level syntax checks and prints out the resulting abstract syntax tree.
 
 ```
@@ -288,8 +254,57 @@ $ echo "workflow wf {}" | java -jar cromwell.jar parse /dev/stdin
 )
 ```
 
+## validate
+
+Given a WDL file, this runs the full syntax checker over the file and resolves imports in the process.  If any syntax errors are found, they are printed out.  Otherwise the program exits.
+
+Error if a `call` references a task that doesn't exist:
+
+```
+$ java -jar cromwell.jar validate 2.wdl
+ERROR: Call references a task (BADps) that doesn't exist (line 22, col 8)
+
+  call BADps
+       ^
+
+```
+
+Error if namespace and task have the same name:
+
+```
+$ java -jar cromwell.jar validate 5.wdl
+ERROR: Task and namespace have the same name:
+
+Task defined here (line 3, col 6):
+
+task ps {
+     ^
+
+Import statement defined here (line 1, col 20):
+
+import "ps.wdl" as ps
+                   ^
+```
+
+## inputs
+
+This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+
+Examine a WDL file with one workflow in it, compute all the inputs needed for that workflow and output a JSON template that the user can fill in with values.  The keys in this document should remain unchanged.  The values tell you what type the parameter is expecting.  For example, if the value were `Array[String]`, then it's expecting a JSON array of JSON strings, like this: `["string1", "string2", "string3"]`
+
+```
+$ java -jar cromwell.jar inputs 3step.wdl
+{
+  "three_step.cgrep.pattern": "String"
+}
+```
+
+This inputs document is used as input to the `run` subcommand.
+
 ## highlight
 
+This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool
+
 Formats a WDL file and semantically tags it.  This takes a second parameter (`html` or `console`) which determines what the output format will be.
 
 test.wdl
@@ -328,10 +343,6 @@ $ java -jar cromwell.jar highlight test.wdl html
 }
 ```
 
-## server
-
-Start a server on port 8000, the API for the server is described in the [REST API](#rest-api) section.
-
 # Getting Started with WDL
 
 For many examples on how to use WDL see [the WDL site](https://github.com/broadinstitute/wdl/tree/develop#getting-started-with-wdl)
diff --git a/src/main/scala/cromwell/Main.scala b/src/main/scala/cromwell/Main.scala
index 3259a759b10..731e8294894 100644
--- a/src/main/scala/cromwell/Main.scala
+++ b/src/main/scala/cromwell/Main.scala
@@ -5,6 +5,7 @@ import java.nio.file.{Files, Path, Paths}
 
 import akka.actor.{Actor, Props, Status}
 import better.files._
+import cromwell.util.TerminalUtil
 import wdl4s.formatter.{AnsiSyntaxHighlighter, HtmlSyntaxHighlighter, SyntaxFormatter}
 import wdl4s.{AstTools, _}
 import cromwell.engine.WorkflowSourceFiles
@@ -20,6 +21,7 @@ import scala.concurrent.duration._
 import scala.concurrent.{Future, Await, Promise}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
+import Main.deprecationNotice
 
 object Actions extends Enumeration {
   val Parse, Validate, Highlight, Run, Inputs, Server = Value
@@ -66,6 +68,10 @@ object Main extends App {
     argCapitalized = arg.capitalize
     action <- Actions.values find (_.toString == argCapitalized)
   } yield action
+
+  def deprecationNotice(): Unit = {
+    Console.err.println(TerminalUtil.highlight(1, "This functionality is deprecated and will be removed in 0.18. Please use wdltool: https://github.com/broadinstitute/wdltool"))
+  }
 }
 
 /** A simplified version of the Akka `PromiseActorRef` that doesn't time out. */
@@ -102,12 +108,14 @@ class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
   }
 
   def validate(args: Seq[String]): Int = {
+    deprecationNotice()
     continueIf(args.length == 1) {
       loadWdl(args.head) { _ => 0 }
     }
   }
 
   def highlight(args: Seq[String]): Int = {
+    deprecationNotice()
     continueIf(args.length == 2 && Seq("html", "console").contains(args(1))) {
       loadWdl(args.head) { namespace =>
         val formatter = new SyntaxFormatter(if (args(1) == "html") HtmlSyntaxHighlighter else AnsiSyntaxHighlighter)
@@ -118,6 +126,7 @@ class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
   }
 
   def inputs(args: Seq[String]): Int = {
+    deprecationNotice()
     continueIf(args.length == 1) {
       loadWdl(args.head) { namespace =>
         import wdl4s.types.WdlTypeJsonFormatter._
@@ -273,6 +282,7 @@ class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
   }
 
   def parse(args: Seq[String]): Int = {
+    deprecationNotice()
     continueIf(args.length == 1) {
       println(AstTools.getAst(new JFile(args.head)).toPrettyString)
       0
@@ -304,6 +314,8 @@ class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
         |
         |parse <WDL file>
         |
+        |  This functionality is deprecated. Please use wdltool: https://github.com/broadinstitute/wdltool
+        |
         |  Compares a WDL file against the grammar and prints out an
         |  abstract syntax tree if it is valid, and a syntax error
         |  otherwise.  Note that higher-level AST checks are not done
@@ -312,17 +324,23 @@ class Main private[cromwell](managerSystem: WorkflowManagerSystem) {
         |
         |validate <WDL file>
         |
+        |  This functionality is deprecated. Please use wdltool: https://github.com/broadinstitute/wdltool
+        |
         |  Performs full validation of the WDL file including syntax
         |  and semantic checking
         |
         |inputs <WDL file>
         |
+        |  This functionality is deprecated. Please use wdltool: https://github.com/broadinstitute/wdltool
+        |
         |  Print a JSON skeleton file of the inputs needed for this
         |  workflow.  Fill in the values in this JSON document and
         |  pass it in to the 'run' subcommand.
         |
         |highlight <WDL file> <html|console>
         |
+        |  This functionality is deprecated. Please use wdltool: https://github.com/broadinstitute/wdltool
+        |
         |  Reformats and colorizes/tags a WDL file. The second
         |  parameter is the output type.  "html" will output the WDL
         |  file with <span> tags around elements.  "console" mode

From 8524593ae6132f8b4a9c7e55968073eb9ecc2b54 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Thu, 21 Jan 2016 14:45:15 -0500
Subject: [PATCH 27/31] Refactor backoffs + retries + 429 transient

---
 .../cromwell/engine/WorkflowDescriptor.scala  |  6 +-
 .../engine/backend/jes/JesBackend.scala       | 31 +++++----
 src/main/scala/cromwell/util/Backoff.scala    | 48 ++++++++++++++
 src/main/scala/cromwell/util/TryUtil.scala    | 66 +++++++++++--------
 .../engine/backend/jes/JesBackendSpec.scala   | 13 ++++
 .../scala/cromwell/util/BackoffSpec.scala     | 51 ++++++++++++++
 .../scala/cromwell/util/TryUtilSpec.scala     | 19 ++----
 7 files changed, 179 insertions(+), 55 deletions(-)
 create mode 100644 src/main/scala/cromwell/util/Backoff.scala
 create mode 100644 src/test/scala/cromwell/util/BackoffSpec.scala

diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index 1908fc11024..2e93e31fa99 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -16,7 +16,7 @@ import cromwell.engine.io.shared.SharedFileSystemIoInterface
 import cromwell.engine.io.{IoInterface, IoManager}
 import cromwell.engine.workflow.WorkflowOptions
 import cromwell.logging.WorkflowLogger
-import cromwell.util.TryUtil
+import cromwell.util.{SimpleExponentialBackoff, TryUtil}
 import lenthall.config.ScalaConfig._
 import org.slf4j.helpers.NOPLogger
 import org.slf4j.{Logger, LoggerFactory}
@@ -118,9 +118,7 @@ case class WorkflowDescriptor(id: WorkflowId,
       TryUtil.retryBlock(
         fn = (retries: Option[Unit]) => copy(),
         retryLimit = Option(5),
-        pollingInterval = 5 seconds,
-        pollingBackOffFactor = 1,
-        maxPollingInterval = 10 seconds,
+        backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D),
         logger = logger,
         failMessage = Option(s"Failed to copy file $src to $dest"),
         isFatal = (t: Throwable) => t.isInstanceOf[FileAlreadyExistsException]
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index ed09e8fe890..029652fb4bb 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -6,6 +6,7 @@ import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorSystem
 import com.google.api.client.http.HttpResponseException
+import com.google.api.client.util.ExponentialBackOff
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.engine.ExecutionIndex.IndexEnhancedInt
@@ -24,7 +25,7 @@ import cromwell.engine.io.gcs._
 import cromwell.engine.workflow.{BackendCallKey, WorkflowOptions}
 import cromwell.engine.{AbortRegistrationFunction, CallOutput, CallOutputs, HostInputs, _}
 import cromwell.logging.WorkflowLogger
-import cromwell.util.{AggregatedException, TryUtil}
+import cromwell.util.{SimpleExponentialBackoff, AggregatedException, TryUtil}
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
 import wdl4s.{Call, CallInputs, Scatter, UnsatisfiedInputsException, _}
@@ -112,20 +113,28 @@ object JesBackend {
   }
 
   def isFatalJesException(t: Throwable): Boolean = t match {
+      // Unauthorized
     case e: HttpResponseException if e.getStatusCode == 403 => true
     case _ => false
   }
 
-  protected def withRetry[T](f: Option[T] => T, logger: WorkflowLogger, failureMessage: String) = TryUtil.retryBlock(
-    fn = f,
-    retryLimit = Option(10),
-    pollingInterval = 5 seconds,
-    pollingBackOffFactor = 1,
-    maxPollingInterval = 10 seconds,
-    logger = logger,
-    failMessage = Option(failureMessage),
-    isFatal = isFatalJesException
-  )
+  def isTransientJesException(t: Throwable): Boolean = t match {
+      // Quota exceeded
+    case e: HttpResponseException if e.getStatusCode == 429 => true
+    case _ => false
+  }
+
+  protected def withRetry[T](f: Option[T] => T, logger: WorkflowLogger, failureMessage: String) = {
+    TryUtil.retryBlock(
+      fn = f,
+      retryLimit = Option(10),
+      backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D),
+      logger = logger,
+      failMessage = Option(failureMessage),
+      isFatal = isFatalJesException,
+      isTransient = isTransientJesException
+    )
+  }
 
   sealed trait JesParameter {
     def name: String
diff --git a/src/main/scala/cromwell/util/Backoff.scala b/src/main/scala/cromwell/util/Backoff.scala
new file mode 100644
index 00000000000..512ae600559
--- /dev/null
+++ b/src/main/scala/cromwell/util/Backoff.scala
@@ -0,0 +1,48 @@
+package cromwell.util
+
+import com.google.api.client.util.ExponentialBackOff
+
+import scala.concurrent.duration.{Duration, FiniteDuration}
+
+sealed trait Backoff {
+  /** Next interval in millis */
+  def backoffMillis: Long
+  /** Get the next instance of backoff. This should be called after every call to backoffMillis */
+  def next: Backoff
+}
+
+object InitialGapBackoff {
+  def apply(initialGap: FiniteDuration, initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double) = {
+    new InitialGapBackoff(initialGap, new ExponentialBackOff.Builder()
+      .setInitialIntervalMillis(initialInterval.toMillis.toInt)
+      .setMaxIntervalMillis(maxInterval.toMillis.toInt)
+      .setMultiplier(multiplier)
+      .setMaxElapsedTimeMillis(Int.MaxValue)
+      .build())
+  }
+}
+
+case class InitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) extends Backoff {
+  assert(initialGapMillis.compareTo(Duration.Zero) != 0, "Initial gap cannot be null, use SimpleBackoff instead.")
+
+  override val backoffMillis = initialGapMillis.toMillis
+  /** Switch to a SimpleExponentialBackoff after the initial gap has been used */
+  override def next = new SimpleExponentialBackoff(googleBackoff)
+}
+
+object SimpleExponentialBackoff {
+  def apply(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double) = {
+    new SimpleExponentialBackoff(new ExponentialBackOff.Builder()
+      .setInitialIntervalMillis(initialInterval.toMillis.toInt)
+      .setMaxIntervalMillis(maxInterval.toMillis.toInt)
+      .setMultiplier(multiplier)
+      .setMaxElapsedTimeMillis(Int.MaxValue)
+      .build())
+  }
+}
+
+case class SimpleExponentialBackoff(googleBackoff: ExponentialBackOff) extends Backoff {
+  override def backoffMillis = googleBackoff.nextBackOffMillis()
+  /** google ExponentialBackOff is mutable so we can keep returning the same instance */
+  override def next = this
+}
diff --git a/src/main/scala/cromwell/util/TryUtil.scala b/src/main/scala/cromwell/util/TryUtil.scala
index 7738ca62ac9..19d76c0ed23 100644
--- a/src/main/scala/cromwell/util/TryUtil.scala
+++ b/src/main/scala/cromwell/util/TryUtil.scala
@@ -5,7 +5,7 @@ import java.io.{PrintWriter, StringWriter}
 import cromwell.engine.CromwellFatalException
 import cromwell.logging.WorkflowLogger
 
-import scala.concurrent.duration.Duration
+import scala.annotation.tailrec
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -30,34 +30,41 @@ object TryUtil {
 
   private def defaultSuccessFunction(a: Any): Boolean = true
   private def defaultIsFatal(t: Throwable): Boolean = false
+  private def defaultIsTransient(t: Throwable): Boolean = false
+
 
   /**
-   * Runs a block of code (`fn`) `retries` number of times until it succeeds.
-   * It will wait `pollingInterval` amount of time between retry attempts and
-   * The `pollingBackOffFactor` is for exponentially backing off the `pollingInterval`
-   * on subsequent retries.  The `pollingInterval` shall not exceed `maxPollingInterval`
-   *
-   * Returns a Try[T] where T is the return value of `fn`, the function to be retried.
-   * If the return value is Success[T] then at least one retry succeeded.
-   *
-   * The isSuccess function is optional but if provided, then isSuccess(fn) must be true
-   * or it will trigger another retry.  if isSuccess is omitted, the only way the fn can
-   * fail is if it throws an exception.
-   *
-   * Use `retryLimit` value of None indicates to retry indefinitely.
-   */
-  @annotation.tailrec
+    * Runs a block of code (`fn`) `retries` number of times until it succeeds.
+    * It will use an InitializedBackoff instance to handle backoff timings and wait
+    * between each retry.
+    *
+    * Returns a Try[T] where T is the return value of `fn`, the function to be retried.
+    * If the return value is Success[T] then at least one retry succeeded.
+    *
+    * The isSuccess function is optional but if provided, then isSuccess(fn) must be true
+    * or it will trigger another retry.  if isSuccess is omitted, the only way the fn can
+    * fail is if it throws an exception.
+    *
+    * The isFatal function is optional but if provided, then a failure f for which
+    * isFatal(f) returns true would terminate the retry loop, even if the retry limit has not been reached.
+    *
+    * The isTransient function is optional but if provided, then a failure f for which
+    * isTransient(f) returns true would not count against the retry limit.
+    *
+    * Note that if the retry limit is reached, the last failure is wrapped into a CromwellFatalException.
+    *
+    * Use `retryLimit` value of None indicates to retry indefinitely.
+    */
+  @tailrec
   def retryBlock[T](fn: Option[T] => T,
+                    backoff: Backoff,
                     isSuccess: T => Boolean = defaultSuccessFunction _,
                     retryLimit: Option[Int],
-                    pollingInterval: Duration,
-                    pollingBackOffFactor: Double,
-                    maxPollingInterval: Duration,
                     logger: WorkflowLogger,
                     failMessage: Option[String] = None,
                     priorValue: Option[T] = None,
-                    isFatal: (Throwable) => Boolean = defaultIsFatal _): Try[T] = {
-
+                    isFatal: (Throwable) => Boolean = defaultIsFatal _,
+                    isTransient: (Throwable) => Boolean = defaultIsTransient _): Try[T] = {
     def logFailures(attempt: Try[T]): Unit = {
       attempt recover {
         case t: Throwable => logger.warn(t.getMessage, t)
@@ -69,19 +76,24 @@ object TryUtil {
       case Failure(f) if isFatal(f) => Failure(new CromwellFatalException(f))
       case value if (retryLimit.isDefined && retryLimit.get > 1) || retryLimit.isEmpty =>
         logFailures(value)
-        val retryCountMessage = if (retryLimit.getOrElse(0) > 0) s" (${retryLimit.getOrElse(0) - 1} more retries) " else ""
+
+        val transient = isTransient(value.failed.get)
+        val newLimit = if (transient) retryLimit else retryLimit.map(_ - 1)
+        val retryCountMessage = {
+          val transientMessage = if (transient) " - This error is transient and does not count against the retry limit." else ""
+          if (retryLimit.getOrElse(0) > 0 || transient) s" (${newLimit.getOrElse(0)} more retries)$transientMessage" else ""
+        }
+        val pollingInterval = backoff.backoffMillis
         val retryMessage = s"Retrying in $pollingInterval$retryCountMessage..."
         failMessage foreach { m => logger.warn(s"$m.  $retryMessage") }
 
-        Thread.sleep(pollingInterval.toMillis)
+        Thread.sleep(pollingInterval)
 
         retryBlock(
           fn,
+          backoff.next,
           isSuccess,
-          retryLimit.map(_ - 1),
-          Duration(Math.min((pollingInterval.toMillis * pollingBackOffFactor).toLong, maxPollingInterval.toMillis), "milliseconds"),
-          pollingBackOffFactor,
-          maxPollingInterval,
+          newLimit,
           logger,
           failMessage,
           value.toOption
diff --git a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
index db4354cd6ac..9e09c2d9dbf 100644
--- a/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
+++ b/src/test/scala/cromwell/engine/backend/jes/JesBackendSpec.scala
@@ -59,6 +59,19 @@ class JesBackendSpec extends FlatSpec with Matchers with Mockito with BeforeAndA
     JesBackend.isFatalJesException(mockedResponse) shouldBe true
   }
 
+  it should "consider 429 as a transient exception" in {
+    val transport = new MockHttpTransport() {
+      override def buildRequest(method: String, url: String) = {
+        new MockLowLevelHttpRequest() {
+          override def execute() = new MockLowLevelHttpResponse().setStatusCode(429)
+        }
+      }
+    }
+    val request = transport.createRequestFactory().buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL)
+    val mockedResponse = Try(request.execute()).failed.get
+    JesBackend.isTransientJesException(mockedResponse) shouldBe true
+  }
+
   "adjustInputPaths" should "map GCS paths and *only* GCS paths to local" in {
     val ignoredCall = mock[BackendCallKey]
     val stringKey = "abc"
diff --git a/src/test/scala/cromwell/util/BackoffSpec.scala b/src/test/scala/cromwell/util/BackoffSpec.scala
new file mode 100644
index 00000000000..11c4348d49a
--- /dev/null
+++ b/src/test/scala/cromwell/util/BackoffSpec.scala
@@ -0,0 +1,51 @@
+package cromwell.util
+
+import com.google.api.client.util.ExponentialBackOff.Builder
+import org.scalatest.{FlatSpec, Matchers}
+
+import scala.concurrent.duration._
+
+class BackoffSpec extends FlatSpec with Matchers {
+
+  it should "honor initial gap" in {
+    val exponentialBackoff = new InitialGapBackoff(
+      3.seconds,
+      new Builder()
+        .setInitialIntervalMillis(1.second.toMillis.toInt)
+        .setMaxIntervalMillis(2.seconds.toMillis.toInt)
+        .setMaxElapsedTimeMillis(Integer.MAX_VALUE)
+        .setRandomizationFactor(0D)
+        .build()
+    )
+
+
+    exponentialBackoff.backoffMillis shouldBe 3.seconds.toMillis
+    exponentialBackoff.next.backoffMillis shouldBe 1.second.toMillis
+  }
+
+  it should "work for simple backoffs" in {
+    new SimpleExponentialBackoff(
+      new Builder()
+        .setInitialIntervalMillis(1.second.toMillis.toInt)
+        .setMaxIntervalMillis(2.seconds.toMillis.toInt)
+        .setMaxElapsedTimeMillis(Integer.MAX_VALUE)
+        .setRandomizationFactor(0D)
+        .build()
+    ).backoffMillis shouldBe 1.second.toMillis
+  }
+
+  it should "throw an exception when trying create a initial gap equal to 0" in {
+    an[AssertionError] should be thrownBy {
+      new InitialGapBackoff(
+        0.millis,
+        new Builder()
+          .setInitialIntervalMillis(1.second.toMillis.toInt)
+          .setMaxIntervalMillis(2.seconds.toMillis.toInt)
+          .setMaxElapsedTimeMillis(Integer.MAX_VALUE)
+          .setRandomizationFactor(0D)
+          .build()
+      )
+    }
+  }
+
+}
diff --git a/src/test/scala/cromwell/util/TryUtilSpec.scala b/src/test/scala/cromwell/util/TryUtilSpec.scala
index 8f5a8e16564..2877b24db63 100644
--- a/src/test/scala/cromwell/util/TryUtilSpec.scala
+++ b/src/test/scala/cromwell/util/TryUtilSpec.scala
@@ -1,14 +1,12 @@
 package cromwell.util
 
-import java.util.UUID
-
-import cromwell.engine.{CromwellFatalException, WorkflowId, WorkflowSourceFiles, WorkflowDescriptor}
+import cromwell.engine.CromwellFatalException
 import cromwell.logging.WorkflowLogger
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.{FlatSpec, Matchers}
 
-import scala.language.postfixOps
 import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.util.Success
 
 class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {
@@ -30,15 +28,14 @@ class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {
   }
 
   val logger = mock[WorkflowLogger]
+  val backoff = SimpleExponentialBackoff(50 milliseconds, 10 seconds, 1D)
 
   it should "Retry a function until it works" in {
     val work = new MockWork
     val value = TryUtil.retryBlock(
       fn = work.failNTimes(4),
       retryLimit = Some(5),
-      pollingInterval = 50 milliseconds,
-      pollingBackOffFactor = 1,
-      maxPollingInterval = 10 seconds,
+      backoff = backoff,
       logger = logger,
       failMessage = Some(s"failed attempt (on purpose)")
     )
@@ -51,9 +48,7 @@ class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {
     val value = TryUtil.retryBlock(
       fn = work.failNTimes(4),
       retryLimit = Some(4),
-      pollingInterval = 50 milliseconds,
-      pollingBackOffFactor = 1,
-      maxPollingInterval = 10 seconds,
+      backoff = backoff,
       logger = logger,
       failMessage = Some(s"failed attempt (on purpose)")
     )
@@ -66,9 +61,7 @@ class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {
     val value = TryUtil.retryBlock(
       fn = work.failNTimes(4),
       retryLimit = Some(4),
-      pollingInterval = 50 milliseconds,
-      pollingBackOffFactor = 1,
-      maxPollingInterval = 10 seconds,
+      backoff = backoff,
       logger = logger,
       failMessage = Some(s"failed attempt (on purpose)"),
       isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException]

From 7785e93e09cf5bde9e179842a280566034402242 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Tue, 19 Jan 2016 13:23:29 -0500
Subject: [PATCH 28/31] Make Call backoff polling backend specific

---
 .../cromwell/engine/WorkflowDescriptor.scala   |  2 +-
 .../cromwell/engine/backend/Backend.scala      |  3 +++
 .../cromwell/engine/backend/BackendCall.scala  |  7 +------
 .../engine/backend/jes/JesBackend.scala        | 16 +++++++++++++---
 .../engine/backend/jes/JesBackendCall.scala    |  8 +++++---
 .../engine/backend/local/LocalBackend.scala    | 17 ++++++++++++++---
 .../backend/local/LocalBackendCall.scala       |  5 ++++-
 .../engine/backend/sge/SgeBackend.scala        | 13 +++++++++++++
 .../engine/backend/sge/SgeBackendCall.scala    |  4 +++-
 .../BackendCallExecutionActor.scala            |  5 ++++-
 .../callexecution/CallExecutionActor.scala     | 18 +++++++-----------
 .../FinalCallExecutionActor.scala              | 11 +++++++++++
 src/main/scala/cromwell/util/TryUtil.scala     |  3 +--
 .../engine/db/slick/SlickDataAccessSpec.scala  |  3 ++-
 src/test/scala/cromwell/util/TryUtilSpec.scala |  6 ++++--
 15 files changed, 86 insertions(+), 35 deletions(-)

diff --git a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
index 2e93e31fa99..3e8eef193fb 100644
--- a/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
+++ b/src/main/scala/cromwell/engine/WorkflowDescriptor.scala
@@ -9,7 +9,7 @@ import ch.qos.logback.core.FileAppender
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.engine.backend.jes.JesBackend
 import cromwell.engine.backend.runtimeattributes.CromwellRuntimeAttributes
-import cromwell.engine.backend.{BackendType, Backend, CromwellBackend}
+import cromwell.engine.backend.{Backend, BackendType, CromwellBackend}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.io.gcs.{GcsFileSystem, GoogleCloudStorage}
 import cromwell.engine.io.shared.SharedFileSystemIoInterface
diff --git a/src/main/scala/cromwell/engine/backend/Backend.scala b/src/main/scala/cromwell/engine/backend/Backend.scala
index 90d34cdc05d..33246a5be0a 100644
--- a/src/main/scala/cromwell/engine/backend/Backend.scala
+++ b/src/main/scala/cromwell/engine/backend/Backend.scala
@@ -1,6 +1,7 @@
 package cromwell.engine.backend
 
 import akka.actor.ActorSystem
+import com.google.api.client.util.ExponentialBackOff
 import com.typesafe.config.Config
 import cromwell.engine._
 import cromwell.engine.backend.jes.JesBackend
@@ -157,4 +158,6 @@ trait Backend {
   )
 
   lazy val dockerHashClient = new SprayDockerRegistryApiClient()(actorSystem)
+
+  def pollBackoff: ExponentialBackOff
 }
diff --git a/src/main/scala/cromwell/engine/backend/BackendCall.scala b/src/main/scala/cromwell/engine/backend/BackendCall.scala
index 9dc2cc7fbb4..dced091cf6d 100644
--- a/src/main/scala/cromwell/engine/backend/BackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/BackendCall.scala
@@ -3,17 +3,12 @@ package cromwell.engine.backend
 import akka.event.LoggingAdapter
 import cromwell.engine.Hashing._
 import cromwell.engine.backend.runtimeattributes.{ContinueOnReturnCodeFlag, ContinueOnReturnCodeSet, CromwellRuntimeAttributes}
-import cromwell.engine.workflow.CallKey
+import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{CallOutputs, ExecutionEventEntry, ExecutionHash, WorkflowDescriptor}
 import cromwell.logging.WorkflowLogger
 import wdl4s._
 import wdl4s.expression.WdlStandardLibraryFunctions
 import wdl4s.values.WdlValue
-import cromwell.engine.workflow.BackendCallKey
-import cromwell.engine.{ExecutionEventEntry, ExecutionHash, WorkflowDescriptor}
-import cromwell.engine.CallOutputs
-import cromwell.logging.WorkflowLogger
-import cromwell.engine.Hashing._
 
 import scala.concurrent.{ExecutionContext, Future}
 
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
index 029652fb4bb..0a2d9b58728 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackend.scala
@@ -6,7 +6,7 @@ import java.nio.file.{Path, Paths}
 
 import akka.actor.ActorSystem
 import com.google.api.client.http.HttpResponseException
-import com.google.api.client.util.ExponentialBackOff
+import com.google.api.client.util.ExponentialBackOff.Builder
 import com.google.api.services.genomics.model.Parameter
 import com.typesafe.scalalogging.LazyLogging
 import cromwell.engine.ExecutionIndex.IndexEnhancedInt
@@ -25,7 +25,7 @@ import cromwell.engine.io.gcs._
 import cromwell.engine.workflow.{BackendCallKey, WorkflowOptions}
 import cromwell.engine.{AbortRegistrationFunction, CallOutput, CallOutputs, HostInputs, _}
 import cromwell.logging.WorkflowLogger
-import cromwell.util.{SimpleExponentialBackoff, AggregatedException, TryUtil}
+import cromwell.util.{AggregatedException, SimpleExponentialBackoff, TryUtil}
 import wdl4s.expression.NoFunctions
 import wdl4s.values._
 import wdl4s.{Call, CallInputs, Scatter, UnsatisfiedInputsException, _}
@@ -113,7 +113,6 @@ object JesBackend {
   }
 
   def isFatalJesException(t: Throwable): Boolean = t match {
-      // Unauthorized
     case e: HttpResponseException if e.getStatusCode == 403 => true
     case _ => false
   }
@@ -190,6 +189,17 @@ case class JesBackend(actorSystem: ActorSystem)
   with ProductionJesConfiguration {
   type BackendCall = JesBackendCall
 
+  /**
+    * Exponential Backoff Builder to be used when polling for call status.
+    */
+  final private lazy val pollBackoffBuilder = new Builder()
+    .setInitialIntervalMillis(20.seconds.toMillis.toInt)
+    .setMaxElapsedTimeMillis(Int.MaxValue)
+    .setMaxIntervalMillis(10.minutes.toMillis.toInt)
+    .setMultiplier(1.1D)
+
+  override def pollBackoff = pollBackoffBuilder.build()
+
   // FIXME: Add proper validation of jesConf and have it happen up front to provide fail-fast behavior (will do as a separate PR)
 
   override def adjustInputPaths(callKey: BackendCallKey,
diff --git a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
index ad131c752b6..ef4c42a7cec 100644
--- a/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/jes/JesBackendCall.scala
@@ -3,9 +3,9 @@ package cromwell.engine.backend.jes
 import java.nio.file.Paths
 
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
+import com.google.api.client.util.ExponentialBackOff.Builder
 import com.typesafe.scalalogging.LazyLogging
-import wdl4s._
-import wdl4s.values.WdlFile
+import cromwell.engine.Hashing._
 import cromwell.engine.backend.jes.JesBackend._
 import cromwell.engine.backend.jes.Run.TerminalRunStatus
 import cromwell.engine.backend.jes.authentication.ProductionJesAuthentication
@@ -13,11 +13,13 @@ import cromwell.engine.backend.{BackendCall, CallLogs, JobKey, _}
 import cromwell.engine.io.gcs.GcsPath
 import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, CallContext, WorkflowDescriptor, _}
+import wdl4s._
+import wdl4s.values.WdlFile
 
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
-import Hashing._
 
 object JesBackendCall {
   def jesLogBasename(key: BackendCallKey) = {
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
index 9ddab24b4de..b497f9a0901 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackend.scala
@@ -5,19 +5,19 @@ import java.nio.file.{Files, Path, Paths}
 
 import akka.actor.ActorSystem
 import better.files._
+import com.google.api.client.util.ExponentialBackOff.Builder
 import com.typesafe.config.ConfigFactory
 import cromwell.engine.ExecutionIndex._
 import cromwell.engine._
-import cromwell.engine.backend._
+import cromwell.engine.backend.{BackendType, _}
 import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.{CallStatus, ExecutionDatabaseKey}
-import cromwell.engine.workflow.CallKey
 import cromwell.engine.workflow.BackendCallKey
-import cromwell.engine.backend.BackendType
 import cromwell.util.FileUtil._
 import org.slf4j.LoggerFactory
 import wdl4s._
 
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.language.postfixOps
 import scala.sys.process._
@@ -115,6 +115,17 @@ case class LocalBackend(actorSystem: ActorSystem) extends Backend with SharedFil
 
   import LocalBackend._
 
+  /**
+    * Exponential Backoff Builder to be used when polling for call status.
+    */
+  final private lazy val pollBackoffBuilder = new Builder()
+    .setInitialIntervalMillis(10.seconds.toMillis.toInt)
+    .setMaxElapsedTimeMillis(Int.MaxValue)
+    .setMaxIntervalMillis(10.minutes.toMillis.toInt)
+    .setMultiplier(1.1D)
+
+  override def pollBackoff = pollBackoffBuilder.build()
+
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
                         key: BackendCallKey,
                         locallyQualifiedInputs: CallInputs,
diff --git a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
index d4ccaf24c50..1c07a3ec53d 100644
--- a/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/local/LocalBackendCall.scala
@@ -1,11 +1,14 @@
 package cromwell.engine.backend.local
 
 import better.files._
-import wdl4s.CallInputs
+import com.google.api.client.util.ExponentialBackOff
+import com.google.api.client.util.ExponentialBackOff.Builder
 import cromwell.engine.backend.{BackendCall, LocalFileSystemBackendCall, _}
 import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, CallContext, WorkflowDescriptor}
+import wdl4s.CallInputs
 
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 
 case class LocalBackendCall(backend: LocalBackend,
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
index e69ba56e63c..785bafe271b 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackend.scala
@@ -4,6 +4,7 @@ import java.nio.file.Files
 
 import akka.actor.ActorSystem
 import better.files._
+import com.google.api.client.util.ExponentialBackOff.Builder
 import cromwell.engine.backend._
 import cromwell.engine.backend.local.{LocalBackend, SharedFileSystem}
 import cromwell.engine.db.DataAccess._
@@ -16,6 +17,7 @@ import wdl4s.CallInputs
 
 import scala.annotation.tailrec
 import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
 import scala.language.postfixOps
 import scala.sys.process._
 import scala.util.{Failure, Success, Try}
@@ -27,6 +29,17 @@ case class SgeBackend(actorSystem: ActorSystem) extends Backend with SharedFileS
 
   override def backendType = BackendType.SGE
 
+  /**
+    * Exponential Backoff Builder to be used when polling for call status.
+    */
+  final private lazy val pollBackoffBuilder = new Builder()
+    .setInitialIntervalMillis(10.seconds.toMillis.toInt)
+    .setMaxElapsedTimeMillis(Int.MaxValue)
+    .setMaxIntervalMillis(10.minutes.toMillis.toInt)
+    .setMultiplier(1.1D)
+
+  override def pollBackoff = pollBackoffBuilder.build()
+
   override def bindCall(workflowDescriptor: WorkflowDescriptor,
                         key: BackendCallKey,
                         locallyQualifiedInputs: CallInputs,
diff --git a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
index b2eed057a27..77258004546 100644
--- a/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
+++ b/src/main/scala/cromwell/engine/backend/sge/SgeBackendCall.scala
@@ -1,11 +1,13 @@
 package cromwell.engine.backend.sge
 
-import wdl4s.CallInputs
+import com.google.api.client.util.ExponentialBackOff.Builder
 import cromwell.engine.backend.local.LocalBackend
 import cromwell.engine.backend.{BackendCall, LocalFileSystemBackendCall, _}
 import cromwell.engine.workflow.BackendCallKey
 import cromwell.engine.{AbortRegistrationFunction, WorkflowDescriptor}
+import wdl4s.CallInputs
 
+import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 
 case class SgeBackendCall(backend: SgeBackend,
diff --git a/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
index d21eeea3be3..4222d518a24 100644
--- a/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
+++ b/src/main/scala/cromwell/engine/callexecution/BackendCallExecutionActor.scala
@@ -1,10 +1,11 @@
 package cromwell.engine.callexecution
 
+import com.google.api.client.util.ExponentialBackOff
 import cromwell.engine.backend._
+import cromwell.engine.callexecution.CallExecutionActor._
 import cromwell.logging.WorkflowLogger
 
 import scala.language.postfixOps
-import CallExecutionActor._
 
 /**
   * Actor to manage the execution of a single backend call.
@@ -24,4 +25,6 @@ class BackendCallExecutionActor(backendCall: BackendCall) extends CallExecutionA
     case Resume(jobKey) => backendCall.resume(jobKey)
     case UseCachedCall(cachedBackendCall) => backendCall.useCachedCall(cachedBackendCall)
   }
+
+  override val backoff: ExponentialBackOff = backendCall.backend.pollBackoff
 }
diff --git a/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
index ace0968695d..e9a61e41d4d 100644
--- a/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
+++ b/src/main/scala/cromwell/engine/callexecution/CallExecutionActor.scala
@@ -1,18 +1,19 @@
 package cromwell.engine.callexecution
 
 import akka.actor.{Actor, Props}
-import akka.event.{LoggingAdapter, LoggingReceive, Logging}
+import akka.event.{Logging, LoggingReceive}
 import com.google.api.client.util.ExponentialBackOff
+import cromwell.engine.backend.{BackendCall, ExecutionHandle, JobKey, _}
 import cromwell.engine.callactor.CallActor
+import cromwell.engine.callexecution.CallExecutionActor.{ExecutionMode, Finish, IssuePollRequest, PollResponseReceived}
 import cromwell.engine.finalcall.FinalCall
-import cromwell.engine.{CromwellFatalException, CromwellActor}
-import cromwell.engine.backend.{FailedExecutionHandle, BackendCall, JobKey, ExecutionHandle}
-import cromwell.engine.callexecution.CallExecutionActor.{ExecutionMode, PollResponseReceived, IssuePollRequest, Finish}
+import cromwell.engine.{CromwellActor, CromwellFatalException}
 import cromwell.logging.WorkflowLogger
 import wdl4s.Scope
+
 import scala.concurrent.Future
-import scala.language.postfixOps
 import scala.concurrent.duration._
+import scala.language.postfixOps
 import scala.util.{Failure, Success}
 
 object CallExecutionActor {
@@ -48,12 +49,7 @@ trait CallExecutionActor extends Actor with CromwellActor {
     }
   }
 
-  private val backoff = new ExponentialBackOff.Builder()
-    .setInitialIntervalMillis(5.seconds.toMillis.toInt)
-    .setMaxIntervalMillis(30.seconds.toMillis.toInt)
-    .setMaxElapsedTimeMillis(Integer.MAX_VALUE)
-    .setMultiplier(1.1)
-    .build()
+  def backoff: ExponentialBackOff
 
   /**
     * If the `work` `Future` completes successfully, perform the `onSuccess` work, otherwise schedule
diff --git a/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala b/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
index 0decd8118c4..ac48e201fd4 100644
--- a/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
+++ b/src/main/scala/cromwell/engine/callexecution/FinalCallExecutionActor.scala
@@ -1,11 +1,14 @@
 package cromwell.engine.callexecution
 
+import com.google.api.client.util.ExponentialBackOff
+import com.google.api.client.util.ExponentialBackOff.Builder
 import cromwell.engine.backend.ExecutionHandle
 import cromwell.engine.callexecution.CallExecutionActor._
 import cromwell.engine.finalcall.FinalCall
 import cromwell.logging.WorkflowLogger
 
 import scala.concurrent.Future
+import scala.concurrent.duration._
 
 case class FinalCallExecutionActor(override val call: FinalCall) extends CallExecutionActor {
 
@@ -21,4 +24,12 @@ case class FinalCallExecutionActor(override val call: FinalCall) extends CallExe
     case UseCachedCall(cachedBackendCall) => Future.failed(new UnsupportedOperationException("Cannot use cached results for a FinalCall"))
     case _ => call.execute
   }
+
+  /** There is no polling for Final Calls (yet) as they are executed in Cromwell. */
+  override val backoff: ExponentialBackOff = new Builder()
+    .setInitialIntervalMillis(10.seconds.toMillis.toInt)
+    .setMaxElapsedTimeMillis(Int.MaxValue)
+    .setMaxIntervalMillis(10.minutes.toMillis.toInt)
+    .setMultiplier(1.1D)
+    .build()
 }
diff --git a/src/main/scala/cromwell/util/TryUtil.scala b/src/main/scala/cromwell/util/TryUtil.scala
index 19d76c0ed23..6dfd1eeaecc 100644
--- a/src/main/scala/cromwell/util/TryUtil.scala
+++ b/src/main/scala/cromwell/util/TryUtil.scala
@@ -5,7 +5,6 @@ import java.io.{PrintWriter, StringWriter}
 import cromwell.engine.CromwellFatalException
 import cromwell.logging.WorkflowLogger
 
-import scala.annotation.tailrec
 import scala.language.postfixOps
 import scala.util.{Failure, Success, Try}
 
@@ -55,7 +54,7 @@ object TryUtil {
     *
     * Use `retryLimit` value of None indicates to retry indefinitely.
     */
-  @tailrec
+  @annotation.tailrec
   def retryBlock[T](fn: Option[T] => T,
                     backoff: Backoff,
                     isSuccess: T => Boolean = defaultSuccessFunction _,
diff --git a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
index 73f11d24fc8..d3595b40c5f 100644
--- a/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
+++ b/src/test/scala/cromwell/engine/db/slick/SlickDataAccessSpec.scala
@@ -3,6 +3,7 @@ package cromwell.engine.db.slick
 import java.sql.SQLException
 import java.util.UUID
 
+import com.google.api.client.util.ExponentialBackOff
 import cromwell.CromwellSpec.IntegrationTest
 import cromwell.CromwellTestkitSpec.TestWorkflowManagerSystem
 import cromwell.engine.Hashing._
@@ -88,7 +89,7 @@ class SlickDataAccessSpec extends FlatSpec with Matchers with ScalaFutures with
 
     override def backendType: BackendType = throw new NotImplementedError
     override def workflowContext(workflowOptions: WorkflowOptions, workflowId: WorkflowId, name: String): WorkflowContext = throw new NotImplementedError
-
+    override def pollBackoff: ExponentialBackOff = throw new NotImplementedError
   }
 
   // Tests against main database used for command line
diff --git a/src/test/scala/cromwell/util/TryUtilSpec.scala b/src/test/scala/cromwell/util/TryUtilSpec.scala
index 2877b24db63..391c482eea8 100644
--- a/src/test/scala/cromwell/util/TryUtilSpec.scala
+++ b/src/test/scala/cromwell/util/TryUtilSpec.scala
@@ -1,12 +1,14 @@
 package cromwell.util
 
-import cromwell.engine.CromwellFatalException
+import java.util.UUID
+
+import cromwell.engine.{CromwellFatalException, WorkflowId, WorkflowSourceFiles, WorkflowDescriptor}
 import cromwell.logging.WorkflowLogger
 import org.scalatest.mock.MockitoSugar
 import org.scalatest.{FlatSpec, Matchers}
 
-import scala.concurrent.duration._
 import scala.language.postfixOps
+import scala.concurrent.duration._
 import scala.util.Success
 
 class TryUtilSpec extends FlatSpec with Matchers with MockitoSugar {

From 2c70ed3da72a99af962cc37a71e40887b2603e7c Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Tue, 26 Jan 2016 13:17:25 -0500
Subject: [PATCH 29/31] update to wdl4s 0.2

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 9acd8382eac..2818458baf3 100644
--- a/build.sbt
+++ b/build.sbt
@@ -11,7 +11,7 @@ scalaVersion := "2.11.7"
 
 val lenthallV = "0.15"
 
-val wdl4sV = "0.2-b0ec447-SNAPSHOT"
+val wdl4sV = "0.2"
 
 val sprayV = "1.3.2"
 

From c4e4b45e7e1606ecfc56da2c8bea0bbdd87561b0 Mon Sep 17 00:00:00 2001
From: Thibault Jeandet <tjeandet@broadinstitute.org>
Date: Tue, 26 Jan 2016 15:20:36 -0500
Subject: [PATCH 30/31] adjust yaml file

---
 src/main/resources/swagger/cromwell.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/resources/swagger/cromwell.yaml b/src/main/resources/swagger/cromwell.yaml
index c91796f0eee..83f40a404aa 100644
--- a/src/main/resources/swagger/cromwell.yaml
+++ b/src/main/resources/swagger/cromwell.yaml
@@ -597,12 +597,12 @@ definitions:
     required:
       - valid
     properties:
-      valid:
-        type: boolean
-        description: The validation of the workflow
-      error:
+      status:
+        type: string
+        description: The validation status of the workflow
+      message:
         type: string
-        description: The validation error of the workflow
+        description: Details about the validation
   WorkflowSubmitResponse:
     description: ''
     required:

From 8a48eee787184ee960ec8c580b3b542f8dfae04e Mon Sep 17 00:00:00 2001
From: dgtester <dtester@google.com>
Date: Thu, 21 Jan 2016 16:15:27 -0500
Subject: [PATCH 31/31] Aborts jobs on shutdown.

Added configuration option for abortJobsOnTerminate. Added support for server mode. SingleWorkflowRunnerActor now uses the internal state to determine whether the workflow is done (and not a separately maintained flag).

Added configuration option for abortJobsOnTerminate. Added support for server mode. SingleWorkflowRunnerActor now uses the internal state to determine whether the workflow is done (and not a separately maintained flag).
---
 .../scala/cromwell/engine/CromwellActor.scala | 16 +++++++++++++
 .../workflow/SingleWorkflowRunnerActor.scala  | 12 ++++++++++
 .../workflow/WorkflowManagerActor.scala       | 24 +++++++++++++++++--
 3 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/cromwell/engine/CromwellActor.scala b/src/main/scala/cromwell/engine/CromwellActor.scala
index 11d19aacc3f..6017d71a75b 100644
--- a/src/main/scala/cromwell/engine/CromwellActor.scala
+++ b/src/main/scala/cromwell/engine/CromwellActor.scala
@@ -1,10 +1,26 @@
 package cromwell.engine
 
 import akka.util.Timeout
+import com.typesafe.config.{ConfigException, ConfigFactory}
 
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
 trait CromwellActor {
   protected implicit val timeout = Timeout(5 seconds)
+
+  /**
+   * Retrieves the configuration option that determines whether this actor should abort all jobs if it receives
+   * a shutdown hook.
+   * @return - The value of the configuration option, or 'false' if the option isn't specified.
+   */
+  def getAbortJobsOnTerminate: Boolean = {
+    val config=ConfigFactory.load.getConfig("backend")
+    try {
+      config.getBoolean("abortJobsOnTerminate")
+    } catch {
+      case _:ConfigException  => false
+    }
+  }
+
 }
diff --git a/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala b/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
index 0e5457c6d14..8e69829c9fa 100644
--- a/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/SingleWorkflowRunnerActor.scala
@@ -79,6 +79,18 @@ case class SingleWorkflowRunnerActor(source: WorkflowSourceFiles,
     case Event(id: WorkflowId, data) =>
       log.info(s"$tag: workflow ID UUID($id)")
       workflowManager ! SubscribeToWorkflow(id)
+
+      if (getAbortJobsOnTerminate) {
+        Runtime.getRuntime.addShutdownHook(new Thread() {
+          override def run(): Unit = {
+            workflowManager ! WorkflowAbort(id)
+            log.info(s"$tag: Waiting for workflow $id to abort...")
+            while(stateName != Done)
+              Thread.sleep(1000)
+            log.info(s"$tag: Workflow $id aborted.")
+          }
+        })
+      }
       stay using data.copy(id = Option(id))
     case Event(Transition(_, _, WorkflowSucceeded), data) =>
       workflowManager ! WorkflowOutputs(data.id.get)
diff --git a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
index 6b5e6dd0380..6a01b80572c 100644
--- a/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
+++ b/src/main/scala/cromwell/engine/workflow/WorkflowManagerActor.scala
@@ -14,14 +14,15 @@ import cromwell.engine.db.DataAccess._
 import cromwell.engine.db.ExecutionDatabaseKey
 import cromwell.engine.db.slick._
 import cromwell.engine.workflow.WorkflowActor.{Restart, Start}
+import cromwell.server.CromwellServer
 import cromwell.util.WriteOnceStore
 import cromwell.webservice._
 import org.joda.time.DateTime
 import spray.json._
 import wdl4s._
-
+import wdl4s.values.WdlFile
 import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
+import scala.concurrent.{Await, Future}
 import scala.concurrent.duration._
 import scala.io.Source
 import scala.language.postfixOps
@@ -68,6 +69,25 @@ class WorkflowManagerActor(backend: Backend) extends Actor with CromwellActor {
 
   private val workflowStore = new WriteOnceStore[WorkflowId, WorkflowActorRef]
 
+  if (getAbortJobsOnTerminate) {
+    Runtime.getRuntime.addShutdownHook(new Thread() {
+      override def run(): Unit = {
+        log.info(s"$tag: Received shutdown signal. Aborting all running workflows...")
+        workflowStore.toMap.foreach{case (id, actor)=>
+          CromwellServer.workflowManagerActor ! WorkflowManagerActor.WorkflowAbort(id)
+        }
+        var numRemaining = -1
+        while(numRemaining != 0) {
+          Thread.sleep(1000)
+          val result = globalDataAccess.getWorkflowsByState(Seq(WorkflowRunning, WorkflowAborting))
+          numRemaining = Await.result(result,Duration.Inf).size
+          log.info(s"$tag: Waiting for all workflows to abort ($numRemaining remaining).")
+        }
+        log.info(s"$tag: All workflows aborted.")
+      }
+    })
+  }
+
   override def preStart() {
     restartIncompleteWorkflows()
   }