From 1b2faaf249fbe07464be2f421c70b11f12d9e93f Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sat, 1 Jun 2019 12:30:42 -0400
Subject: [PATCH 01/28] Google Auth Updates - Replaced to-be-deprecated
 Credential (no 's') with Adapter around Credentials - Removed dupe
 credentials adapting from PipelinesApiFactoryInterface - Move service
 specific scopes (KMS, Genomics) out of GoogleAuthMode - Changed credential
 creation methods to take scala collections

---
 .../src/main/scala/centaur/test/Test.scala    |  6 +-
 .../gcp/auth/GoogleAuthMode.scala             | 98 ++++---------------
 .../drs/DrsPathBuilderFactory.scala           |  9 +-
 .../filesystems/gcs/GcsPathBuilder.scala      |  3 +-
 .../cromwell/filesystems/gcs/GoogleUtil.scala | 11 ++-
 .../WorkbenchHealthMonitorServiceActor.scala  |  7 +-
 .../PipelinesApiInitializationActor.scala     | 51 ++++++++--
 .../api/PipelinesApiFactoryInterface.scala    | 15 +--
 .../pipelines/v2alpha1/GenomicsFactory.scala  | 26 +++--
 9 files changed, 98 insertions(+), 128 deletions(-)

diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala
index 89a528c1e97..e384539ae46 100644
--- a/centaur/src/main/scala/centaur/test/Test.scala
+++ b/centaur/src/main/scala/centaur/test/Test.scala
@@ -12,7 +12,8 @@ import centaur.test.metadata.WorkflowFlatMetadata
 import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur.test.submit.SubmitHttpResponse
 import centaur.test.workflow.Workflow
-import com.google.api.services.genomics.Genomics
+import com.google.api.services.genomics.{Genomics, GenomicsScopes}
+import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
 import com.google.auth.http.HttpCredentialsAdapter
 import com.google.auth.oauth2.ServiceAccountCredentials
@@ -86,9 +87,10 @@ object Operations {
   lazy val googleConf: Config = CentaurConfig.conf.getConfig("google")
   lazy val authName: String = googleConf.getString("auth")
   lazy val genomicsEndpointUrl: String = googleConf.getString("genomics.endpoint-url")
+  lazy val genomicsAndStorageScopes = List(StorageScopes.CLOUD_PLATFORM_READ_ONLY, GenomicsScopes.GENOMICS)
   lazy val credentials: Credentials = configuration.auth(authName)
     .unsafe
-    .pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup)
+    .credentials(genomicsAndStorageScopes)
   lazy val credentialsProjectOption: Option[String] = {
     Option(credentials) collect {
       case serviceAccountCredentials: ServiceAccountCredentials => serviceAccountCredentials.getProjectId
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala
index 018dff2f364..74f56dbfc68 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala
@@ -4,15 +4,9 @@ import java.io.{ByteArrayInputStream, FileNotFoundException, InputStream}
 import java.net.HttpURLConnection._
 
 import better.files.File
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
-import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential
 import com.google.api.client.http.HttpResponseException
 import com.google.api.client.json.jackson2.JacksonFactory
-import com.google.api.services.cloudkms.v1.CloudKMS
-import com.google.api.services.compute.ComputeScopes
-import com.google.api.services.genomics.v2alpha1.GenomicsScopes
-import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
 import com.google.auth.http.HttpTransportFactory
 import com.google.auth.oauth2.{GoogleCredentials, OAuth2Credentials, ServiceAccountCredentials, UserCredentials}
@@ -55,13 +49,6 @@ object GoogleAuthMode {
   val DockerCredentialsEncryptionKeyNameKey = "docker_credentials_key_name"
   val DockerCredentialsTokenKey = "docker_credentials_token"
 
-  private val PipelinesApiScopes = List(
-    StorageScopes.DEVSTORAGE_FULL_CONTROL,
-    StorageScopes.DEVSTORAGE_READ_WRITE,
-    GenomicsScopes.GENOMICS,
-    ComputeScopes.COMPUTE
-  )
-
   def checkReadable(file: File) = {
     if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable")
   }
@@ -85,24 +72,6 @@ object GoogleAuthMode {
     }
   }
 
-  def encryptKms(keyName: String, credential: GoogleCredential, plainText: String) = {
-    import com.google.api.services.cloudkms.v1.CloudKMSScopes
-
-    // Depending on the environment that provides the default credentials (e.g. Compute Engine, App
-    // Engine), the credentials may require us to specify the scopes we need explicitly.
-    // Check for this case, and inject the scope if required.
-    val scopedCredential = if (credential.createScopedRequired) credential.createScoped(CloudKMSScopes.all) else credential
-
-    val kms = new CloudKMS.Builder(httpTransport, jsonFactory, scopedCredential)
-      .setApplicationName("cromwell")
-      .build()
-
-    import com.google.api.services.cloudkms.v1.model.EncryptRequest
-    val request = new EncryptRequest().encodePlaintext(plainText.toCharArray.map(_.toByte))
-    val response = kms.projects.locations.keyRings.cryptoKeys.encrypt(keyName, request).execute
-    response.getCiphertext
-  }
-
   /** Used for both checking that the credential is valid and creating a fresh credential. */
   private def refreshCredentials(credentials: Credentials): Unit = {
     credentials.refresh()
@@ -115,49 +84,33 @@ sealed trait GoogleAuthMode {
 
   def name: String
 
-  // Create a Credential object from the google.api.client.auth library (https://github.com/google/google-api-java-client)
-  def credentials(options: OptionLookup, scopes: java.util.Collection[String]): OAuth2Credentials
-
   /**
-    * Create a credential object suitable for use with Pipelines API.
-    *
-    * @param options A lookup for external credential information.
-    * @return Credentials with scopes compatible with the Genomics API compute and storage.
+    * Creates OAuth credentials with the specified scopes.
     */
-  def pipelinesApiCredentials(options: OptionLookup): OAuth2Credentials = {
-    credentials(options, PipelinesApiScopes.asJavaCollection)
-  }
+  def credentials(options: OptionLookup, scopes: Iterable[String]): OAuth2Credentials
 
   /**
     * Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes).
     * Only valid for credentials that are NOT externally provided, such as ApplicationDefault.
     */
   def credentials(scopes: Iterable[String]): OAuth2Credentials = {
-    credentials(GoogleAuthMode.NoOptionLookup, scopes.asJavaCollection)
-  }
-
-  /**
-    * Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes).
-    * Only valid for credentials that are NOT externally provided, such as ApplicationDefault.
-    */
-  def credentials(scopes: java.util.Collection[String]): OAuth2Credentials = {
     credentials(GoogleAuthMode.NoOptionLookup, scopes)
   }
 
   /**
-    * Alias for credentials(GoogleAuthMode.NoOptionLookup, Set.empty).
+    * Alias for credentials(GoogleAuthMode.NoOptionLookup, Nil).
     * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault.
     */
   private[auth] def credentials(): OAuth2Credentials = {
-    credentials(GoogleAuthMode.NoOptionLookup, java.util.Collections.emptySet[String])
+    credentials(GoogleAuthMode.NoOptionLookup, Nil)
   }
 
   /**
-    * Alias for credentials(options, Set.empty).
+    * Alias for credentials(options, Nil).
     * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault.
     */
   private[auth] def credentials(options: OptionLookup): OAuth2Credentials = {
-    credentials(options, java.util.Collections.emptySet[String])
+    credentials(options, Nil)
   }
 
   def requiresAuthFile: Boolean = false
@@ -176,20 +129,14 @@ sealed trait GoogleAuthMode {
       case Success(_) => credential
     }
   }
-
-  def apiClientGoogleCredential(options: OptionLookup): Option[GoogleCredential] = None
 }
 
 case object MockAuthMode extends GoogleAuthMode {
   override val name = "no_auth"
 
-  override def credentials(unusedOptions: OptionLookup, unusedScopes: java.util.Collection[String]): NoCredentials = {
+  override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = {
     NoCredentials.getInstance
   }
-
-  override def apiClientGoogleCredential(options: OptionLookup): Option[MockGoogleCredential] = {
-    Option(new MockGoogleCredential.Builder().build())
-  }
 }
 
 object ServiceAccountMode {
@@ -204,20 +151,12 @@ object ServiceAccountMode {
 
 }
 
-trait HasApiClientGoogleCredentialStream { self: GoogleAuthMode =>
-  protected def credentialStream(options: OptionLookup): InputStream
-
-  override def apiClientGoogleCredential(options: OptionLookup): Option[GoogleCredential] = Option(GoogleCredential.fromStream(credentialStream(options)))
-}
-
 final case class ServiceAccountMode(override val name: String,
                                     fileFormat: CredentialFileFormat)
-  extends GoogleAuthMode with HasApiClientGoogleCredentialStream {
+  extends GoogleAuthMode {
   private val credentialsFile = File(fileFormat.file)
   checkReadable(credentialsFile)
 
-  override protected def credentialStream(options: OptionLookup): InputStream = credentialsFile.newInputStream
-
   private lazy val serviceAccountCredentials: ServiceAccountCredentials = {
     fileFormat match {
       case PemFileFormat(accountId, _) =>
@@ -228,25 +167,24 @@ final case class ServiceAccountMode(override val name: String,
   }
 
   override def credentials(unusedOptions: OptionLookup,
-                           scopes: java.util.Collection[String]): GoogleCredentials = {
-    val scopedCredentials = serviceAccountCredentials.createScoped(scopes)
+                           scopes: Iterable[String]): GoogleCredentials = {
+    val scopedCredentials = serviceAccountCredentials.createScoped(scopes.asJavaCollection)
     validateCredentials(scopedCredentials)
   }
 }
 
-final case class UserServiceAccountMode(override val name: String)
-  extends GoogleAuthMode with HasApiClientGoogleCredentialStream {
+final case class UserServiceAccountMode(override val name: String) extends GoogleAuthMode {
   private def extractServiceAccount(options: OptionLookup): String = {
     extract(options, UserServiceAccountKey)
   }
 
-  override protected def credentialStream(options: OptionLookup): InputStream = {
+  private def credentialStream(options: OptionLookup): InputStream = {
     new ByteArrayInputStream(extractServiceAccount(options).getBytes("UTF-8"))
   }
 
-  override def credentials(options: OptionLookup, scopes: java.util.Collection[String]): GoogleCredentials = {
+  override def credentials(options: OptionLookup, scopes: Iterable[String]): GoogleCredentials = {
     val newCredentials = ServiceAccountCredentials.fromStream(credentialStream(options))
-    val scopedCredentials: GoogleCredentials = newCredentials.createScoped(scopes)
+    val scopedCredentials: GoogleCredentials = newCredentials.createScoped(scopes.asJavaCollection)
     validateCredentials(scopedCredentials)
   }
 }
@@ -267,7 +205,7 @@ final case class UserMode(override val name: String,
     validateCredentials(UserCredentials.fromStream(secretsStream))
   }
 
-  override def credentials(unusedOptions: OptionLookup, unusedScopes: java.util.Collection[String]): OAuth2Credentials = {
+  override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): OAuth2Credentials = {
     userCredentials
   }
 }
@@ -278,11 +216,9 @@ object ApplicationDefaultMode {
 
 final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode {
   override def credentials(unusedOptions: OptionLookup,
-                           unusedScopes: java.util.Collection[String]): GoogleCredentials = {
+                           unusedScopes: Iterable[String]): GoogleCredentials = {
     ApplicationDefaultMode.applicationDefaultCredentials
   }
-
-  override def apiClientGoogleCredential(unused: OptionLookup): Option[GoogleCredential] = Option(GoogleCredential.getApplicationDefault(httpTransport, jsonFactory))
 }
 
 final case class RefreshTokenMode(name: String,
@@ -297,7 +233,7 @@ final case class RefreshTokenMode(name: String,
     extract(options, RefreshTokenOptionKey)
   }
 
-  override def credentials(options: OptionLookup, unusedScopes: java.util.Collection[String]): UserCredentials = {
+  override def credentials(options: OptionLookup, unusedScopes: Iterable[String]): UserCredentials = {
     val refreshToken = extractRefreshToken(options)
     val newCredentials: UserCredentials = UserCredentials
       .newBuilder()
diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
index 4bb95ac6216..3f4689ca588 100644
--- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
+++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
@@ -16,10 +16,8 @@ import cromwell.core.WorkflowOptions
 import cromwell.core.path.{PathBuilder, PathBuilderFactory}
 import org.apache.http.impl.client.HttpClientBuilder
 
-import scala.collection.JavaConverters._
 import scala.concurrent.{ExecutionContext, Future}
 
-
 /**
   * Cromwell Wrapper around DrsFileSystems to load the configuration.
   * This class is used as the global configuration class in the drs filesystem
@@ -54,10 +52,9 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single
 
   private def inputReadChannel(url: String, urlScheme: String, serviceAccount: String): IO[ReadableByteChannel] =  {
     urlScheme match {
-      case GcsScheme => {
+      case GcsScheme =>
         val Array(bucket, fileToBeLocalized) = url.replace(s"$GcsScheme://", "").split("/", 2)
         gcsInputStream(GcsFilePath(bucket, fileToBeLocalized), serviceAccount)
-      }
       case otherScheme => IO.raiseError(new UnsupportedOperationException(s"DRS currently doesn't support reading files for $otherScheme."))
     }
   }
@@ -82,8 +79,8 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single
       // Profile and Email scopes are requirements for interacting with Martha v2
       Oauth2Scopes.USERINFO_EMAIL,
       Oauth2Scopes.USERINFO_PROFILE
-    ).asJavaCollection
-    val authCredentials = googleAuthMode.credentials((key: String) => options.get(key).get, marthaScopes)
+    )
+    val authCredentials = googleAuthMode.credentials(options.get(_).get, marthaScopes)
 
     Future.successful(DrsPathBuilder(new DrsCloudNioFileSystemProvider(singletonConfig.config, authCredentials, httpClientBuilder, drsReadInterpreter)))
   }
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
index 258e0991c32..93aad37be57 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
@@ -8,6 +8,7 @@ import akka.http.scaladsl.model.ContentTypes
 import better.files.File.OpenOptions
 import cats.effect.IO
 import com.google.api.gax.retrying.RetrySettings
+import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
 import com.google.cloud.storage.Storage.BlobTargetOption
 import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStoragePath}
@@ -100,7 +101,7 @@ object GcsPathBuilder {
                    cloudStorageConfiguration: CloudStorageConfiguration,
                    options: WorkflowOptions,
                    defaultProject: Option[String])(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = {
-    authMode.retryPipelinesApiCredentials(options) map { credentials =>
+    authMode.retryCredentials(options, List(StorageScopes.DEVSTORAGE_FULL_CONTROL)) map { credentials =>
       fromCredentials(credentials,
         applicationName,
         retrySettings,
diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala
index 8f3e5abf9eb..9fa2a200060 100644
--- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala
+++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala
@@ -24,13 +24,16 @@ object GoogleUtil {
 
   implicit class EnhancedGoogleAuthMode(val googleAuthMode: GoogleAuthMode) extends AnyVal {
     /**
-      * Retries getting the pipelines API credentials three times.
+      * Retries getting the credentials three times.
+      *
+      * There is nothing GCS specific about this method. This package just happens to be the lowest level with access
+      * to core's version of Retry + cloudSupport's implementation of GoogleAuthMode.
       */
-    def retryPipelinesApiCredentials(options: WorkflowOptions)
-                                    (implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
+    def retryCredentials(options: WorkflowOptions, scopes: Iterable[String])
+                        (implicit actorSystem: ActorSystem, executionContext: ExecutionContext): Future[Credentials] = {
       def credential(): Credentials = {
         try {
-          googleAuthMode.pipelinesApiCredentials((key: String) => options.get(key).get)
+          googleAuthMode.credentials(options.get(_).get, scopes)
         } catch {
           case exception: OptionLookupException =>
             throw new IllegalArgumentException(s"Missing parameters in workflow options: ${exception.key}", exception)
diff --git a/services/src/main/scala/cromwell/services/healthmonitor/impl/workbench/WorkbenchHealthMonitorServiceActor.scala b/services/src/main/scala/cromwell/services/healthmonitor/impl/workbench/WorkbenchHealthMonitorServiceActor.scala
index c974ec32261..694cdfd8722 100644
--- a/services/src/main/scala/cromwell/services/healthmonitor/impl/workbench/WorkbenchHealthMonitorServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/healthmonitor/impl/workbench/WorkbenchHealthMonitorServiceActor.scala
@@ -8,6 +8,8 @@ import cats.instances.future._
 import cats.syntax.functor._
 import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
 import com.google.api.gax.retrying.RetrySettings
+import com.google.api.services.genomics.v2alpha1.GenomicsScopes
+import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
 import com.google.auth.http.HttpCredentialsAdapter
 import com.typesafe.config.Config
@@ -57,7 +59,8 @@ abstract class WorkbenchHealthMonitorServiceActor(val serviceConfig: Config, glo
   private def checkGcs(): Future[SubsystemStatus] = {
     // For any expected production usage of this check, the GCS bucket should be public read */
     val gcsBucketToCheck = serviceConfig.as[String]("gcs-bucket-to-check")
-    val storage = Future(googleAuth.pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup)) map { credentials =>
+    val storageScopes = List(StorageScopes.DEVSTORAGE_READ_ONLY)
+    val storage = Future(googleAuth.credentials(storageScopes)) map { credentials =>
       GcsStorage.gcsStorage(googleConfig.applicationName, credentials, RetrySettings.newBuilder().build())
     }
     storage map { _.buckets.get(gcsBucketToCheck).execute() } as OkStatus
@@ -71,7 +74,7 @@ abstract class WorkbenchHealthMonitorServiceActor(val serviceConfig: Config, glo
     val papiProjectId = papiConfig.as[String]("project")
 
     val check = for {
-      credentials <- Future(googleAuth.pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup))
+      credentials <- Future(googleAuth.credentials(List(GenomicsScopes.GENOMICS)))
       genomicsChecker = if (papiProviderConfig.as[String]("actor-factory").contains("v2alpha1"))
         GenomicsCheckerV2(googleConfig.applicationName, googleAuth, endpointUrl, credentials, papiProjectId)
       else
diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiInitializationActor.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiInitializationActor.scala
index 3a01f744e0c..7e56eed5701 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiInitializationActor.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiInitializationActor.scala
@@ -1,10 +1,18 @@
 package cromwell.backend.google.pipelines.common
 
 import akka.actor.ActorRef
+import com.google.api.services.cloudkms.v1.model.EncryptRequest
+import com.google.api.services.cloudkms.v1.{CloudKMS, CloudKMSScopes}
+import com.google.api.services.genomics.GenomicsScopes
+import com.google.api.services.storage.StorageScopes
 import com.google.auth.Credentials
+import com.google.auth.http.HttpCredentialsAdapter
+import com.google.auth.oauth2.OAuth2Credentials
+import cromwell.backend.google.pipelines.common.PipelinesApiInitializationActor._
 import cromwell.backend.google.pipelines.common.api.PipelinesApiRequestFactory
 import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder}
 import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor}
+import cromwell.cloudsupport.gcp.auth.GoogleAuthMode.{httpTransport, jsonFactory}
 import cromwell.cloudsupport.gcp.auth.{GoogleAuthMode, UserServiceAccountMode}
 import cromwell.core.Dispatcher
 import cromwell.core.io.AsyncIoActorClient
@@ -40,12 +48,31 @@ class PipelinesApiInitializationActor(pipelinesParams: PipelinesApiInitializatio
     PipelinesApiRuntimeAttributes.runtimeAttributesBuilder(pipelinesConfiguration)
 
   // Credentials object for the GCS API
-  private lazy val gcsCredentials: Future[Credentials] =
-    pipelinesConfiguration.papiAttributes.auths.gcs.retryPipelinesApiCredentials(workflowOptions)
+  private lazy val gcsCredentials: Future[Credentials] = pipelinesConfiguration.papiAttributes.auths.gcs
+      .retryCredentials(workflowOptions, List(StorageScopes.DEVSTORAGE_FULL_CONTROL))
 
   // Credentials object for the Genomics API
-  private lazy val genomicsCredentials: Future[Credentials] =
-    pipelinesConfiguration.papiAttributes.auths.genomics.retryPipelinesApiCredentials(workflowOptions)
+  private lazy val genomicsCredentials: Future[Credentials] = pipelinesConfiguration.papiAttributes.auths.genomics
+    .retryCredentials(workflowOptions, List(
+      GenomicsScopes.GENOMICS,
+      /*
+      Genomics Pipelines API v1alpha2 requires the COMPUTE scope. Does not seem to be required for v2alpha1.
+       */
+      GenomicsScopes.COMPUTE,
+      /*
+      Used to write so-called "auth" files. The `gcsAuthFilePath` could probably be refactored such that *this* created
+      `genomicsCredentials` doesn't actually need DEVSTORAGE_FULL_CONTROL, but it's also not clear how the magic
+      Genomics Pipelines API parameter "__extra_config_gcs_path" works nor where it's documented.
+
+      See also:
+      - cromwell.backend.google.pipelines.common.PipelinesApiWorkflowPaths#gcsAuthFilePath
+      - https://github.com/broadinstitute/cromwell/pull/2435
+      - cromwell.backend.google.pipelines.common.PipelinesApiConfiguration#needAuthFileUpload
+      - cromwell.cloudsupport.gcp.auth.GoogleAuthMode#requiresAuthFile
+      - cromwell.backend.google.pipelines.common.PipelinesApiAsyncBackendJobExecutionActor#gcsAuthParameter
+       */
+      StorageScopes.DEVSTORAGE_FULL_CONTROL,
+    ))
 
   // Genomics object to access the Genomics API
   private lazy val genomics: Future[PipelinesApiRequestFactory] = {
@@ -97,8 +124,9 @@ class PipelinesApiInitializationActor(pipelinesParams: PipelinesApiInitializatio
       plain <- unencrypted
       auth <- effectiveAuth
       key <- privateDockerEncryptionKeyName
-      cred <- auth.apiClientGoogleCredential(k => workflowOptions.get(k).get)
-    } yield GoogleAuthMode.encryptKms(key, cred, plain)
+      credentials = auth.credentials(workflowOptions.get(_).get, List(CloudKMSScopes.CLOUD_PLATFORM))
+      encrypted = encryptKms(key, credentials, plain)
+    } yield encrypted
   }
 
   override lazy val workflowPaths: Future[PipelinesApiWorkflowPaths] = for {
@@ -140,4 +168,15 @@ class PipelinesApiInitializationActor(pipelinesParams: PipelinesApiInitializatio
 object PipelinesApiInitializationActor {
   // For metadata publishing purposes default to using the name of a standard stream as the stream's filename.
   def defaultStandardStreamNameToFileNameMetadataMapper(pipelinesApiJobPaths: PipelinesApiJobPaths, streamName: String): String = streamName
+
+  def encryptKms(keyName: String, credentials: OAuth2Credentials, plainText: String): String = {
+    val httpCredentialsAdapter = new HttpCredentialsAdapter(credentials)
+    val kms = new CloudKMS.Builder(httpTransport, jsonFactory, httpCredentialsAdapter)
+      .setApplicationName("cromwell")
+      .build()
+
+    val request = new EncryptRequest().encodePlaintext(plainText.toCharArray.map(_.toByte))
+    val response = kms.projects.locations.keyRings.cryptoKeys.encrypt(keyName, request).execute
+    response.getCiphertext
+  }
 }
diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiFactoryInterface.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiFactoryInterface.scala
index 1b5ccc49300..fe5908fae9b 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiFactoryInterface.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/api/PipelinesApiFactoryInterface.scala
@@ -1,9 +1,8 @@
 package cromwell.backend.google.pipelines.common.api
 
-import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
+import com.google.api.client.http.HttpRequestInitializer
 import com.google.auth.Credentials
 import com.google.auth.http.HttpCredentialsAdapter
-import mouse.all._
 
 /**
   * The interface provides a single method to build a PipelinesApiRequestFactory
@@ -13,17 +12,11 @@ import mouse.all._
   * Pipelines API for this workflow.
   */
 abstract class PipelinesApiFactoryInterface {
-  private def httpRequestInitializerFromCredentials(credentials: Credentials) = {
-    val delegate = new HttpCredentialsAdapter(credentials)
-    new HttpRequestInitializer() {
-      def initialize(httpRequest: HttpRequest) = {
-        delegate.initialize(httpRequest)
-      }
-    }
+  final def fromCredentials(credentials: Credentials): PipelinesApiRequestFactory = {
+    val httpCredentialsAdapter = new HttpCredentialsAdapter(credentials)
+    build(httpCredentialsAdapter)
   }
   
-  final def fromCredentials(credentials: Credentials): PipelinesApiRequestFactory = build(credentials |> httpRequestInitializerFromCredentials)
-  
   protected def build(httpRequestInitializer: HttpRequestInitializer): PipelinesApiRequestFactory
 
   def usesEncryptedDocker: Boolean
diff --git a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
index f0d852e3ccc..6e970aa03aa 100644
--- a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
+++ b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
@@ -10,6 +10,7 @@ import com.google.api.services.genomics.v2alpha1.model._
 import com.google.api.services.genomics.v2alpha1.{Genomics, GenomicsScopes}
 import com.google.api.services.oauth2.Oauth2Scopes
 import com.google.api.services.storage.StorageScopes
+import com.google.auth.http.HttpCredentialsAdapter
 import cromwell.backend.google.pipelines.common.PipelinesApiConfigurationAttributes.{LocalizationConfiguration, VirtualPrivateCloudConfiguration}
 import cromwell.backend.google.pipelines.common.api.PipelinesApiRequestFactory.CreatePipelineParameters
 import cromwell.backend.google.pipelines.common.api.{PipelinesApiFactoryInterface, PipelinesApiRequestFactory}
@@ -39,7 +40,7 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
   override def build(initializer: HttpRequestInitializer): PipelinesApiRequestFactory = new PipelinesApiRequestFactory {
     implicit lazy val googleProjectMetadataLabelDecoder: Decoder[ProjectLabels] = deriveDecoder
 
-    val ResourceManagerAuthScopes = List(GenomicsScopes.CLOUD_PLATFORM).asJava
+    val ResourceManagerAuthScopes = List(GenomicsScopes.CLOUD_PLATFORM)
     val VirtualPrivateCloudNetworkPath = "projects/%s/global/networks/%s/"
 
     val genomics = new Genomics.Builder(
@@ -61,23 +62,18 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
     override def runRequest(createPipelineParameters: CreatePipelineParameters, jobLogger: JobLogger): HttpRequest = {
 
       def projectMetadataRequest(vpcConfig: VirtualPrivateCloudConfiguration): IO[HttpRequest] = {
-        val workflowOptions = createPipelineParameters.jobDescriptor.workflowDescriptor.workflowOptions
+        IO {
+          val workflowOptions = createPipelineParameters.jobDescriptor.workflowDescriptor.workflowOptions
+          val credentials = vpcConfig.auth.credentials(workflowOptions.get(_).get, ResourceManagerAuthScopes)
 
-        val googleCredentialOption = vpcConfig.auth.apiClientGoogleCredential((key: String) => workflowOptions.get(key).get)
+          val httpCredentialsAdapter = new HttpCredentialsAdapter(credentials)
+          val cloudResourceManagerBuilder = new CloudResourceManager
+          .Builder(GoogleAuthMode.httpTransport, GoogleAuthMode.jsonFactory, httpCredentialsAdapter)
+            .build()
 
-        googleCredentialOption match {
-          case None => IO.raiseError(new RuntimeException(s"Programmer Error: Unable to find Google Credential for auth `${vpcConfig.auth.name}`."))
-          case Some(googleCredential) => IO {
-            val auth = googleCredential.createScoped(ResourceManagerAuthScopes)
+          val project = cloudResourceManagerBuilder.projects().get(createPipelineParameters.projectId)
 
-            val cloudResourceManagerBuilder = new CloudResourceManager
-              .Builder(GoogleAuthMode.httpTransport, GoogleAuthMode.jsonFactory, auth)
-              .build()
-
-            val project = cloudResourceManagerBuilder.projects().get(createPipelineParameters.projectId)
-
-            project.buildHttpRequest()
-          }
+          project.buildHttpRequest()
         }
       }
 

From d446f833b0646c563938103ba08d17c8331d67d6 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Sun, 2 Jun 2019 20:36:43 -0400
Subject: [PATCH 02/28] Increase bootDiskSizeGb for gpu_cuda_image

---
 .../resources/standardTestCases/gpu_on_papi/gpu_cuda_image.wdl   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/centaur/src/main/resources/standardTestCases/gpu_on_papi/gpu_cuda_image.wdl b/centaur/src/main/resources/standardTestCases/gpu_on_papi/gpu_cuda_image.wdl
index 5ce0a5c33a8..f7c784cb574 100644
--- a/centaur/src/main/resources/standardTestCases/gpu_on_papi/gpu_cuda_image.wdl
+++ b/centaur/src/main/resources/standardTestCases/gpu_on_papi/gpu_cuda_image.wdl
@@ -33,6 +33,7 @@ task get_machine_info {
 
     runtime {
         docker: "nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04"
+        bootDiskSizeGb: 20
         gpuType: "nvidia-tesla-k80"
         gpuCount: 1
         nvidiaDriverVersion: driver_version

From 5c6b6f1a58e134796a0516b91fc6b768f5cff4cb Mon Sep 17 00:00:00 2001
From: root <anichols@broadinstitute.org>
Date: Mon, 3 Jun 2019 21:57:58 +0000
Subject: [PATCH 03/28] Update cromwell version from 42 to 43

---
 project/Version.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/Version.scala b/project/Version.scala
index 542cb313ae7..e6392717a64 100644
--- a/project/Version.scala
+++ b/project/Version.scala
@@ -5,7 +5,7 @@ import sbt._
 
 object Version {
   // Upcoming release, or current if we're on a master / hotfix branch
-  val cromwellVersion = "42"
+  val cromwellVersion = "43"
 
   /**
     * Returns true if this project should be considered a snapshot.

From b8b99c3b5cbc9e780e4ee9612b8aba6c3c5747fc Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Wed, 5 Jun 2019 22:58:09 +0200
Subject: [PATCH 04/28] Retry jobs that have been killed by a cluster backend.
 [BA-5669] (#5003)

---
 .../sfs/config/ConfigAsyncJobExecutionActor.scala     | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
index 97e382b4c44..71656cc10dd 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/impl/sfs/config/ConfigAsyncJobExecutionActor.scala
@@ -304,7 +304,16 @@ class DispatchedConfigAsyncJobExecutionActor(override val standardParams: Standa
           jobLogger.error(s"Return file not found after ${exitCodeTimeout.getOrElse(backupError)} seconds, assuming external kill")
 
           val returnCodeTemp = jobPaths.returnCode.plusExt("kill")
-          returnCodeTemp.write(s"$SIGTERM${System.lineSeparator}")
+
+          // If SIGTERM, SIGKILL or SIGINT codes are used, cromwell will assume the job has been aborted by cromwell.
+          // And it will therefore NOT retry the job. Which makes perfect sense. Best not to change that in the
+          // StandardAsyncExecutionActor code, but give a user-defined return code here.
+          // http://tldp.org/LDP/abs/html/exitcodes.html gives information on exit codes and suggests restricting
+          // user-defined exit codes to the range 64-113.
+          // Since it is arbitrary which code is chosen from that range, and it has to relate with the unpleasant
+          // business of 'killing'. 79 was chosen. The year that video killed the radio star: https://youtu.be/W8r-tXRLazs
+          returnCodeTemp.appendLine("79")
+
           try {
             returnCodeTemp.moveTo(jobPaths.returnCode)
             SharedFileSystemJobFailed

From 0e1a32057fd90fe8d344baaa3fac7962f2fbce51 Mon Sep 17 00:00:00 2001
From: Kris Ganjam <kganjam@uw.edu>
Date: Fri, 7 Jun 2019 08:41:02 -0700
Subject: [PATCH 05/28] Ensure workflowLogger always gets closed regardless of
 whether the log is temporary or not. (#5016)

---
 .../main/scala/cromwell/engine/workflow/WorkflowActor.scala    | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 0adbb164b6f..687781defe4 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -441,11 +441,10 @@ class WorkflowActor(workflowToStart: WorkflowToStart,
         workflowOptions.get(FinalWorkflowLogDir).toOption match {
           case Some(destinationDir) =>
             pathBuilders.map(pb => workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pb)))(ec)
-          case None if WorkflowLogger.isTemporary => workflowLogger.close(andDelete = true) match {
+          case None => workflowLogger.close(andDelete = WorkflowLogger.isTemporary) match {
             case Failure(f) => log.error(f, "Failed to delete workflow log")
             case _ =>
           }
-          case _ =>
         }
       }
       context stop self

From f0ae471331922b7302bffeeb2b01effe264973eb Mon Sep 17 00:00:00 2001
From: Alejandro Dubrovsky <Alex.Dubrovsky@agrf.org.au>
Date: Sat, 8 Jun 2019 04:41:33 +1000
Subject: [PATCH 06/28] Fix submit-docker to point to the right script in
 example configuration (#5015)

---
 cromwell.example.backends/cromwell.examples.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf
index 11bbf533cb3..1dc77f0f1cf 100644
--- a/cromwell.example.backends/cromwell.examples.conf
+++ b/cromwell.example.backends/cromwell.examples.conf
@@ -397,7 +397,7 @@ backend {
           ${"--user " + docker_user} \
           --entrypoint ${job_shell} \
           -v ${cwd}:${docker_cwd} \
-          ${docker} ${script}
+          ${docker} ${docker_script}
         """
 
         # Root directory where Cromwell writes job results.  This directory must be

From fe38ba71aa25839c505939edd31ba454f76dfa0e Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 7 Jun 2019 14:51:23 -0400
Subject: [PATCH 07/28] Added logging and cleanup to
 DockerComposeCromwellConfiguration

---
 .../scala/centaur/CromwellConfiguration.scala | 19 +++++++
 .../DockerComposeCromwellConfiguration.scala  | 53 ++++++++++---------
 .../centaur/JarCromwellConfiguration.scala    | 17 ++----
 .../docker-compose-horicromtal.yml            |  2 +-
 4 files changed, 51 insertions(+), 40 deletions(-)

diff --git a/centaur/src/main/scala/centaur/CromwellConfiguration.scala b/centaur/src/main/scala/centaur/CromwellConfiguration.scala
index 1b79a6573b0..f86268da4c3 100644
--- a/centaur/src/main/scala/centaur/CromwellConfiguration.scala
+++ b/centaur/src/main/scala/centaur/CromwellConfiguration.scala
@@ -1,5 +1,8 @@
 package centaur
 
+import java.lang.ProcessBuilder.Redirect
+import better.files.File
+
 trait CromwellProcess {
   def logFile: String
   def displayString: String
@@ -7,6 +10,22 @@ trait CromwellProcess {
   def stop(): Unit
   def isAlive: Boolean
   def cromwellConfiguration: CromwellConfiguration
+
+  protected def runProcess(command: Array[String]): Process = {
+    val processBuilder = new java.lang.ProcessBuilder()
+      .command(command: _*)
+      .redirectOutput(Redirect.appendTo(File(logFile).toJava))
+      .redirectErrorStream(true)
+    processBuilder.start()
+  }
+
+  protected def waitProcess(process: Process, destroy: Boolean = false): Unit = {
+    process.getOutputStream.flush()
+    if (destroy)
+      process.destroy()
+    process.waitFor()
+    ()
+  }
 }
 
 trait CromwellConfiguration {
diff --git a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala
index 12ef2bba2f3..182391491bd 100644
--- a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala
+++ b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala
@@ -1,11 +1,8 @@
 package centaur
 
-import java.lang.ProcessBuilder.Redirect
-
-import better.files.File
+import centaur.CromwellManager.ManagedCromwellPort
 import com.typesafe.config.Config
 
-
 object DockerComposeCromwellConfiguration {
   def apply(conf: Config): CromwellConfiguration = {
     val dockerTag = conf.getString("docker-tag")
@@ -20,37 +17,41 @@ object DockerComposeCromwellConfiguration {
 case class DockerComposeCromwellConfiguration(dockerTag: String, dockerComposeFile: String, conf: String, logFile: String) extends CromwellConfiguration {
   override def createProcess: CromwellProcess = {
     case class DockerComposeCromwellProcess(override val cromwellConfiguration: DockerComposeCromwellConfiguration) extends CromwellProcess {
-      private val startCommand = Array(
-        "/bin/bash",
-        "-c",
-        s"CROMWELL_TAG=$dockerTag CROMWELL_CONFIG=$conf docker-compose -f $dockerComposeFile up -d")
+
+      private def composeCommand(command: String): Array[String] = {
+        Array(
+          "/bin/bash",
+          "-c",
+          s"MANAGED_CROMWELL_PORT=$ManagedCromwellPort " +
+            s"CROMWELL_TAG=$dockerTag " +
+            s"CROMWELL_CONFIG=$conf " +
+            s"docker-compose -f $dockerComposeFile $command")
+      }
+
+      private val startCommand = composeCommand("up")
+      private val stopCommand = composeCommand("down -v")
+      private val rmCommand = composeCommand("rm -fsv")
 
       override def displayString: String = startCommand.mkString(" ")
 
+      private var process: Option[Process] = None
+
       override def start(): Unit = {
-        val processBuilder = new java.lang.ProcessBuilder()
-          .command(startCommand: _*)
-          .redirectOutput(Redirect.appendTo(File(logFile).toJava))
-          .redirectErrorStream(true)
-        processBuilder.start().waitFor()
-        ()
+        process = Option(runProcess(startCommand))
       }
 
       override def stop(): Unit = {
-        val command = Array(
-          "/bin/bash",
-          "-c",
-          s"docker-compose -f $dockerComposeFile down"
-        )
-        val processBuilder = new java.lang.ProcessBuilder()
-          .command(command: _*)
-          .redirectOutput(Redirect.appendTo(File(logFile).toJava))
-          .redirectErrorStream(true)
-        processBuilder.start().waitFor()
-        ()
+        waitProcess(runProcess(stopCommand))
+        waitProcess(runProcess(rmCommand))
+        process foreach {
+          waitProcess(_, destroy = true)
+        }
+        process = None
       }
 
-      override def isAlive: Boolean = true
+      override def isAlive: Boolean = process.exists {
+        _.isAlive
+      }
 
       override def logFile: String = cromwellConfiguration.logFile
     }
diff --git a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala
index caccb8ef46f..3069e555316 100644
--- a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala
+++ b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala
@@ -1,8 +1,5 @@
 package centaur
 
-import java.lang.ProcessBuilder.Redirect
-
-import better.files.File
 import centaur.CromwellManager.ManagedCromwellPort
 import com.typesafe.config.Config
 
@@ -32,20 +29,14 @@ case class JarCromwellConfiguration(jar: String, conf: String, logFile: String)
       override def displayString: String = command.mkString(" ")
 
       override def start(): Unit = {
-        val processBuilder = new java.lang.ProcessBuilder()
-          .command(command: _*)
-          .redirectOutput(Redirect.appendTo(File(logFile).toJava))
-          .redirectErrorStream(true)
-        process = Option(processBuilder.start())
+        process = Option(runProcess(command))
       }
 
       override def stop(): Unit = {
-        process foreach { p =>
-          p.getOutputStream.flush()
-          p.destroy()
-          p.waitFor()
+        process foreach {
+          waitProcess(_, destroy = true)
         }
-        ()
+        process = None
       }
 
       override def isAlive: Boolean = process.exists { _.isAlive }
diff --git a/scripts/docker-compose-mysql/docker-compose-horicromtal.yml b/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
index b554316de74..5faf0bc5bf8 100644
--- a/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
+++ b/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
@@ -39,7 +39,7 @@ services:
       - ${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}
     command: ["server"]
     environment:
-      - JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG} -Dwebservice.port=8008 -Dsystem.cromwell_id=frontend
+      - JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG} -Dwebservice.port=${MANAGED_CROMWELL_PORT-8008} -Dsystem.cromwell_id=frontend
       - CROMWELL_BUILD_RESOURCES_DIRECTORY=${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources
       - CROMWELL_BUILD_MYSQL_USERNAME=${CROMWELL_BUILD_MYSQL_USERNAME}
     healthcheck:

From 7f8d72947f9d8a501bb6aeae8b24678776ec3912 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Tue, 11 Jun 2019 17:50:17 -0400
Subject: [PATCH 08/28] Cromwell perf ad-hoc deploy, test, and destroy scripts
 BA-4980 (#5012)

---
 scripts/perf/clone_database/clone_database.sh | 21 +++++
 .../perf/delete_database/delete_database.sh   | 15 ++++
 .../perf/delete_instance/delete_instance.sh   | 15 ++++
 .../perf/deploy_instance/deploy_instance.sh   | 58 +++++++++++++
 .../perf/deploy_instance/run_on_instance.sh   | 51 +++++++++++
 scripts/perf/helper.inc.sh                    | 84 +++++++++++++++++--
 scripts/perf/startup_script.sh                |  2 +
 7 files changed, 241 insertions(+), 5 deletions(-)
 create mode 100755 scripts/perf/clone_database/clone_database.sh
 create mode 100755 scripts/perf/delete_database/delete_database.sh
 create mode 100755 scripts/perf/delete_instance/delete_instance.sh
 create mode 100755 scripts/perf/deploy_instance/deploy_instance.sh
 create mode 100644 scripts/perf/deploy_instance/run_on_instance.sh

diff --git a/scripts/perf/clone_database/clone_database.sh b/scripts/perf/clone_database/clone_database.sh
new file mode 100755
index 00000000000..99240996ff4
--- /dev/null
+++ b/scripts/perf/clone_database/clone_database.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -euo pipefail
+
+source scripts/perf/helper.inc.sh
+
+mkdir -p mnt
+
+# Read the DB password from vault:
+DB_PASS=$(read_path_from_vault_json "secret/dsp/cromwell/perf" '.data.db_pass')
+
+# Read the service account credentials from vault:
+read_service_account_from_vault
+
+# Clone the CloudSQL DB
+# Note: Cloning the same database in parallel doesn't work.
+# By doing it here we can run the jenkins jobs sequentially and ensure the database is cloned once at a time as well
+SQL_OPERATION=$(gcloud_run_as_service_account "perf_sql_create_gcloud_${BUILD_NUMBER}" "gcloud --project broad-dsde-cromwell-perf sql instances clone --async ${CLOUD_SQL_INSTANCE_TO_CLONE} ${CLOUD_SQL_INSTANCE_NEW_NAME} --format='value(name)'" )
+
+gcloud_run_as_service_account "perf_sql_await_gcloud_${BUILD_NUMBER}" "gcloud beta sql operations wait --timeout=900 --project broad-dsde-cromwell-perf ${SQL_OPERATION}"
+
+gcloud_run_as_service_account "perf_sql_user_gcloud_${BUILD_NUMBER}" "gcloud --project broad-dsde-cromwell-perf sql users create cromwell --instance=${CLOUD_SQL_INSTANCE_NEW_NAME} --password=${DB_PASS}"
diff --git a/scripts/perf/delete_database/delete_database.sh b/scripts/perf/delete_database/delete_database.sh
new file mode 100755
index 00000000000..a393cac5eb2
--- /dev/null
+++ b/scripts/perf/delete_database/delete_database.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euo pipefail
+
+source scripts/perf/helper.inc.sh
+
+mkdir -p mnt
+
+# Read the service account credentials from vault:
+read_service_account_from_vault
+
+gcloud_run_as_service_account "perf_sql_delete_gcloud_${BUILD_NUMBER}" " \
+  gcloud \
+      --verbosity info \
+      --project broad-dsde-cromwell-perf \
+      sql instances delete ${CLOUD_SQL_INSTANCE} -q"
diff --git a/scripts/perf/delete_instance/delete_instance.sh b/scripts/perf/delete_instance/delete_instance.sh
new file mode 100755
index 00000000000..29d40ce7fb6
--- /dev/null
+++ b/scripts/perf/delete_instance/delete_instance.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -euo pipefail
+
+source scripts/perf/helper.inc.sh
+
+mkdir -p mnt
+
+# Read the service account credentials from vault:
+read_service_account_from_vault
+
+gcloud_run_as_service_account "perf_delete_gcloud_instance_${BUILD_NUMBER}" " \
+  gcloud \
+    --verbosity info \
+    --project broad-dsde-cromwell-perf \
+    compute instances delete ${CROMWELL_INSTANCE_NAME} --zone=us-central1-c -q"
diff --git a/scripts/perf/deploy_instance/deploy_instance.sh b/scripts/perf/deploy_instance/deploy_instance.sh
new file mode 100755
index 00000000000..25016d3a08d
--- /dev/null
+++ b/scripts/perf/deploy_instance/deploy_instance.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+
+source scripts/perf/helper.inc.sh
+
+GCS_BUCKET=gs://cromwell-perf-test/
+
+DB_PASS=$(read_path_from_vault_json "secret/dsp/cromwell/perf" '.data.db_pass')
+
+read_service_account_from_vault
+	
+function join() { local IFS=","; echo "$*"; }
+
+metadata=(
+  "BUILD_NUMBER=$BUILD_NUMBER"
+  "BUILD_TAG=$BUILD_TAG"
+  "CLEAN_UP=false"
+  "CLOUD_SQL_INSTANCE=$CLOUD_SQL_INSTANCE"
+  "CROMWELL_DB_USER=cromwell"
+  "CROMWELL_DB_PASS=$DB_PASS"
+  "CROMWELL_DOCKER_IMAGE=$CROMWELL_DOCKER_IMAGE"
+  "CROMWELL_PROJECT=broad-dsde-cromwell-perf"
+  "CROMWELL_BUCKET=$GCS_BUCKET"
+  "CROMWELL_STATSD_HOST=10.128.0.4"
+  "CROMWELL_STATSD_PORT=8125"
+  "CROMWELL_PERF_SCRIPTS_BRANCH=${REPO_BRANCH}"
+)
+
+cp scripts/perf/deploy_instance/run_on_instance.sh mnt/
+
+gcloud_run_as_service_account "perf_deploy_instance_${BUILD_NUMBER}" \
+  "gcloud \
+    --verbosity info \
+    --project broad-dsde-cromwell-perf \
+    compute \
+    instances \
+    create ${CROMWELL_INSTANCE_NAME} \
+    --zone us-central1-c \
+    --source-instance-template $INSTANCE_TEMPLATE \
+    --metadata-from-file startup-script=$DOCKER_ETC_PATH/run_on_instance.sh \
+    --metadata \
+        $(join ${metadata[@]})" | tee dockerResult.txt
+
+
+typeset CROMWELL_UNDER_TEST=$(cat dockerResult.txt | tail -n1 | awk '{print $5}' )
+
+if test -z "$CROMWELL_UNDER_TEST"
+then
+  echo "\$CROMWELL_UNDER_TEST is empty"
+  exit 1
+else
+  echo "Determined that CROMWELL_UNDER_TEST=${CROMWELL_UNDER_TEST}"
+fi
+
+custom_wait_for_cromwell
+
+mkdir -p output
+echo "CROMWELL_UNDER_TEST=${CROMWELL_UNDER_TEST}" > output/cromwell.properties
diff --git a/scripts/perf/deploy_instance/run_on_instance.sh b/scripts/perf/deploy_instance/run_on_instance.sh
new file mode 100644
index 00000000000..fab0bd5b839
--- /dev/null
+++ b/scripts/perf/deploy_instance/run_on_instance.sh
@@ -0,0 +1,51 @@
+#! /bin/bash
+
+set -x
+
+### /!\ This script assumes docker and docker compose are already installed on the host
+
+# Utility function to extract values from instance metadata
+extract_metadata() {
+  curl "http://metadata.google.internal/computeMetadata/v1/instance/attributes/$1" -H "Metadata-Flavor: Google"
+}
+
+# Exports an env var and also adds it to the root bashrc. This way if there is a need to ssh onto the machine
+# for debugging one will have the env variables already set when using root
+addVar() {
+  export $1
+  echo "export $1" >> /root/.bashrc
+}
+
+# Make sure ip forwarding is enabled by default so that docker doesn't lose connectivity
+echo "net.ipv4.ip_forward = 1" > /etc/sysctl.conf
+
+# Set up env variables
+addVar CROMWELL_BRANCH=$(extract_metadata CROMWELL_PERF_SCRIPTS_BRANCH)
+addVar CLOUD_SQL_DB_USER=$(extract_metadata CROMWELL_DB_USER)
+addVar CLOUD_SQL_DB_PASSWORD=$(extract_metadata CROMWELL_DB_PASS)
+addVar CLOUD_SQL_INSTANCE=$(extract_metadata CLOUD_SQL_INSTANCE)
+addVar CROMWELL_DOCKER_IMAGE=$(extract_metadata CROMWELL_DOCKER_IMAGE)
+addVar CROMWELL_PROJECT=$(extract_metadata CROMWELL_PROJECT)
+addVar CROMWELL_EXECUTION_ROOT=$(extract_metadata CROMWELL_BUCKET)
+addVar CROMWELL_STATSD_HOST=$(extract_metadata CROMWELL_STATSD_HOST)
+addVar CROMWELL_STATSD_PORT=$(extract_metadata CROMWELL_STATSD_PORT)
+addVar BUILD_ID=$(extract_metadata BUILD_TAG)
+
+# Use the instance name as statsd prefix to avoid metrics collisions
+addVar CROMWELL_STATSD_PREFIX=${BUILD_ID}
+addVar REPORT_BUCKET=cromwell-perf-test-reporting
+
+addVar CROMWELL_ROOT=/app
+addVar PERF_ROOT=${CROMWELL_ROOT}/scripts/perf
+addVar TEST_WORKFLOW_ROOT=${PERF_ROOT}/test_cases
+
+# Clone cromwell to get the perf scripts. Use https to avoid ssh fingerprint prompt when the script runs
+git clone -b ${CROMWELL_BRANCH} --depth 1 --single-branch https://github.com/broadinstitute/cromwell.git ${CROMWELL_ROOT}
+
+source ${PERF_ROOT}/helper.inc.sh
+
+addVar CROMWELL_CONF_DIR=${PERF_ROOT}/vm_scripts/cromwell
+
+# Start cromwell and cloud sql proxy
+prepare_statsd_proxy
+docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml up -d
diff --git a/scripts/perf/helper.inc.sh b/scripts/perf/helper.inc.sh
index 35a6b31e2ca..cd99337ab2a 100644
--- a/scripts/perf/helper.inc.sh
+++ b/scripts/perf/helper.inc.sh
@@ -1,19 +1,28 @@
 #!/usr/bin/env bash
 
-set -x
+# Set some global constants:
+typeset DOCKER_ETC_PATH="/usr/share/etc"
+
+if test -f "/etc/vault-token-dsde"
+then
+  typeset VAULT_TOKEN=$(cat /etc/vault-token-dsde)
+fi
+
 
 wait_for_cromwell() {
   git clone https://github.com/vishnubob/wait-for-it.git /wait-for-it
   chmod u+x /wait-for-it/wait-for-it.sh
   # Give 5 minutes to cromwell to be online
-  /wait-for-it/wait-for-it.sh localhost:8000 -t 300
+
+  echo "[$(date)] Waiting for http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version to appear..."
+  /wait-for-it/wait-for-it.sh ${CROMWELL_UNDER_TEST}:8000 -t 300
   READY=$?
   if [ ${READY} -eq 0 ]
   then
       # Just wait a bit longer - no rush, this is a chill VM - this is because cromwell responds to requests before being really ready...
       sleep 30
     
-      CROMWELL_VERSION=$(curl -X GET "http://localhost:8000/engine/v1/version" -H "accept: application/json" | jq -r '.cromwell')
+      CROMWELL_VERSION=$(curl -X GET "http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version" -H "accept: application/json" | jq -r '.cromwell')
       if [ -z ${CROMWELL_VERSION} ]
       then
         echo "Cromwell was up but failed to return its version, so something went wrong, shutting down"
@@ -26,6 +35,42 @@ wait_for_cromwell() {
   fi
 }
 
+# Waits for Cromwell. Works even from *outside* a google instance
+custom_wait_for_cromwell() {
+
+  set +e
+
+  RESULT=1
+  ATTEMPTS=0
+  MAX_ATTEMPTS=20
+
+  while [ "${ATTEMPTS}" -le "${MAX_ATTEMPTS}" -a "${RESULT}" -gt "0" ]
+  do
+    echo "[$(date)] Waiting for Cromwell (http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version) to come up (tried ${ATTEMPTS} times so far)"
+    sleep 30
+    ATTEMPTS=$((ATTEMPTS + 1))
+
+    CROMWELL_VERSION_JSON=$(curl -X GET "http://${CROMWELL_UNDER_TEST}:8000/engine/v1/version" -H "accept: application/json")
+    RESULT=$?
+
+    CROMWELL_VERSION=$(echo "${CROMWELL_VERSION_JSON}" | jq -r '.cromwell')
+
+    if [ "${RESULT}" -eq "0" -a -z "${CROMWELL_VERSION}" ]
+    then
+      echo "Cromwell was up but failed to return its version, maybe something went wrong? Continuing waiting..."
+      RESULT=1
+    fi
+  done
+
+  if [ "${RESULT}" -gt "0" ]
+  then
+    echo "Cromwell never came up after ${ATTEMPTS} attempts"
+    exit 1
+  else
+    echo "Cromwell came up after ${ATTEMPTS} attempts"
+  fi
+}
+
 export_logs() {
     export REPORT_URL="gs://${GCS_REPORT_BUCKET}/${GCS_REPORT_PATH}"
 
@@ -58,10 +103,14 @@ clean_up() {
     if [ "${CLEAN_UP}" = true ]
     then
         gcloud sql instances delete ${CLOUD_SQL_INSTANCE}
-        gcloud compute instances delete $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") --zone=us-central1-c -q
+        clean_up_instance
     fi
 }
 
+clean_up_instance() {
+    gcloud compute instances delete $(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google") --zone=us-central1-c -q
+}
+
 run_test() {
     cd ${CROMWELL_ROOT}
 
@@ -70,7 +119,11 @@ run_test() {
         export GCS_REPORT_PATH="${TEST_CASE_DIRECTORY}/${CROMWELL_VERSION}/${BUILD_ID}"
     fi
 
-    sbt -Dconfig.file=${CENTAUR_CONF_DIR}/centaur.conf "centaur/it:testOnly centaur.ExternalTestCaseSpec" | tee centaur.log
+    sbt \
+      -Dconfig.file=${CENTAUR_CONF_DIR}/centaur.conf \
+      -Dcentaur.cromwell.mode="url" \
+      -Dcentaur.cromwell.url="http://${CROMWELL_UNDER_TEST}:8000" \
+      "centaur/it:testOnly centaur.ExternalTestCaseSpec" | tee centaur.log
     export TEST_RC=$?
 }
 
@@ -79,3 +132,24 @@ shutdown() {
     docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml down
     clean_up
 }
+
+read_path_from_vault_json() {
+  local path=$1
+  local field=$2
+
+  docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN broadinstitute/dsde-toolbox vault read -format=json "${path}" | jq --exit-status -r "${field}"
+}
+
+
+read_service_account_from_vault() {
+  mkdir -p mnt
+  read_path_from_vault_json "secret/dsp/cromwell/perf/service-account-deployer" '.data.service_account' > mnt/sa.json
+}
+
+gcloud_run_as_service_account() {
+  local name="$1"
+  local command="$2"
+  docker run --name $name -v "$(pwd)"/mnt:${DOCKER_ETC_PATH} --rm google/cloud-sdk:slim /bin/bash -c "\
+    gcloud auth activate-service-account --key-file ${DOCKER_ETC_PATH}/sa.json 2> /dev/null &&\
+    ${command}"
+}
diff --git a/scripts/perf/startup_script.sh b/scripts/perf/startup_script.sh
index 484b1093e9c..8194811fa30 100644
--- a/scripts/perf/startup_script.sh
+++ b/scripts/perf/startup_script.sh
@@ -43,6 +43,8 @@ addVar CROMWELL_ROOT=/app
 addVar PERF_ROOT=${CROMWELL_ROOT}/scripts/perf
 addVar TEST_WORKFLOW_ROOT=${PERF_ROOT}/test_cases
 
+addVar CROMWELL_UNDER_TEST="localhost"
+
 # Clone cromwell to get the perf scripts. Use https to avoid ssh fingerprint prompt when the script runs
 git clone -b ${CROMWELL_BRANCH} --depth 1 --single-branch https://github.com/broadinstitute/cromwell.git ${CROMWELL_ROOT}
 

From 521cee9152519f5952a1a4021674047f4a592d93 Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Wed, 12 Jun 2019 08:34:40 -0400
Subject: [PATCH 09/28] Stackdriver Instrumentation BA-4788 (#5010)

---
 CHANGELOG.md                                  |  13 ++
 core/src/main/resources/reference.conf        |   6 +
 .../cromwell.examples.conf                    |  16 +-
 docs/developers/Instrumentation.md            |  87 +++++++--
 project/Dependencies.scala                    |  25 ++-
 .../impl/stackdriver/StackdriverConfig.scala  |  66 +++++++
 ...ackdriverInstrumentationServiceActor.scala | 181 ++++++++++++++++++
 .../impl/statsd/StatsDConfig.scala            |  10 +-
 .../stackdriver/StackdriverConfigSpec.scala   | 134 +++++++++++++
 ...riverInstrumentationServiceActorSpec.scala | 111 +++++++++++
 ...ackdriverInstrumentationServiceActor.scala |  28 +++
 .../impl/statsd/StatsDConfigSpec.scala        |   5 +-
 ...rumentationServiceActorBenchmarkSpec.scala |   3 +-
 ...tatsDInstrumentationServiceActorSpec.scala |   3 +-
 14 files changed, 653 insertions(+), 35 deletions(-)
 create mode 100644 services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfig.scala
 create mode 100644 services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
 create mode 100644 services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfigSpec.scala
 create mode 100644 services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActorSpec.scala
 create mode 100644 services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/TestStackdriverInstrumentationServiceActor.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8895f64ab67..73ea7d40e65 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Cromwell Change Log
 
+## 43 Release Notes
+
+### Stackdriver Instrumentation
+
+Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). 
+Learn more on how to configure [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
+
+### StatsD Instrumentation Config change
+
+There is a small change in StatsD's configuration path. Originally, the path to the config was `services.Instrumentation.config.statsd`
+which now has been updated to `services.Instrumentation.config`. More info on its configuration can be found
+[here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
+
 ## 42 Release Notes
 
 ### Womtool endpoint
diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
index 0eabd4eb146..c729c46af6e 100644
--- a/core/src/main/resources/reference.conf
+++ b/core/src/main/resources/reference.conf
@@ -497,6 +497,12 @@ services {
   Instrumentation {
     # Default noop service - instrumentation metrics are ignored
     class = "cromwell.services.instrumentation.impl.noop.NoopInstrumentationServiceActor"
+
+    # StatsD instrumentation service actor
+    # class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
+
+    # Stackdriver instrumentation service actor
+    # class = "cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor"
   }
   HealthMonitor {
     class = "cromwell.services.healthmonitor.impl.HealthMonitorServiceActor"
diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf
index 1dc77f0f1cf..4318bce04bd 100644
--- a/cromwell.example.backends/cromwell.examples.conf
+++ b/cromwell.example.backends/cromwell.examples.conf
@@ -495,12 +495,26 @@ services {
   Instrumentation {
     # StatsD - Send metrics to a StatsD server over UDP
     # class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
-    # config.statsd {
+    # config {
     #   hostname = "localhost"
     #   port = 8125
     #   prefix = "" # can be used to prefix all metrics with an api key for example
     #   flush-rate = 1 second # rate at which aggregated metrics will be sent to statsd
     # }
+
+    # Stackdriver - Send metrics to Google's monitoring API
+    # class = "cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor"
+    # config {
+    #   # auth scheme can be `application_default` or `service_account`
+    #   auth = "service-account"
+    #   google-project = "my-project"
+    #   # rate at which aggregated metrics will be sent to Stackdriver API, must be 1 minute or more.
+    #   flush-rate = 1 minute
+    #   # below 3 keys are attached as labels to each metric. `cromwell-perf-test-case` is specifically meant for perf env.
+    #   cromwell-instance-identifier = "cromwell-101"
+    #   cromwell-instance-role = "role"
+    #   cromwell-perf-test-case = "perf-test-1"
+    # }
   }
   HealthMonitor {
     config {
diff --git a/docs/developers/Instrumentation.md b/docs/developers/Instrumentation.md
index c5a250542e1..ef3f72a76a5 100644
--- a/docs/developers/Instrumentation.md
+++ b/docs/developers/Instrumentation.md
@@ -1,28 +1,28 @@
-**Overview**
+# Overview
 
 Cromwell's instrumentation support can be useful to collect utilization data in long-running, high-volume
-production environments. While this instrumentation support can be used in smaller environments it will still require setting up a
-[StatsD](https://github.com/etsy/statsd) server outside of Cromwell and it's possible not enough data would be produced to be useful.  
+production environments. The default implementation of this ignores these metrics, but Cromwell includes alternate implementations that can forward metrics to a
+specific server.
 
-**StatsD**
+### StatsD
 
-Cromwell collects metrics while running and sends them to an internal service. The default implementation of this service
-ignores these metrics, but Cromwell includes an alternate implementation that can forward metrics to a
-[StatsD](https://github.com/etsy/statsd) server.
-To specify this implementation in your configuration file:
+While this instrumentation support can be used in smaller environments it will still require setting up a
+[StatsD](https://github.com/etsy/statsd) server outside of Cromwell and it's possible not enough data would be produced to be useful. 
+Cromwell collects metrics while running and sends them to an internal service. 
 
-```hocon
-services.Instrumentation.class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
-```
 Make sure to configure your StatsD service:
 
 ```hocon
-services.Instrumentation.config.statsd {
-    hostname = "localhost" # Replace with your host
-    port = 8125 # Replace with your port
-    # prefix = "my_prefix" # All metrics will be prefixed by this value if present.
-    flush-rate = 1 second # Rate at which metrics are sent to the StatsD server
-  }
+services.Instrumentation {
+	class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
+
+	config {
+	    hostname = "localhost" # Replace with your host
+	    port = 8125 # Replace with your port
+	    # prefix = "my_prefix" # All metrics will be prefixed by this value if present.
+	    flush-rate = 1 second # Rate at which metrics are sent to the StatsD server
+  	}
+}
 ```
 
 There is also an additional configuration value that can be set: 
@@ -39,7 +39,7 @@ system.cromwell_id = "cromwell-instance-1"
 will prepend all the metrics with path `cromwell.cromwell-instance-1...` for that instance.
 
 
-**Metrics**
+##### Metrics
 
 The current StatsD implementation uses metrics-statsd to report instrumentation values.
 metrics-statsd reports all metrics with a gauge type.
@@ -51,3 +51,54 @@ These are the current high level categories:
 * `job`
 * `workflow`
 * `io`
+
+
+### Stackdriver
+
+Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). To use the Stackdriver instrumentation
+specify this in your config:
+```hocon
+services.Instrumentation {
+    class = "cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor"
+
+    config {
+        # auth scheme can be `application_default` or `service_account`
+        auth = "service-account"
+        google-project = "my-project"
+        # rate at which aggregated metrics will be sent to Stackdriver. It needs to be equal or greater than 1 minute.
+        # Google's Stackdriver API needs each metric to be sent not more than once per minute.
+        flush-rate = 1 minute
+        # below 3 keys are attached as labels to each metric. `cromwell-perf-test-case` is specifically meant for perf env.
+        cromwell-instance-role = "role"
+        cromwell-perf-test-case = "perf-test-1"
+    }
+ }
+```
+The 2 label keys are optional. If specified, each metric will have label(s) added in the form of a (key, value) pair.
+So for example, if `cromwell-instance-role = "backend"` is mentioned in config, each metric data point sent to Stackdriver
+will have a label (cromwell_instance_role, backend) added to it.
+
+There is another optional label that can be added to each metric. `cromwell_id` represents the identifier for different Cromwell instances.
+```hocon
+# Unique Cromwell instance identifier
+system.cromwell_id = "cromwell-instance-1"
+```
+
+
+##### Metric type and Label keys naming convention
+More details on the this can be found [here](https://cloud.google.com/monitoring/api/v3/metrics-details#metric-kinds).
+
+You must adhere to the following spelling rules for metric type names:
+- You can use upper and lower-case letters, digits, and underscores (_) in the names.
+- You can use periods (.) in the domain part of the names.
+- You can use forward slashes (/) to separate path elements.
+- You can start each path element with a letter or digit.
+- The maximum length of a metric type name is 200 characters.
+
+You must adhere to the following spelling rules for metric label names:
+- You can use upper and lower-case letters, digits, underscores (_) in the names.
+- You can start names with a letter or digit.
+- The maximum length of a metric label name is 100 characters.
+
+
+
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 4ceea23f4a8..54c0e1241f7 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -29,6 +29,7 @@ object Dependencies {
   private val googleApiClientV = "1.28.0"
   private val googleCloudCoreV = "1.61.0"
   private val googleCloudKmsV = "v1-rev63-1.25.0"
+  private val googleCloudMonitoringV = "1.70.0"
   private val googleCloudNioV = "0.61.0-alpha"
   private val googleGenomicsServicesV1ApiV = "v1alpha2-rev495-1.23.0"
   private val googleGenomicsServicesV2ApiV = "v2alpha1-rev31-1.25.0"
@@ -36,7 +37,7 @@ object Dependencies {
   private val googleHttpClientV = "1.29.1"
   private val googleOauth2V = "0.13.0"
   private val googleCloudResourceManagerV = "0.87.0-alpha"
-  private val grpcV = "1.18.0"
+  private val grpcV = "1.20.0"
   private val guavaV = "27.0.1-jre"
   private val heterodonV = "1.0.0-beta3"
   private val hsqldbV = "2.4.1"
@@ -345,6 +346,10 @@ object Dependencies {
     "com.readytalk" % "metrics3-statsd" % metrics3StatsdV
   )
 
+  val stackdriverDependencies = List(
+    "com.google.cloud" % "google-cloud-monitoring" % googleCloudMonitoringV
+  )
+
   val gcsFileSystemDependencies = akkaHttpDependencies
 
   val httpFileSystemDependencies = akkaHttpDependencies
@@ -434,7 +439,7 @@ object Dependencies {
   ) ++ akkaStreamDependencies ++ configDependencies ++ catsDependencies ++ circeDependencies ++
     googleApiClientDependencies ++ statsDDependencies ++ betterFilesDependencies ++
     // TODO: We're not using the "F" in slf4j. Core only supports logback, specifically the WorkflowLogger.
-    slf4jBindingDependencies
+    slf4jBindingDependencies ++ stackdriverDependencies
 
   val databaseMigrationDependencies = liquibaseDependencies ++ dbmsDependencies
 
@@ -601,6 +606,19 @@ object Dependencies {
     "util",
   ).map(m => "org.eclipse.rdf4j" % s"rdf4j-$m" % rdf4jV)
 
+  // Some libraries are importing older version of these dependencies, causing conflicts. Hence the need to override them.
+  val grpcDependencyOverrides = List(
+    "alts",
+    "auth",
+    "context",
+    "core",
+    "grpclb",
+    "netty-shaded",
+    "protobuf-lite",
+    "protobuf",
+    "stub",
+  ).map(m => "io.grpc" % s"grpc-$m" % grpcV)
+
   /*
   If we use a version in one of our projects, that's the one we want all the libraries to use
   ...plus other groups of transitive dependencies shared across multiple projects
@@ -609,5 +627,6 @@ object Dependencies {
     allProjectDependencies ++
       googleHttpClientDependencies ++
       nettyDependencyOverrides ++
-      rdf4jDependencyOverrides
+      rdf4jDependencyOverrides ++
+      grpcDependencyOverrides
 }
diff --git a/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfig.scala b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfig.scala
new file mode 100644
index 00000000000..2131eefbeaf
--- /dev/null
+++ b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfig.scala
@@ -0,0 +1,66 @@
+package cromwell.services.instrumentation.impl.stackdriver
+
+import cats.data.Validated._
+import cats.syntax.apply._
+import cats.syntax.validated._
+import com.typesafe.config.Config
+import common.exception.AggregatedMessageException
+import common.validation.ErrorOr.ErrorOr
+import common.validation.Validation.validate
+import cromwell.cloudsupport.gcp.GoogleConfiguration
+import cromwell.cloudsupport.gcp.auth._
+import net.ceedubs.ficus.Ficus._
+
+import scala.concurrent.duration._
+
+case class StackdriverConfig(googleProject: String,
+                             auth: GoogleAuthMode,
+                             flushRate: FiniteDuration,
+                             cromwellInstanceIdentifier: Option[String],
+                             cromwellInstanceRole: Option[String],
+                             cromwellPerfTestCase: Option[String])
+
+object StackdriverConfig {
+  val CromwellInstanceIdentifier = "cromwell-instance-identifier"
+  val CromwellInstanceRole = "cromwell-instance-role"
+  val CromwellPerfTest = "cromwell-perf-test-case"
+
+  private def validateFlushRate(rateFunc: => FiniteDuration): ErrorOr[FiniteDuration] = {
+    validate[FiniteDuration](rateFunc) match {
+      case Valid(duration) => duration match {
+        case _ if duration < 1.minute => (s"`flush-rate` must be 1 minute or longer, specified rate is `$duration`. " +
+          s"Google's Stackdriver API needs each metric to be sent not more than once per minute.").invalidNel
+        case _ => duration.validNel
+      }
+      case Invalid(e) => e.invalid
+    }
+  }
+
+
+  private def validateAuth(authSchemeFunc: => String, googleConfiguration: GoogleConfiguration): ErrorOr[GoogleAuthMode] = {
+    validate[String](authSchemeFunc) match {
+      case Valid(schemeString) => googleConfiguration.auth(schemeString) match {
+        case Valid(auth @ (_:ApplicationDefaultMode | _:ServiceAccountMode)) => auth.valid
+        case Valid(_) => s"`auth` scheme: $schemeString is not allowed for Stackdriver instrumentation. Only `application_default` and `service_account` modes are valid.".invalidNel
+        case Invalid(error) => s"`auth` scheme is invalid. Errors: $error".invalidNel
+      }
+      case Invalid(e) => e.invalid
+    }
+  }
+
+
+  def apply(serviceConfig: Config, globalConfig: Config): StackdriverConfig = {
+    val googleConfiguration: GoogleConfiguration = GoogleConfiguration(globalConfig)
+    val cromwellInstanceId: ErrorOr[Option[String]] = globalConfig.getAs[String]("system.cromwell_id").validNel
+
+    val googleProject: ErrorOr[String] = validate[String] { serviceConfig.as[String]("google-project") }
+    val authScheme: ErrorOr[GoogleAuthMode] = validateAuth(serviceConfig.as[String]("auth"), googleConfiguration)
+    val flushRate: ErrorOr[FiniteDuration] = validateFlushRate(serviceConfig.as[FiniteDuration]("flush-rate"))
+    val cromwellInstanceRole: ErrorOr[Option[String]] = serviceConfig.getAs[String](CromwellInstanceRole).validNel
+    val cromwellPerfTestCase: ErrorOr[Option[String]] = serviceConfig.getAs[String](CromwellPerfTest).validNel
+
+    (googleProject, authScheme, flushRate, cromwellInstanceId, cromwellInstanceRole, cromwellPerfTestCase).mapN({ (p, a, f, i, r, t) =>
+      new StackdriverConfig(p, a, f, i, r, t)
+    }).valueOr(errors => throw AggregatedMessageException("Stackdriver instrumentation config is invalid. Error(s)", errors.toList))
+  }
+}
diff --git a/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
new file mode 100644
index 00000000000..1dd5462f62d
--- /dev/null
+++ b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
@@ -0,0 +1,181 @@
+package cromwell.services.instrumentation.impl.stackdriver
+
+import java.util
+
+import akka.actor.{Actor, ActorRef, Props}
+import com.google.api.gax.core.FixedCredentialsProvider
+import com.google.api.{Metric, MetricDescriptor, MonitoredResource}
+import com.google.cloud.monitoring.v3.{MetricServiceClient, MetricServiceSettings}
+import com.google.monitoring.v3._
+import com.google.protobuf.util.Timestamps
+import com.typesafe.config.Config
+import com.typesafe.scalalogging.StrictLogging
+import cromwell.services.instrumentation.InstrumentationService.InstrumentationServiceMessage
+import cromwell.services.instrumentation._
+import cromwell.services.instrumentation.impl.stackdriver.StackdriverConfig._
+import cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor._
+import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+
+import scala.collection.JavaConverters._
+import scala.concurrent.duration._
+import scala.util.Try
+
+
+class StackdriverInstrumentationServiceActor(serviceConfig: Config, globalConfig: Config, serviceRegistryActor: ActorRef) extends Actor with StrictLogging {
+  implicit lazy val executionContext = context.dispatcher
+
+  val stackdriverConfig = StackdriverConfig(serviceConfig, globalConfig)
+
+  lazy val projectName: ProjectName = ProjectName.of(stackdriverConfig.googleProject)
+  val credentials = stackdriverConfig.auth.credentials(List(MonitoringScope))
+  lazy val metricLabelsMap = generateMetricLabels()
+
+  var metricsMap = Map.empty[StackdriverMetric, Vector[Double]]
+
+  // Instantiates a client
+  val metricServiceSettings = MetricServiceSettings.newBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)).build
+  final val metricServiceClient = MetricServiceClient.create(metricServiceSettings)
+
+  // Prepares the monitored resource descriptor
+  lazy val resourceLabels = Map[String, String](("project_id", stackdriverConfig.googleProject))
+  lazy val monitoredResource = MonitoredResource.newBuilder.setType("global").putAllLabels(resourceLabels.asJava).build
+
+  // Give the actor time to warm up, then start sending the metrics to Stackdriver at an interval
+  context.system.scheduler.schedule(InitialDelay, stackdriverConfig.flushRate, self, SendStackdriverMetricCommand)
+
+
+  override def receive = {
+    case SendStackdriverMetricCommand => sendMetricData()
+    case InstrumentationServiceMessage(cromwellMetric) => cromwellMetric match {
+      case CromwellTiming(bucket, value, _) => updateMetricMap(bucket, value.toMillis.toDouble, StackdriverGauge)
+      case CromwellGauge(bucket, value) => updateMetricMap(bucket, value.toDouble, StackdriverGauge)
+      case CromwellCount(bucket, value, _) => updateMetricMap(bucket, value.toDouble, StackdriverCumulative)
+      case CromwellIncrement(bucket) => updateMetricMap(bucket, metricValue = 1D, metricKind = StackdriverCumulative)
+    }
+    case ShutdownCommand =>
+      // flush out metrics (if any) before shut down
+      sendMetricData()
+      metricServiceClient.close()
+      context stop self
+  }
+
+
+  private def generateMetricLabels(): Map[String, String] = {
+    def labelFromConfig(op: StackdriverConfig => Option[String], key: String): Option[(String, String)] = {
+      op(stackdriverConfig).map(v => (key.replace("-", "_"), v))
+    }
+
+    labelFromConfig(_.cromwellInstanceIdentifier, CromwellInstanceIdentifier).toMap ++
+      labelFromConfig(_.cromwellInstanceRole, CromwellInstanceRole) ++
+      labelFromConfig(_.cromwellPerfTestCase, CromwellPerfTest)
+  }
+
+
+  private def updateMetricMap(bucket: CromwellBucket, metricValue: Double, metricKind: StackdriverMetricKind): Unit = {
+    val metricObj = StackdriverMetric(bucket.toStackdriverString, metricKind)
+
+    if (metricsMap.contains(metricObj)) {
+      val valueVector: Vector[Double] = metricsMap(metricObj) :+ metricValue
+      metricsMap += metricObj -> valueVector
+    }
+    else metricsMap += metricObj -> Vector(metricValue)
+  }
+
+
+  private def sendMetricData(): Unit = {
+    metricsMap.foreach { case (key, value: Vector[Double]) =>
+      val dataPointVectorSum = value.sum
+      val dataPoint = key.kind match {
+        case StackdriverGauge => dataPointVectorSum / value.length
+        case StackdriverCumulative => dataPointVectorSum
+      }
+
+      writeMetrics(key, dataPoint) recover {
+        case e => logger.error(s"Failed to send metrics to Stackdriver API for metric ${key.name} with value $dataPoint.", e)
+      }
+    }
+
+    metricsMap = Map.empty[StackdriverMetric, Vector[Double]]
+  }
+
+
+  private def writeMetrics(metricObj: StackdriverMetric, value: Double): Try[Unit] = {
+    def timeInterval(metricKind: StackdriverMetricKind): TimeInterval = {
+      metricKind match {
+        case StackdriverGauge => TimeInterval.newBuilder.setEndTime(Timestamps.fromMillis(System.currentTimeMillis)).build
+        case StackdriverCumulative => TimeInterval.newBuilder.setStartTime(ActorCreationTime).setEndTime(Timestamps.fromMillis(System.currentTimeMillis)).build
+      }
+    }
+
+    def createTimeSeries(metricKind: StackdriverMetricKind, metric: Metric, resource: MonitoredResource, dataPointList: util.List[Point]): TimeSeries = {
+      metricKind match {
+        case StackdriverGauge => TimeSeries.newBuilder.setMetric(metric).setResource(resource).addAllPoints(dataPointList).build
+        case StackdriverCumulative => TimeSeries.newBuilder.setMetric(metric).setResource(resource).setMetricKind(MetricDescriptor.MetricKind.CUMULATIVE).addAllPoints(dataPointList).build
+      }
+    }
+
+    // Prepares an individual data point
+    val interval = timeInterval(metricObj.kind)
+    val pointValue = TypedValue.newBuilder().setDoubleValue(value).build()
+    val dataPoint: Point = Point.newBuilder.setInterval(interval).setValue(pointValue).build
+    val dataPointList: List[Point] = List[Point](dataPoint)
+
+    // Prepares the metric descriptor
+    val metric: Metric = Metric.newBuilder.setType(s"$CustomMetricDomain/${metricObj.name}").putAllLabels(metricLabelsMap.asJava).build
+
+    // Prepares the time series request
+    val timeSeries = createTimeSeries(metricObj.kind, metric, monitoredResource, dataPointList.asJava)
+    val timeSeriesList = List[TimeSeries](timeSeries)
+
+    val timeSeriesRequest = CreateTimeSeriesRequest.newBuilder.setName(projectName.toString).addAllTimeSeries(timeSeriesList.asJava).build
+
+    // Writes time series data
+    Try(sendTimeSeriesToStackdriver(timeSeriesRequest))
+  }
+
+
+  // This single line of code is a separate function to help with StackdriverInstrumentationActor tests
+  def sendTimeSeriesToStackdriver(timeSeriesRequest: CreateTimeSeriesRequest): Unit = {
+    metricServiceClient.createTimeSeries(timeSeriesRequest)
+  }
+}
+
+
+object StackdriverInstrumentationServiceActor {
+  val CromwellMetricPrefix = List("cromwell")
+
+  val ActorCreationTime = Timestamps.fromMillis(System.currentTimeMillis())
+
+  // Custom metrics must begin with this domain
+  val CustomMetricDomain = "custom.googleapis.com"
+
+  /**
+    * Scope to write metrics to Stackdriver Monitoring API.
+    * Used by the monitoring action.
+    *
+    * For some reason we couldn't find this scope within Google libraries
+    */
+  val MonitoringScope = "https://www.googleapis.com/auth/monitoring"
+
+  val InitialDelay = 1.minute
+
+  def props(serviceConfig: Config, globalConfig: Config, serviceRegistryActor: ActorRef) = Props(new StackdriverInstrumentationServiceActor(serviceConfig, globalConfig, serviceRegistryActor))
+
+  implicit class CromwellBucketEnhanced(val cromwellBucket: CromwellBucket) extends AnyVal {
+    /**
+      * Transforms a CromwellBucket to a Stackdriver path
+      */
+    def toStackdriverString = (CromwellMetricPrefix ++ cromwellBucket.prefix ++ cromwellBucket.path.toList).mkString("/").replace(" ", "_")
+  }
+}
+
+
+sealed trait StackdriverMetricKind
+object StackdriverGauge extends StackdriverMetricKind
+object StackdriverCumulative extends StackdriverMetricKind
+
+
+case class StackdriverMetric(name: String, kind: StackdriverMetricKind)
+
+
+object SendStackdriverMetricCommand
diff --git a/services/src/main/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfig.scala b/services/src/main/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfig.scala
index 4e18a479d23..d67eb348bdf 100644
--- a/services/src/main/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfig.scala
+++ b/services/src/main/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfig.scala
@@ -2,7 +2,6 @@ package cromwell.services.instrumentation.impl.statsd
 
 import cats.data.Validated._
 import cats.syntax.apply._
-import cats.syntax.validated._
 import com.typesafe.config.Config
 import common.exception.MessageAggregation
 import common.validation.ErrorOr.ErrorOr
@@ -15,12 +14,11 @@ case class StatsDConfig(hostname: String, port: Int, prefix: Option[String], flu
 
 object StatsDConfig {
   def apply(serviceConfig: Config): StatsDConfig = {
-    val statsDConfig = serviceConfig.getConfig("statsd")
 
-    val hostname: ErrorOr[String] = validate[String] { statsDConfig.as[String]("hostname") }
-    val port: ErrorOr[Int] = validate[Int] { statsDConfig.as[Int]("port") }
-    val prefix: ErrorOr[Option[String]] = statsDConfig.as[Option[String]]("prefix").validNel
-    val flushRate: ErrorOr[FiniteDuration] = validate[FiniteDuration] { statsDConfig.as[FiniteDuration]("flush-rate") }
+    val hostname: ErrorOr[String] = validate[String] { serviceConfig.as[String]("hostname") }
+    val port: ErrorOr[Int] = validate[Int] { serviceConfig.as[Int]("port") }
+    val prefix: ErrorOr[Option[String]] = validate { serviceConfig.as[Option[String]]("prefix") }
+    val flushRate: ErrorOr[FiniteDuration] = validate[FiniteDuration] { serviceConfig.as[FiniteDuration]("flush-rate") }
 
     (hostname, port, prefix, flushRate).mapN({ (h, p, n, f) => 
       new StatsDConfig(h, p, n, f)
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfigSpec.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfigSpec.scala
new file mode 100644
index 00000000000..aad69c94212
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverConfigSpec.scala
@@ -0,0 +1,134 @@
+package cromwell.services.instrumentation.impl.stackdriver
+
+import com.typesafe.config.ConfigFactory
+import common.exception.AggregatedMessageException
+import cromwell.core.TestKitSuite
+import org.scalatest.{BeforeAndAfterAll, FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+
+class StackdriverConfigSpec extends TestKitSuite with FlatSpecLike with BeforeAndAfterAll with Matchers {
+  behavior of "StackdriverConfig"
+
+  val googleConfig = ConfigFactory.parseString(
+    s"""
+       |google {
+       |  application-name = "cromwell"
+       |  auths = [
+       |    {
+       |      name = "application-default"
+       |      scheme = "application_default"
+       |    }
+       |  ]
+       |}
+      """.stripMargin
+  )
+
+  it should "correctly parse all config" in {
+    val config = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+        |cromwell-instance-role = "backend"
+        |cromwell-perf-test-case = "perf-test-1"
+      """.stripMargin
+    )
+
+    val stackdriverConfig = StackdriverConfig(config, googleConfig)
+
+    stackdriverConfig.auth.name shouldBe "application-default"
+    stackdriverConfig.googleProject shouldBe "my-project"
+    stackdriverConfig.flushRate shouldBe 1.minute
+    stackdriverConfig.cromwellInstanceRole shouldBe Option("backend")
+    stackdriverConfig.cromwellPerfTestCase shouldBe Option("perf-test-1")
+  }
+
+
+  it should "correctly parse config with optional values" in {
+    val config = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+      """.stripMargin
+    )
+
+    val stackdriverConfig = StackdriverConfig(config, googleConfig)
+
+    stackdriverConfig.auth.name shouldBe "application-default"
+    stackdriverConfig.googleProject shouldBe "my-project"
+    stackdriverConfig.flushRate shouldBe 1.minute
+    stackdriverConfig.cromwellInstanceIdentifier shouldBe None
+    stackdriverConfig.cromwellInstanceRole shouldBe None
+    stackdriverConfig.cromwellPerfTestCase shouldBe None
+  }
+
+
+  it should "correctly add cromwell instance identifier to config" in {
+    val globalConfig = ConfigFactory.parseString(
+      s"""
+         |google {
+         |  application-name = "cromwell"
+         |  auths = [
+         |    {
+         |      name = "application-default"
+         |      scheme = "application_default"
+         |    }
+         |  ]
+         |}
+         |
+         |system.cromwell_id = "cromwell-101"
+      """.stripMargin
+    )
+
+    val config = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+      """.stripMargin
+    )
+
+    val stackdriverConfig = StackdriverConfig(config, globalConfig)
+
+    stackdriverConfig.auth.name shouldBe "application-default"
+    stackdriverConfig.googleProject shouldBe "my-project"
+    stackdriverConfig.flushRate shouldBe 1.minute
+    stackdriverConfig.cromwellInstanceIdentifier shouldBe Option("cromwell-101")
+    stackdriverConfig.cromwellInstanceRole shouldBe None
+    stackdriverConfig.cromwellPerfTestCase shouldBe None
+  }
+
+
+  it should "throw error for invalid auth" in {
+    val config = ConfigFactory.parseString(
+      """
+        |auth = "my-auth"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+      """.stripMargin
+    )
+
+    val exception = the[AggregatedMessageException] thrownBy StackdriverConfig(config, googleConfig)
+
+    exception.getMessage shouldBe """Stackdriver instrumentation config is invalid. Error(s):
+                                    |`auth` scheme is invalid. Errors: NonEmptyList(`google` configuration stanza does not contain an auth named 'my-auth'.  Known auth names: application-default)""".stripMargin
+  }
+
+
+  it should "throw error for invalid flush rate" in {
+    val config = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 30 seconds
+      """.stripMargin
+    )
+
+    val exception = the[AggregatedMessageException] thrownBy StackdriverConfig(config, googleConfig)
+
+    exception.getMessage shouldBe """Stackdriver instrumentation config is invalid. Error(s):
+                                    |`flush-rate` must be 1 minute or longer, specified rate is `30 seconds`. Google's Stackdriver API needs each metric to be sent not more than once per minute.""".stripMargin
+  }
+}
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActorSpec.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActorSpec.scala
new file mode 100644
index 00000000000..02d4a6acdcf
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActorSpec.scala
@@ -0,0 +1,111 @@
+package cromwell.services.instrumentation.impl.stackdriver
+
+import akka.testkit.{TestActorRef, TestProbe}
+import cats.data.NonEmptyList
+import com.typesafe.config.ConfigFactory
+import cromwell.core.TestKitSuite
+import cromwell.services.instrumentation.InstrumentationService.InstrumentationServiceMessage
+import cromwell.services.instrumentation._
+import org.scalatest.concurrent.Eventually
+import org.scalatest.{FlatSpecLike, Matchers}
+
+import scala.concurrent.duration._
+
+class StackdriverInstrumentationServiceActorSpec extends TestKitSuite with FlatSpecLike with Matchers with Eventually {
+  behavior of "StackdriverInstrumentationServiceActor"
+
+  val MaxWaitTime = 2.minutes
+  implicit val pc: PatienceConfig = PatienceConfig(MaxWaitTime)
+
+  val globalConfig = ConfigFactory.parseString(
+    s"""
+       |google {
+       |  application-name = "cromwell"
+       |  auths = [
+       |    {
+       |      name = "application-default"
+       |      scheme = "application_default"
+       |    }
+       |  ]
+       |}
+      """.stripMargin
+  )
+
+  val registryProbe = TestProbe().ref
+  val resourceLabels = Map("project_id" -> "my-project")
+
+  val testBucket = CromwellBucket(List("test_prefix"), NonEmptyList.of("test", "metric", "bucket"))
+  val testGaugeBucket = CromwellBucket(List("test_prefix"), NonEmptyList.of("test", "gauge", "metric", "bucket"))
+
+
+  it should "correctly receive the metrics with resource labels" in {
+    val stackdriverConfig = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+      """.stripMargin
+    )
+
+    val rawMetricList = List(
+      CromwellIncrement(testBucket),
+      CromwellCount(testBucket, 80),
+      CromwellGauge(testGaugeBucket, 20),
+      CromwellTiming(testBucket.expand("timing"), 5.seconds),
+      CromwellIncrement(testBucket),
+      CromwellCount(testBucket, 80),
+      CromwellGauge(testGaugeBucket, 60),
+      CromwellTiming(testBucket.expand("timing"), 10.seconds)
+    )
+
+    val expectedCumulativeMetrics = ("custom.googleapis.com/cromwell/test_prefix/test/metric/bucket", 162.0)
+    val expectedGaugeMetrics = ("custom.googleapis.com/cromwell/test_prefix/test/gauge/metric/bucket", 40.0)
+    val expectedTimingMetrics = ("custom.googleapis.com/cromwell/test_prefix/test/metric/bucket/timing", 7500.0)
+
+    val stackdriverActor = TestActorRef(new TestStackdriverInstrumentationServiceActor(stackdriverConfig, globalConfig, registryProbe))
+
+    rawMetricList foreach (metric => stackdriverActor ! InstrumentationServiceMessage(metric))
+
+    eventually {
+      stackdriverActor.underlyingActor.metricsReceived should have length 3
+
+      stackdriverActor.underlyingActor.metricsReceived.map(m => (m.metricPath, m.metricValue)) should contain (expectedCumulativeMetrics)
+      stackdriverActor.underlyingActor.metricsReceived.map(m => (m.metricPath, m.metricValue)) should contain (expectedGaugeMetrics)
+      stackdriverActor.underlyingActor.metricsReceived.map(m => (m.metricPath, m.metricValue)) should contain (expectedTimingMetrics)
+
+      stackdriverActor.underlyingActor.metricsReceived.map(_.resourceLabels) should contain (resourceLabels)
+    }
+  }
+
+
+  it should "correctly receive metrics with metric labels" in {
+    val stackdriverConfig = ConfigFactory.parseString(
+      """
+        |auth = "application-default"
+        |google-project = "my-project"
+        |flush-rate = 1 minute
+        |cromwell-instance-role = "backend"
+        |cromwell-perf-test-case = "perf-test-1"
+      """.stripMargin
+    )
+
+    val rawMetricList = List(
+      CromwellIncrement(testBucket),
+      CromwellCount(testBucket, 49)
+    )
+
+    val expectedCumulativeMetrics = ("custom.googleapis.com/cromwell/test_prefix/test/metric/bucket", 50.0)
+    val metricLabels = Map("cromwell_instance_role" -> "backend", "cromwell_perf_test_case" -> "perf-test-1")
+
+    val stackdriverActor = TestActorRef(new TestStackdriverInstrumentationServiceActor(stackdriverConfig, globalConfig, registryProbe))
+
+    rawMetricList foreach (metric => stackdriverActor ! InstrumentationServiceMessage(metric))
+
+    eventually {
+      stackdriverActor.underlyingActor.metricsReceived should have length 1
+      stackdriverActor.underlyingActor.metricsReceived.map(m => (m.metricPath, m.metricValue)) should contain (expectedCumulativeMetrics)
+      stackdriverActor.underlyingActor.metricsReceived.map(_.resourceLabels) should contain (resourceLabels)
+      stackdriverActor.underlyingActor.metricsReceived.map(_.metricLabels) should contain (metricLabels)
+    }
+  }
+}
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/TestStackdriverInstrumentationServiceActor.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/TestStackdriverInstrumentationServiceActor.scala
new file mode 100644
index 00000000000..e804b3593ac
--- /dev/null
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/stackdriver/TestStackdriverInstrumentationServiceActor.scala
@@ -0,0 +1,28 @@
+package cromwell.services.instrumentation.impl.stackdriver
+
+import akka.actor.ActorRef
+import com.google.monitoring.v3.CreateTimeSeriesRequest
+import com.typesafe.config.Config
+import scala.collection.JavaConverters._
+
+class TestStackdriverInstrumentationServiceActor(serviceConfig: Config, globalConfig: Config, serviceRegistryActor: ActorRef)
+  extends StackdriverInstrumentationServiceActor(serviceConfig, globalConfig, serviceRegistryActor) {
+
+  var metricsReceived = List[TimeSeriesRequest]()
+
+  override def sendTimeSeriesToStackdriver(timeSeriesRequest: CreateTimeSeriesRequest) = {
+    val timeSeries = timeSeriesRequest.getTimeSeries(0)
+    val metric = timeSeries.getMetric
+
+    metricsReceived = metricsReceived :+ TimeSeriesRequest(metric.getType,
+      timeSeries.getPoints(0).getValue.getDoubleValue,
+      timeSeries.getResource.getLabelsMap.asScala.toMap,
+      metric.getLabelsMap.asScala.toMap)
+  }
+}
+
+
+case class TimeSeriesRequest(metricPath: String,
+                             metricValue: Double,
+                             resourceLabels: Map[String, String],
+                             metricLabels: Map[String, String])
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfigSpec.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfigSpec.scala
index e221587c476..cc491ca4859 100644
--- a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfigSpec.scala
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDConfigSpec.scala
@@ -9,12 +9,11 @@ class StatsDConfigSpec extends FlatSpec with Matchers {
   
   it should "parse correct service configuration" in {
     val config = ConfigFactory.parseString(
-      """statsd {
+      """
         |hostname = "localhost"
         |port = 8125
         |prefix = "prefix_value" # can be used to prefix all metrics with an api key for example
         |flush-rate = 1 second # rate at which aggregated metrics will be sent to statsd
-        |}
       """.stripMargin
     )
 
@@ -27,7 +26,7 @@ class StatsDConfigSpec extends FlatSpec with Matchers {
   }
 
   it should "not parse incorrect service configuration" in {
-    val config = ConfigFactory.parseString("statsd {}")
+    val config = ConfigFactory.parseString("{}")
 
     val exception = the[IllegalArgumentException] thrownBy StatsDConfig(config)
 
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorBenchmarkSpec.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorBenchmarkSpec.scala
index 14b0e7b9817..f806b24a98e 100644
--- a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorBenchmarkSpec.scala
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorBenchmarkSpec.scala
@@ -16,12 +16,11 @@ class StatsDInstrumentationServiceActorBenchmarkSpec extends TestKitSuite with F
   behavior of "StatsDInstrumentationServiceActor"
 
   val config = ConfigFactory.parseString(
-    """statsd {
+    """
       |hostname = "localhost"
       |port = 8125
       |prefix = "prefix_value" # can be used to prefix all metrics with an api key for example
       |flush-rate = 100 ms # rate at which aggregated metrics will be sent to statsd
-      |}
     """.stripMargin
   )
 
diff --git a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorSpec.scala b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorSpec.scala
index 81b6d67c510..5e5087e8bdb 100644
--- a/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/instrumentation/impl/statsd/StatsDInstrumentationServiceActorSpec.scala
@@ -17,12 +17,11 @@ class StatsDInstrumentationServiceActorSpec extends TestKitSuite with FlatSpecLi
   behavior of "StatsDInstrumentationServiceActor"
 
   val config = ConfigFactory.parseString(
-    """statsd {
+    """
       |hostname = "localhost"
       |port = 8125
       |prefix = "prefix_value" # can be used to prefix all metrics with an api key for example
       |flush-rate = 100 ms # rate at which aggregated metrics will be sent to statsd
-      |}
     """.stripMargin
   )
 

From 305e6640709122d7f36151f2da4c4db75a2f744e Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Wed, 12 Jun 2019 09:25:03 -0400
Subject: [PATCH 10/28] Running dockers for postgres and mariadb during tests -
 Switched from running Travis' mysql/postgresql instances to dockerized
 versions - Added a couple of centaur and some unit tests for mariadb, using
 mysql liquibase/drivers for now - NOTE: No postgresql tests enabled in this
 commit - Changed mysql CI login from travis: to cromwell:test - DRYed out
 more of the CI scripts into test.inc.sh - Switched CI
 broadinstitute/dsde-toolbox from :latest to :dev - Don't fail with false
 negatives when MetadataDatabaseAccessSpec is re-run against persistent RDBMS

---
 .travis.yml                                   |  93 +++-
 core/src/test/resources/application.conf      |  59 ++-
 .../CallCachingSlickDatabaseSpec.scala        |   2 +
 .../workflowstore/SqlWorkflowStoreSpec.scala  |   2 +
 project/ContinuousIntegration.scala           |   2 +-
 .../docker-compose-horicromtal.yml            |  60 ++-
 .../cromwell/services/ServicesStoreSpec.scala |   2 +
 .../keyvalue/impl/KeyValueDatabaseSpec.scala  |   5 +
 .../impl/MetadataDatabaseAccessSpec.scala     |  41 +-
 src/ci/bin/test.inc.sh                        | 416 +++++++++++++-----
 src/ci/bin/testCentaurEngineUpgradeLocal.sh   |  15 +-
 ...stCentaurHoricromtalEngineUpgradePapiV2.sh |  77 ++--
 src/ci/bin/testCentaurHoricromtalPapiV2.sh    |   1 -
 ...estCentaurPapiUpgradeNewWorkflowsPapiV1.sh |  19 +-
 src/ci/bin/testCentaurPapiUpgradePapiV1.sh    |  17 +-
 .../cromwell-test/docker-setup.sh             |   3 +-
 src/ci/docker-compose/docker-compose.yml      |  26 +-
 src/ci/docker-compose/mariadb-conf.d/init.cnf |   3 +
 src/ci/docker-compose/mysql-conf.d/init.cnf   |   3 +
 .../postgresql-initdb.d/init.sh               |   5 +
 .../postgresql-initdb.d/init.sql              |   1 +
 src/ci/resources/build_application.inc.conf   |   2 +-
 src/ci/resources/cromwell_database.inc.conf   |  21 +-
 23 files changed, 635 insertions(+), 240 deletions(-)
 create mode 100644 src/ci/docker-compose/mariadb-conf.d/init.cnf
 create mode 100644 src/ci/docker-compose/mysql-conf.d/init.cnf
 create mode 100755 src/ci/docker-compose/postgresql-initdb.d/init.sh
 create mode 100644 src/ci/docker-compose/postgresql-initdb.d/init.sql

diff --git a/.travis.yml b/.travis.yml
index 4a40d3e5af3..724c2eba532 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,7 +2,6 @@ sudo: required
 dist: trusty
 services:
   - docker
-  - mysql
 language: scala
 scala:
   - 2.12.6
@@ -22,27 +21,77 @@ before_cache:
 env:
   matrix:
     # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked
-    - BUILD_TYPE=centaurAws
-    - BUILD_TYPE=centaurBcs
-    - BUILD_TYPE=centaurEngineUpgradeLocal
-    - BUILD_TYPE=centaurEngineUpgradePapiV2
-    - BUILD_TYPE=centaurHoricromtalPapiV2
-    - BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2
-    - BUILD_TYPE=centaurPapiUpgradePapiV1
-    - BUILD_TYPE=centaurPapiUpgradeNewWorkflowsPapiV1
-    - BUILD_TYPE=centaurLocal
-    - BUILD_TYPE=centaurPapiV1
-    - BUILD_TYPE=centaurPapiV2
-    - BUILD_TYPE=centaurSlurm
-    - BUILD_TYPE=centaurTes
-    - BUILD_TYPE=centaurWdlUpgradeLocal
-    - BUILD_TYPE=checkPublish
-    - BUILD_TYPE=conformanceLocal
-    - BUILD_TYPE=conformancePapiV2
-    - BUILD_TYPE=conformanceTesk
-    - BUILD_TYPE=dockerDeadlock
-    - BUILD_TYPE=dockerScripts
-    - BUILD_TYPE=sbt
+    - >-
+      BUILD_TYPE=centaurAws
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurBcs
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurEngineUpgradeLocal
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurEngineUpgradePapiV2
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurHoricromtalPapiV2
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurHoricromtalPapiV2
+      BUILD_MARIADB=10.3
+    - >-
+      BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2
+      BUILD_MARIADB=10.3
+    - >-
+      BUILD_TYPE=centaurPapiUpgradePapiV1
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurPapiUpgradeNewWorkflowsPapiV1
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurLocal
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurPapiV1
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurPapiV2
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurSlurm
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurTes
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurWdlUpgradeLocal
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=checkPublish
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=conformanceLocal
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=conformancePapiV2
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=conformanceTesk
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=dockerDeadlock
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=dockerScripts
+      BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=sbt
+      BUILD_MYSQL=5.7
+      BUILD_POSTGRESQL=11.3
+      BUILD_MARIADB=10.3
 script:
   - src/ci/bin/test.sh
 notifications:
diff --git a/core/src/test/resources/application.conf b/core/src/test/resources/application.conf
index d6d14d3c9b7..6d479c92e23 100644
--- a/core/src/test/resources/application.conf
+++ b/core/src/test/resources/application.conf
@@ -21,21 +21,60 @@ database.db.connectionTimeout = 30000
 
 database-test-mysql {
   # Run the following to (optionally) drop and (re-)create the database:
-  # mysql -utravis -e "DROP DATABASE IF EXISTS cromwell_test" && mysql -utravis -e "CREATE DATABASE cromwell_test"
+  # mysql -ucromwell -ptest -e "DROP DATABASE IF EXISTS cromwell_test; CREATE DATABASE cromwell_test;"
   profile = "slick.jdbc.MySQLProfile$"
   db {
-    hostname = localhost
-    hostname = ${?CROMWELL_BUILD_MYSQL_HOSTNAME}
-    port = 3306
-    port = ${?CROMWELL_BUILD_MYSQL_PORT}
-    schema = cromwell_test
-    schema = ${?CROMWELL_BUILD_MYSQL_SCHEMA}
-    url = "jdbc:mysql://"${database-test-mysql.db.hostname}":"${database-test-mysql.db.port}"/"${database-test-mysql.db.schema}"?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
-    user = "travis"
+    driver = "com.mysql.cj.jdbc.Driver"
+    url = "jdbc:mysql://localhost:3306/cromwell_test?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    url = ${?CROMWELL_BUILD_MYSQL_JDBC_URL}
+    user = "cromwell"
     user = ${?CROMWELL_BUILD_MYSQL_USERNAME}
-    password = ""
+    password = "test"
     password = ${?CROMWELL_BUILD_MYSQL_PASSWORD}
+    connectionTimeout = 5000
+  }
+}
+
+database-test-mariadb {
+  # Installing both mysql and mariadb takes skill... Instead, try running this docker from the cromwell directory:
+  #
+  # docker run \
+  #   --rm \
+  #   --env MYSQL_ROOT_PASSWORD=private \
+  #   --env MYSQL_USER=cromwell \
+  #   --env MYSQL_PASSWORD=test \
+  #   --env MYSQL_DATABASE=cromwell_test \
+  #   --publish 13306:3306 \
+  #   --volume ${PWD}/src/ci/docker-compose/mariadb-conf.d:/etc/mysql/conf.d \
+  #   mariadb:10.3
+
+  # Run the following to (optionally) drop and (re-)create the database:
+  # mysql --protocol=tcp -P13306 -ucromwell -ptest -e "DROP DATABASE IF EXISTS cromwell_test; CREATE DATABASE cromwell_test;"
+  profile = "slick.jdbc.MySQLProfile$"
+  db {
     driver = "com.mysql.cj.jdbc.Driver"
+    url = "jdbc:mysql://localhost:13306/cromwell_test?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    url = ${?CROMWELL_BUILD_MARIADB_JDBC_URL}
+    user = "cromwell"
+    user = ${?CROMWELL_BUILD_MARIADB_USERNAME}
+    password = "test"
+    password = ${?CROMWELL_BUILD_MARIADB_PASSWORD}
+    connectionTimeout = 5000
+  }
+}
+
+database-test-postgres {
+  # Run the following to (optionally) drop and (re-)create the database:
+  # psql postgres <<< 'drop database if exists cromwell_test; create database cromwell_test;'
+  profile = "slick.jdbc.PostgresProfile$"
+  db {
+    driver = "org.postgresql.Driver"
+    url = "jdbc:postgresql://localhost:5432/cromwell_test"
+    url = ${?CROMWELL_BUILD_POSTGRES_JDBC_URL}
+    user = "cromwell"
+    user = ${?CROMWELL_BUILD_POSTGRES_USERNAME}
+    password = "test"
+    password = ${?CROMWELL_BUILD_POSTGRES_PASSWORD}
     connectionTimeout = 5000
   }
 }
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
index e6ddda07103..b28c250490b 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
@@ -25,6 +25,8 @@ class CallCachingSlickDatabaseSpec extends FlatSpec with Matchers with ScalaFutu
 
   "SlickDatabase (mysql)" should behave like testWith("database-test-mysql")
 
+  "SlickDatabase (mariadb)" should behave like testWith("database-test-mariadb")
+
   def testWith(configPath: String): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
     lazy val dataAccess = new EngineSlickDatabase(databaseConfig)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index 69a3b901f30..aab401c9934 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -29,6 +29,8 @@ class SqlWorkflowStoreSpec extends FlatSpec with Matchers with ScalaFutures with
 
   "SqlWorkflowStore (mysql)" should behave like testWith("database-test-mysql")
 
+  "SqlWorkflowStore (mariadb)" should behave like testWith("database-test-mariadb")
+
   def testWith(configPath: String): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
 
diff --git a/project/ContinuousIntegration.scala b/project/ContinuousIntegration.scala
index b4cff3af85f..ec6a3107a42 100644
--- a/project/ContinuousIntegration.scala
+++ b/project/ContinuousIntegration.scala
@@ -41,7 +41,7 @@ object ContinuousIntegration {
         "-e", "ENVIRONMENT=not_used",
         "-e", s"INPUT_PATH=${srcCiResources.value}",
         "-e", s"OUT_PATH=${targetCiResources.value}",
-        "broadinstitute/dsde-toolbox", "render-templates.sh"
+        "broadinstitute/dsde-toolbox:dev", "render-templates.sh"
       )
       val result = cmd ! log
       if (result != 0) {
diff --git a/scripts/docker-compose-mysql/docker-compose-horicromtal.yml b/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
index 5faf0bc5bf8..f6458ecc6c2 100644
--- a/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
+++ b/scripts/docker-compose-mysql/docker-compose-horicromtal.yml
@@ -13,6 +13,33 @@
 # into the CI config files.
 version: '2.3'
 services:
+  # Runs the database initialization but is NOT a workflow-running backend.
+  cromwell_database_master:
+    image: "broadinstitute/cromwell:${CROMWELL_TAG}"
+    network_mode: host
+    working_dir: /cromwell-working-dir
+    volumes:
+      - ${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}
+    command: ["server"]
+    environment:
+      - >-
+        JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG}
+        -Dwebservice.port=8080
+        -Dsystem.cromwell_id=master
+        -Dsystem.max-workflow-launch-count=0
+        -Dsystem.new-workflow-poll-rate=999999
+        -Dservices.MetadataService.config.metadata-summary-refresh-interval=Inf
+      - CROMWELL_BUILD_RESOURCES_DIRECTORY
+      - CROMWELL_BUILD_CENTAUR_SLICK_PROFILE
+      - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER
+      - CROMWELL_BUILD_CENTAUR_JDBC_USERNAME
+      - CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD
+      - CROMWELL_BUILD_CENTAUR_JDBC_URL
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8080"]
+      interval: 2s
+      timeout: 30s
+      retries: 60
   # Runs the summarizer and is also a regular Cromwell workflow-running backend.
   cromwell_summarizer_plus_backend:
     image: "broadinstitute/cromwell:${CROMWELL_TAG}"
@@ -22,9 +49,19 @@ services:
       - ${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}
     command: ["server"]
     environment:
-      - JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG} -Dwebservice.port=8000 -Dsystem.cromwell_id=summarizer
-      - CROMWELL_BUILD_RESOURCES_DIRECTORY=${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources
-      - CROMWELL_BUILD_MYSQL_USERNAME=${CROMWELL_BUILD_MYSQL_USERNAME}
+      - >-
+        JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG}
+        -Dwebservice.port=8000
+        -Dsystem.cromwell_id=summarizer
+      - CROMWELL_BUILD_RESOURCES_DIRECTORY
+      - CROMWELL_BUILD_CENTAUR_SLICK_PROFILE
+      - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER
+      - CROMWELL_BUILD_CENTAUR_JDBC_USERNAME
+      - CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD
+      - CROMWELL_BUILD_CENTAUR_JDBC_URL
+    depends_on:
+      cromwell_database_master:
+        condition: service_healthy
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8000"]
       interval: 2s
@@ -37,11 +74,22 @@ services:
     working_dir: /cromwell-working-dir
     volumes:
       - ${CROMWELL_BUILD_ROOT_DIRECTORY}:${CROMWELL_BUILD_ROOT_DIRECTORY}
+    depends_on:
+      cromwell_database_master:
+        condition: service_healthy
     command: ["server"]
     environment:
-      - JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG} -Dwebservice.port=${MANAGED_CROMWELL_PORT-8008} -Dsystem.cromwell_id=frontend
-      - CROMWELL_BUILD_RESOURCES_DIRECTORY=${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources
-      - CROMWELL_BUILD_MYSQL_USERNAME=${CROMWELL_BUILD_MYSQL_USERNAME}
+      - >-
+        JAVA_OPTS=-Dconfig.file=${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_CONFIG}
+        -Dwebservice.port=${MANAGED_CROMWELL_PORT-8008}
+        -Dsystem.cromwell_id=frontend
+        -Dservices.MetadataService.config.metadata-summary-refresh-interval=Inf
+      - CROMWELL_BUILD_RESOURCES_DIRECTORY
+      - CROMWELL_BUILD_CENTAUR_SLICK_PROFILE
+      - CROMWELL_BUILD_CENTAUR_JDBC_DRIVER
+      - CROMWELL_BUILD_CENTAUR_JDBC_USERNAME
+      - CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD
+      - CROMWELL_BUILD_CENTAUR_JDBC_URL
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:8008"]
       interval: 2s
diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index 119114ccc84..275398a6609 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -251,6 +251,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
   "SlickDatabase (mysql)" should behave like testWith("database-test-mysql")
 
+  "SlickDatabase (mariadb)" should behave like testWith("database-test-mariadb")
+
   def testWith(configPath: String): Unit = {
     import ServicesStore.EnhancedSqlDatabase
 
diff --git a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
index d6c0d591f42..95749caddd7 100644
--- a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
+++ b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
@@ -32,6 +32,11 @@ class KeyValueDatabaseSpec extends FlatSpec with Matchers with ScalaFutures with
     "Column 'STORE_VALUE' cannot be null"
   )
 
+  "SlickDatabase (mariadb)" should behave like testWith[BatchUpdateException](
+    "database-test-mariadb",
+    "Column 'STORE_VALUE' cannot be null"
+  )
+
   def testWith[E <: Throwable](configPath: String, failureMessage: String)(implicit classTag: ClassTag[E]): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
     lazy val dataAccess = new EngineSlickDatabase(databaseConfig)
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
index de476a1eee7..1e74eb0bb49 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
@@ -39,6 +39,8 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
 
   "MetadataDatabaseAccess (mysql)" should behave like testWith("database-test-mysql")
 
+  "MetadataDatabaseAccess (mariadb)" should behave like testWith("database-test-mariadb")
+
   implicit val ec = ExecutionContext.global
 
   implicit val defaultPatience = PatienceConfig(scaled(Span(30, Seconds)), scaled(Span(100, Millis)))
@@ -117,6 +119,12 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
     }
 
     it should "sort metadata events by timestamp from older to newer" taggedAs DbmsTest in {
+
+      if (configPath == "database-test-mariadb") {
+        // Do NOT want to change the test. Instead should fix it so that MariaDB is storing at least milliseconds.
+        cancel("Having issues with MariaDB and fractional seconds. https://broadworkbench.atlassian.net/browse/BA-5692")
+      }
+
       def unorderedEvents(id: WorkflowId): Future[Vector[MetadataEvent]] = {
         val workflowKey = MetadataKey(id, jobKey = None, key = null)
         val now = OffsetDateTime.now()
@@ -273,7 +281,9 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
           WorkflowQueryKey.ExcludeLabelAndKeyValue.name -> s"${testLabel2.key}:${testLabel2.value}"))) map { case (response, _) =>
           val resultByName = response.results groupBy (_.name)
-          withClue("Filter by exclude label using AND") { resultByName.keys.toSet.flatten should equal(Set(Workflow1Name)) }
+          withClue("Filter by exclude label using AND") {
+            resultByName.keys.toSet.flatten should contain(Workflow1Name)
+          }
         }
         // Filter by multiple exclude labels using AND
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
@@ -281,30 +291,31 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
             .map(label => WorkflowQueryKey.ExcludeLabelAndKeyValue.name -> s"${label.key}:${label.value}"))
         ) map { case (response, _) =>
           val resultByName = response.results groupBy (_.name)
-          withClue("Filter by multiple exclude labels using AND") { resultByName.keys.toSet.flatten should equal(Set(Workflow1Name)) }
-          response.totalResultsCount match {
-            case 3 => //good
-            case ct => fail(s"totalResultsCount for multiple exclude labels using AND query is expected to be 3 but is actually $ct. " +
-              s"Something has gone horribly wrong!")
+          val ids = response.results.map(_.id)
+          withClue("Filter by multiple exclude labels using AND") {
+            resultByName.keys.toSet.flatten should contain(Workflow1Name)
+            ids should contain(workflow1Id.toString)
+            ids shouldNot contain(workflow2Id.toString)
           }
         }
         // Filter by exclude label using OR
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
           WorkflowQueryKey.ExcludeLabelOrKeyValue.name -> s"${testLabel2.key}:${testLabel2.value}"))) map { case (response, _) =>
           val resultByName = response.results groupBy (_.name)
-          withClue("Filter to exclude label using OR") { resultByName.keys.toSet.flatten should equal(Set(Workflow1Name)) }
+          withClue("Filter to exclude label using OR") {
+            resultByName.keys.toSet.flatten should contain(Workflow1Name)
+          }
         }
         // Filter by multiple exclude labels using OR
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(
           Seq(testLabel2, testLabel3)
             .map(label => WorkflowQueryKey.ExcludeLabelOrKeyValue.name -> s"${label.key}:${label.value}"))
         ) map { case (response, _) =>
-          val resultByName = response.results groupBy (_.name)
-          withClue("Filter by multiple exclude labels using OR") { resultByName.keys.toSet.flatten should equal(Set(Workflow1Name)) }
-          response.totalResultsCount match {
-            case 2 => //good
-            case ct => fail(s"totalResultsCount is for multiple exclude labels using OR query is expected to be 2 but is actually $ct. " +
-              s"Something has gone horribly wrong!")
+          // NOTE: On persistent databases other workflows will be returned. Just verify that our two workflows are not.
+          val ids = response.results.map(_.id)
+          withClue("Filter by multiple exclude labels using OR") {
+            ids shouldNot contain(workflow1Id.toString)
+            ids shouldNot contain(workflow2Id.toString)
           }
         }
         // Filter by start date
@@ -345,7 +356,9 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
           WorkflowQueryKey.AdditionalQueryResultFields.name -> "labels"))) map {
           case (response, _) =>
-            response.results partition { r => r.labels.isDefined} match {
+            response.results filter {
+              workflowQueryResult => List(workflow1Id.toString, workflow1Id.toString).contains(workflowQueryResult.id)
+            } partition { r => r.labels.isDefined } match {
               case (y, n) if y.nonEmpty && n.isEmpty => //good
               case (y, n) => fail(s"Something went horribly wrong since labels were populated for ${y.size} and were missing for ${n.size} workflow(s)!")
             }
diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh
index d3ee08d39ea..9749082c628 100644
--- a/src/ci/bin/test.inc.sh
+++ b/src/ci/bin/test.inc.sh
@@ -63,6 +63,7 @@ cromwell::private::create_build_variables() {
     CROMWELL_BUILD_LOG_DIRECTORY="${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/logs"
     CROMWELL_BUILD_CROMWELL_LOG="${CROMWELL_BUILD_LOG_DIRECTORY}/cromwell.log"
 
+    CROMWELL_BUILD_DOCKER_DIRECTORY="${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/docker-compose"
     CROMWELL_BUILD_SCRIPTS_DIRECTORY="${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/bin"
     CROMWELL_BUILD_RESOURCES_SOURCES="${CROMWELL_BUILD_ROOT_DIRECTORY}/src/ci/resources"
     CROMWELL_BUILD_RESOURCES_DIRECTORY="${CROMWELL_BUILD_ROOT_DIRECTORY}/target/ci/resources"
@@ -79,12 +80,17 @@ cromwell::private::create_build_variables() {
       CROMWELL_BUILD_IS_VIRTUAL_ENV=false
     fi
 
-    CROMWELL_BUILD_RUN_TESTS=true
-
     case "${CROMWELL_BUILD_PROVIDER}" in
         "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
             CROMWELL_BUILD_IS_CI=true
             CROMWELL_BUILD_IS_SECURE="${TRAVIS_SECURE_ENV_VARS}"
+
+            if [[ -n "${TRAVIS_PULL_REQUEST_BRANCH:+set}" ]]; then
+              CROMWELL_BUILD_IS_PULL_REQUEST=true
+            else
+              CROMWELL_BUILD_IS_PULL_REQUEST=false
+            fi
+
             CROMWELL_BUILD_TYPE="${BUILD_TYPE}"
             CROMWELL_BUILD_BRANCH="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}"
             CROMWELL_BUILD_BRANCH_PULL_REQUEST="${TRAVIS_PULL_REQUEST_BRANCH:-""}"
@@ -95,27 +101,24 @@ cromwell::private::create_build_variables() {
             CROMWELL_BUILD_GIT_USER_EMAIL="travis@travis-ci.com"
             CROMWELL_BUILD_GIT_USER_NAME="Travis CI"
             CROMWELL_BUILD_HEARTBEAT_PATTERN="…"
-            CROMWELL_BUILD_MYSQL_HOSTNAME="localhost"
-            CROMWELL_BUILD_MYSQL_PORT="3306"
-            CROMWELL_BUILD_MYSQL_USERNAME="travis"
-            CROMWELL_BUILD_MYSQL_PASSWORD=""
-            CROMWELL_BUILD_MYSQL_SCHEMA="cromwell_test"
             CROMWELL_BUILD_GENERATE_COVERAGE=true
 
             # Always run on sbt, even for 'push'.
             # This allows quick sanity checks before starting PRs *and* publishing after merges into develop.
-            if [[ "${CROMWELL_BUILD_TYPE}" == "sbt" ]]; then
-              CROMWELL_BUILD_RUN_TESTS=true
-            elif [[ "${TRAVIS_COMMIT_MESSAGE}" != *"[force ci]"* ]] && [[ "${TRAVIS_EVENT_TYPE}" == "push" ]]; then
+            if [[ "${TRAVIS_EVENT_TYPE}" == "push" ]] && \
+                [[ "${BUILD_TYPE}" != "sbt" ]] && \
+                [[ "${TRAVIS_COMMIT_MESSAGE}" != *"[force ci]"* ]]; then
               CROMWELL_BUILD_RUN_TESTS=false
+            else
+              CROMWELL_BUILD_RUN_TESTS=true
             fi
             ;;
         "${CROMWELL_BUILD_PROVIDER_JENKINS}")
             # External variables must be passed through in the ENVIRONMENT of src/ci/docker-compose/docker-compose.yml
             CROMWELL_BUILD_IS_CI=true
             CROMWELL_BUILD_IS_SECURE=true
+            CROMWELL_BUILD_IS_PULL_REQUEST=false
             CROMWELL_BUILD_TYPE="${JENKINS_BUILD_TYPE}"
-            CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS="${CENTAUR_TEST_ADDITIONAL_PARAMETERS:-""}"
             CROMWELL_BUILD_BRANCH="${GIT_BRANCH#origin/}"
             CROMWELL_BUILD_BRANCH_PULL_REQUEST=""
             CROMWELL_BUILD_EVENT=""
@@ -125,16 +128,13 @@ cromwell::private::create_build_variables() {
             CROMWELL_BUILD_GIT_USER_EMAIL="jenkins@jenkins.io"
             CROMWELL_BUILD_GIT_USER_NAME="Jenkins CI"
             CROMWELL_BUILD_HEARTBEAT_PATTERN="…\n"
-            CROMWELL_BUILD_MYSQL_HOSTNAME="mysql-db"
-            CROMWELL_BUILD_MYSQL_PORT="3306"
-            CROMWELL_BUILD_MYSQL_USERNAME="root"
-            CROMWELL_BUILD_MYSQL_PASSWORD=""
-            CROMWELL_BUILD_MYSQL_SCHEMA="cromwell_test"
             CROMWELL_BUILD_GENERATE_COVERAGE=false
+            CROMWELL_BUILD_RUN_TESTS=true
             ;;
         *)
             CROMWELL_BUILD_IS_CI=false
             CROMWELL_BUILD_IS_SECURE=true
+            CROMWELL_BUILD_IS_PULL_REQUEST=false
             CROMWELL_BUILD_TYPE="unknown"
             CROMWELL_BUILD_BRANCH="unknown"
             CROMWELL_BUILD_BRANCH_PULL_REQUEST=""
@@ -145,12 +145,8 @@ cromwell::private::create_build_variables() {
             CROMWELL_BUILD_GIT_USER_EMAIL="unknown.git.user@example.org"
             CROMWELL_BUILD_GIT_USER_NAME="Unknown Git User"
             CROMWELL_BUILD_HEARTBEAT_PATTERN="…"
-            CROMWELL_BUILD_MYSQL_HOSTNAME="${CROMWELL_BUILD_MYSQL_HOSTNAME-localhost}"
-            CROMWELL_BUILD_MYSQL_PORT="${CROMWELL_BUILD_MYSQL_PORT-3306}"
-            CROMWELL_BUILD_MYSQL_USERNAME="${CROMWELL_BUILD_MYSQL_USERNAME-root}"
-            CROMWELL_BUILD_MYSQL_PASSWORD="${CROMWELL_BUILD_MYSQL_PASSWORD-}"
-            CROMWELL_BUILD_MYSQL_SCHEMA="${CROMWELL_BUILD_MYSQL_SCHEMA-cromwell_test}"
             CROMWELL_BUILD_GENERATE_COVERAGE=true
+            CROMWELL_BUILD_RUN_TESTS=true
 
             local bash_script
             for bash_script in "${BASH_SOURCE[@]}"; do
@@ -178,25 +174,24 @@ cromwell::private::create_build_variables() {
     backend_type="$(echo "${backend_type}" | sed 's/\([A-Z]\)/_\1/g' | tr '[:upper:]' '[:lower:]' | cut -c 2-)"
     CROMWELL_BUILD_BACKEND_TYPE="${backend_type}"
 
-    CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf"
-
-    if [[ -z "${CROMWELL_BUILD_OPTIONAL_SECURE-}" ]]; then
-        CROMWELL_BUILD_OPTIONAL_SECURE=false
-    fi
-
-    if [[ -z "${CROMWELL_BUILD_REQUIRES_SECURE-}" ]]; then
-        CROMWELL_BUILD_REQUIRES_SECURE=false
-    fi
-
-    if [[ -z "${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND-}" ]]; then
+    if [[ "${CROMWELL_BUILD_TYPE}" == conformance* ]]; then
+        CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND="server/assembly centaurCwlRunner/assembly"
+    else
         CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND="assembly"
     fi
 
-    if [[ -z "${VAULT_TOKEN-}" ]]; then
-        VAULT_TOKEN="vault token is not set as an environment variable"
+    if [[ "${CROMWELL_BUILD_TYPE}" == centaurPapiUpgrade* ]] || \
+        [[ "${CROMWELL_BUILD_TYPE}" == centaurHoricromtalEngineUpgrade* ]]; then
+        CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v1_v2_upgrade_application.conf"
+    else
+        CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/${CROMWELL_BUILD_BACKEND_TYPE}_application.conf"
     fi
 
-    CROMWELL_BUILD_RANDOM_256_BITS_BASE64="$(dd bs=1 count=32 if=/dev/urandom 2> /dev/null | base64 | tr -d '\n')"
+    CROMWELL_BUILD_OPTIONAL_SECURE="${CROMWELL_BUILD_OPTIONAL_SECURE-false}"
+    CROMWELL_BUILD_REQUIRES_SECURE="${CROMWELL_BUILD_REQUIRES_SECURE-false}"
+    CROMWELL_BUILD_REQUIRES_PULL_REQUEST="${CROMWELL_BUILD_REQUIRES_PULL_REQUEST-false}"
+    CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND="${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND-assembly}"
+    VAULT_TOKEN="${VAULT_TOKEN-vault token is not set as an environment variable}"
 
     local hours_to_minutes
     hours_to_minutes=60
@@ -205,26 +200,22 @@ cromwell::private::create_build_variables() {
     export CROMWELL_BUILD_BACKEND_TYPE
     export CROMWELL_BUILD_BRANCH
     export CROMWELL_BUILD_BRANCH_PULL_REQUEST
-    export CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS
     export CROMWELL_BUILD_CROMWELL_CONFIG
     export CROMWELL_BUILD_CROMWELL_LOG
+    export CROMWELL_BUILD_DOCKER_DIRECTORY
     export CROMWELL_BUILD_EVENT
     export CROMWELL_BUILD_EXIT_FUNCTIONS
     export CROMWELL_BUILD_GENERATE_COVERAGE
     export CROMWELL_BUILD_GIT_USER_EMAIL
     export CROMWELL_BUILD_GIT_USER_NAME
-    export CROMWELL_BUILD_HEARTBEAT_PATTERN
     export CROMWELL_BUILD_HEARTBEAT_MINUTES
+    export CROMWELL_BUILD_HEARTBEAT_PATTERN
     export CROMWELL_BUILD_HOME_DIRECTORY
     export CROMWELL_BUILD_IS_CI
+    export CROMWELL_BUILD_IS_PULL_REQUEST
     export CROMWELL_BUILD_IS_SECURE
     export CROMWELL_BUILD_IS_VIRTUAL_ENV
     export CROMWELL_BUILD_LOG_DIRECTORY
-    export CROMWELL_BUILD_MYSQL_HOSTNAME
-    export CROMWELL_BUILD_MYSQL_PASSWORD
-    export CROMWELL_BUILD_MYSQL_PORT
-    export CROMWELL_BUILD_MYSQL_SCHEMA
-    export CROMWELL_BUILD_MYSQL_USERNAME
     export CROMWELL_BUILD_NUMBER
     export CROMWELL_BUILD_OPTIONAL_SECURE
     export CROMWELL_BUILD_OS
@@ -234,11 +225,11 @@ cromwell::private::create_build_variables() {
     export CROMWELL_BUILD_PROVIDER_JENKINS
     export CROMWELL_BUILD_PROVIDER_TRAVIS
     export CROMWELL_BUILD_PROVIDER_UNKNOWN
-    export CROMWELL_BUILD_RANDOM_256_BITS_BASE64
     export CROMWELL_BUILD_REQUIRES_SECURE
     export CROMWELL_BUILD_RESOURCES_DIRECTORY
     export CROMWELL_BUILD_RESOURCES_SOURCES
     export CROMWELL_BUILD_ROOT_DIRECTORY
+    export CROMWELL_BUILD_RUN_TESTS
     export CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND
     export CROMWELL_BUILD_SCRIPTS_DIRECTORY
     export CROMWELL_BUILD_TAG
@@ -266,6 +257,100 @@ cromwell::private::echo_build_variables() {
     echo "CROMWELL_BUILD_URL='${CROMWELL_BUILD_URL}'"
 }
 
+cromwell::private::create_database_variables() {
+    case "${CROMWELL_BUILD_PROVIDER}" in
+        "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
+            CROMWELL_BUILD_MARIADB_HOSTNAME="localhost"
+            CROMWELL_BUILD_MARIADB_PORT="13306"
+            CROMWELL_BUILD_MARIADB_USERNAME="cromwell"
+            CROMWELL_BUILD_MARIADB_PASSWORD="test"
+            CROMWELL_BUILD_MARIADB_SCHEMA="cromwell_test"
+            CROMWELL_BUILD_MARIADB_DOCKER_TAG="${BUILD_MARIADB-}"
+            CROMWELL_BUILD_MYSQL_HOSTNAME="localhost"
+            CROMWELL_BUILD_MYSQL_PORT="3306"
+            CROMWELL_BUILD_MYSQL_USERNAME="cromwell"
+            CROMWELL_BUILD_MYSQL_PASSWORD="test"
+            CROMWELL_BUILD_MYSQL_SCHEMA="cromwell_test"
+            CROMWELL_BUILD_MYSQL_DOCKER_TAG="${BUILD_MYSQL-}"
+            CROMWELL_BUILD_POSTGRESQL_HOSTNAME="localhost"
+            CROMWELL_BUILD_POSTGRESQL_PORT="5432"
+            CROMWELL_BUILD_POSTGRESQL_USERNAME="cromwell"
+            CROMWELL_BUILD_POSTGRESQL_PASSWORD="test"
+            CROMWELL_BUILD_POSTGRESQL_SCHEMA="cromwell_test"
+            CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG="${BUILD_POSTGRESQL-}"
+            ;;
+        "${CROMWELL_BUILD_PROVIDER_JENKINS}")
+            # NOTE: Jenkins uses src/ci/docker-compose/docker-compose.yml.
+            # We don't define a docker tag because the docker-compose has already spun up the database containers by the
+            # time this script is run. Other variables here must match the database service names and settings the yaml.
+            CROMWELL_BUILD_MARIADB_DOCKER_TAG=""
+            CROMWELL_BUILD_MARIADB_HOSTNAME="mariadb-db"
+            CROMWELL_BUILD_MARIADB_PORT="3306"
+            CROMWELL_BUILD_MARIADB_USERNAME="cromwell"
+            CROMWELL_BUILD_MARIADB_PASSWORD="test"
+            CROMWELL_BUILD_MARIADB_SCHEMA="cromwell_test"
+            CROMWELL_BUILD_MYSQL_DOCKER_TAG=""
+            CROMWELL_BUILD_MYSQL_HOSTNAME="mysql-db"
+            CROMWELL_BUILD_MYSQL_PORT="3306"
+            CROMWELL_BUILD_MYSQL_USERNAME="cromwell"
+            CROMWELL_BUILD_MYSQL_PASSWORD="test"
+            CROMWELL_BUILD_MYSQL_SCHEMA="cromwell_test"
+            CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG=""
+            CROMWELL_BUILD_POSTGRESQL_HOSTNAME="postgresql-db"
+            CROMWELL_BUILD_POSTGRESQL_PORT="5432"
+            CROMWELL_BUILD_POSTGRESQL_USERNAME="cromwell"
+            CROMWELL_BUILD_POSTGRESQL_PASSWORD="test"
+            CROMWELL_BUILD_POSTGRESQL_SCHEMA="cromwell_test"
+            ;;
+        *)
+            CROMWELL_BUILD_MARIADB_HOSTNAME="${CROMWELL_BUILD_MARIADB_HOSTNAME-localhost}"
+            CROMWELL_BUILD_MARIADB_PORT="${CROMWELL_BUILD_MARIADB_PORT-13306}"
+            CROMWELL_BUILD_MARIADB_USERNAME="${CROMWELL_BUILD_MARIADB_USERNAME-cromwell}"
+            CROMWELL_BUILD_MARIADB_PASSWORD="${CROMWELL_BUILD_MARIADB_PASSWORD-test}"
+            CROMWELL_BUILD_MARIADB_SCHEMA="${CROMWELL_BUILD_MARIADB_SCHEMA-cromwell_test}"
+            CROMWELL_BUILD_MARIADB_DOCKER_TAG=""
+            CROMWELL_BUILD_MYSQL_HOSTNAME="${CROMWELL_BUILD_MYSQL_HOSTNAME-localhost}"
+            CROMWELL_BUILD_MYSQL_PORT="${CROMWELL_BUILD_MYSQL_PORT-3306}"
+            CROMWELL_BUILD_MYSQL_USERNAME="${CROMWELL_BUILD_MYSQL_USERNAME-cromwell}"
+            CROMWELL_BUILD_MYSQL_PASSWORD="${CROMWELL_BUILD_MYSQL_PASSWORD-test}"
+            CROMWELL_BUILD_MYSQL_SCHEMA="${CROMWELL_BUILD_MYSQL_SCHEMA-cromwell_test}"
+            CROMWELL_BUILD_MYSQL_DOCKER_TAG=""
+            CROMWELL_BUILD_POSTGRESQL_HOSTNAME="${CROMWELL_BUILD_POSTGRESQL_HOSTNAME-localhost}"
+            CROMWELL_BUILD_POSTGRESQL_PORT="${CROMWELL_BUILD_POSTGRESQL_PORT-5432}"
+            CROMWELL_BUILD_POSTGRESQL_USERNAME="${CROMWELL_BUILD_POSTGRESQL_USERNAME-cromwell}"
+            CROMWELL_BUILD_POSTGRESQL_PASSWORD="${CROMWELL_BUILD_POSTGRESQL_PASSWORD-test}"
+            CROMWELL_BUILD_POSTGRESQL_SCHEMA="${CROMWELL_BUILD_POSTGRESQL_SCHEMA-cromwell_test}"
+            CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG=""
+            ;;
+    esac
+
+    CROMWELL_BUILD_MARIADB_JDBC_URL="jdbc:mysql://${CROMWELL_BUILD_MARIADB_HOSTNAME}:${CROMWELL_BUILD_MARIADB_PORT}/${CROMWELL_BUILD_MARIADB_SCHEMA}?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    CROMWELL_BUILD_MYSQL_JDBC_URL="jdbc:mysql://${CROMWELL_BUILD_MYSQL_HOSTNAME}:${CROMWELL_BUILD_MYSQL_PORT}/${CROMWELL_BUILD_MYSQL_SCHEMA}?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    CROMWELL_BUILD_POSTGRESQL_JDBC_URL="jdbc:postgresql://${CROMWELL_BUILD_POSTGRESQL_HOSTNAME}:${CROMWELL_BUILD_POSTGRESQL_PORT}/${CROMWELL_BUILD_POSTGRESQL_SCHEMA}"
+
+    export CROMWELL_BUILD_MARIADB_DOCKER_TAG
+    export CROMWELL_BUILD_MARIADB_HOSTNAME
+    export CROMWELL_BUILD_MARIADB_JDBC_URL
+    export CROMWELL_BUILD_MARIADB_PASSWORD
+    export CROMWELL_BUILD_MARIADB_PORT
+    export CROMWELL_BUILD_MARIADB_SCHEMA
+    export CROMWELL_BUILD_MARIADB_USERNAME
+    export CROMWELL_BUILD_MYSQL_DOCKER_TAG
+    export CROMWELL_BUILD_MYSQL_HOSTNAME
+    export CROMWELL_BUILD_MYSQL_JDBC_URL
+    export CROMWELL_BUILD_MYSQL_PASSWORD
+    export CROMWELL_BUILD_MYSQL_PORT
+    export CROMWELL_BUILD_MYSQL_SCHEMA
+    export CROMWELL_BUILD_MYSQL_USERNAME
+    export CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG
+    export CROMWELL_BUILD_POSTGRESQL_HOSTNAME
+    export CROMWELL_BUILD_POSTGRESQL_JDBC_URL
+    export CROMWELL_BUILD_POSTGRESQL_PASSWORD
+    export CROMWELL_BUILD_POSTGRESQL_PORT
+    export CROMWELL_BUILD_POSTGRESQL_SCHEMA
+    export CROMWELL_BUILD_POSTGRESQL_USERNAME
+}
+
 cromwell::private::create_centaur_variables() {
     CROMWELL_BUILD_CENTAUR_TYPE_STANDARD="standard"
     CROMWELL_BUILD_CENTAUR_TYPE_INTEGRATION="integration"
@@ -310,6 +395,57 @@ cromwell::private::create_centaur_variables() {
     CROMWELL_BUILD_CENTAUR_TEST_RENDERED="${CROMWELL_BUILD_CENTAUR_TEST_DIRECTORY}/rendered"
     CROMWELL_BUILD_CENTAUR_LOG="${CROMWELL_BUILD_LOG_DIRECTORY}/centaur.log"
 
+    # Pick **one** of the databases to run Centaur against
+    case "${CROMWELL_BUILD_PROVIDER}" in
+        "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
+
+            if [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then
+                CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$"
+                CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="com.mysql.cj.jdbc.Driver"
+                CROMWELL_BUILD_CENTAUR_JDBC_USERNAME="${CROMWELL_BUILD_MYSQL_USERNAME}"
+                CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD="${CROMWELL_BUILD_MYSQL_PASSWORD}"
+                CROMWELL_BUILD_CENTAUR_JDBC_URL="${CROMWELL_BUILD_MYSQL_JDBC_URL}"
+
+            elif [[ -n "${CROMWELL_BUILD_MARIADB_DOCKER_TAG:+set}" ]]; then
+                CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$"
+                CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="com.mysql.cj.jdbc.Driver"
+                CROMWELL_BUILD_CENTAUR_JDBC_USERNAME="${CROMWELL_BUILD_MARIADB_USERNAME}"
+                CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD="${CROMWELL_BUILD_MARIADB_PASSWORD}"
+                CROMWELL_BUILD_CENTAUR_JDBC_URL="${CROMWELL_BUILD_MARIADB_JDBC_URL}"
+
+            elif [[ -n "${CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG:+set}" ]]; then
+                CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.PostgresProfile$"
+                CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="org.postgresql.Driver"
+                CROMWELL_BUILD_CENTAUR_JDBC_USERNAME="${CROMWELL_BUILD_POSTGRESQL_USERNAME}"
+                CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD="${CROMWELL_BUILD_POSTGRESQL_PASSWORD}"
+                CROMWELL_BUILD_CENTAUR_JDBC_URL="${CROMWELL_BUILD_POSTGRESQL_JDBC_URL}"
+
+            else
+                echo "Error: Unable to determine which RDBMS to use for Centaur." >&2
+                exit 1
+
+            fi
+
+            CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS=
+            ;;
+        "${CROMWELL_BUILD_PROVIDER_JENKINS}")
+            CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="slick.jdbc.MySQLProfile$"
+            CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="com.mysql.cj.jdbc.Driver"
+            CROMWELL_BUILD_CENTAUR_JDBC_USERNAME="${CROMWELL_BUILD_MYSQL_USERNAME}"
+            CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD="${CROMWELL_BUILD_MYSQL_PASSWORD}"
+            CROMWELL_BUILD_CENTAUR_JDBC_URL="${CROMWELL_BUILD_MYSQL_JDBC_URL}"
+            CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS="${CENTAUR_TEST_ADDITIONAL_PARAMETERS-}"
+            ;;
+        *)
+            CROMWELL_BUILD_CENTAUR_SLICK_PROFILE="${CROMWELL_BUILD_CENTAUR_SLICK_PROFILE-slick.jdbc.MySQLProfile\$}"
+            CROMWELL_BUILD_CENTAUR_JDBC_DRIVER="${CROMWELL_BUILD_CENTAUR_JDBC_DRIVER-com.mysql.cj.jdbc.Driver}"
+            CROMWELL_BUILD_CENTAUR_JDBC_USERNAME="${CROMWELL_BUILD_CENTAUR_JDBC_USERNAME-${CROMWELL_BUILD_MYSQL_USERNAME}}"
+            CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD="${CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD-${CROMWELL_BUILD_MYSQL_PASSWORD}}"
+            CROMWELL_BUILD_CENTAUR_JDBC_URL="${CROMWELL_BUILD_CENTAUR_JDBC_URL-${CROMWELL_BUILD_MYSQL_JDBC_URL}}"
+            CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS=
+            ;;
+    esac
+
     case "${CROMWELL_BUILD_CENTAUR_TYPE}" in
         "${CROMWELL_BUILD_CENTAUR_TYPE_INTEGRATION}")
             CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT=512000
@@ -319,39 +455,29 @@ cromwell::private::create_centaur_variables() {
             ;;
     esac
 
+    CROMWELL_BUILD_CENTAUR_256_BITS_KEY="$(dd bs=1 count=32 if=/dev/urandom 2> /dev/null | base64 | tr -d '\n')"
+
+    export CROMWELL_BUILD_CENTAUR_256_BITS_KEY
     export CROMWELL_BUILD_CENTAUR_CONFIG
+    export CROMWELL_BUILD_CENTAUR_JDBC_DRIVER
+    export CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD
+    export CROMWELL_BUILD_CENTAUR_JDBC_URL
+    export CROMWELL_BUILD_CENTAUR_JDBC_USERNAME
     export CROMWELL_BUILD_CENTAUR_LOG
+    export CROMWELL_BUILD_CENTAUR_TEST_ADDITIONAL_PARAMETERS
     export CROMWELL_BUILD_CENTAUR_TEST_DIRECTORY
     export CROMWELL_BUILD_CENTAUR_TEST_RENDERED
     export CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT
     export CROMWELL_BUILD_CENTAUR_RESOURCES
+    export CROMWELL_BUILD_CENTAUR_SLICK_PROFILE
     export CROMWELL_BUILD_CENTAUR_TYPE
     export CROMWELL_BUILD_CENTAUR_TYPE_STANDARD
     export CROMWELL_BUILD_CENTAUR_TYPE_INTEGRATION
     export CROMWELL_BUILD_CENTAUR_TYPE_ENGINE_UPGRADE
 }
 
-cromwell::private::verify_secure_build() {
-    case "${CROMWELL_BUILD_PROVIDER}" in
-        "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
-            if [[ "${CROMWELL_BUILD_IS_SECURE}" != "true" ]] && \
-                [[ "${CROMWELL_BUILD_REQUIRES_SECURE}" == "true" ]]; then
-                echo "********************************************************"
-                echo "********************************************************"
-                echo "**                                                    **"
-                echo "**  WARNING: Encrypted keys are unavailable. Exiting. **"
-                echo "**                                                    **"
-                echo "********************************************************"
-                echo "********************************************************"
-                exit 0
-            fi
-            ;;
-        *)
-            ;;
-    esac
-}
-
-cromwell::private::export_conformance_variables() {
+cromwell::private::create_conformance_variables() {
+    CROMWELL_BUILD_CWL_RUNNER_MODE="${CROMWELL_BUILD_BACKEND_TYPE}"
     CROMWELL_BUILD_CWL_TOOL_VERSION="1.0.20190228155703"
     CROMWELL_BUILD_CWL_TEST_VERSION="1.0.20190228134645"
     CROMWELL_BUILD_CWL_TEST_COMMIT="1f501e38ff692a408e16b246ac7d64d32f0822c2" # use known git hash to avoid changes
@@ -363,6 +489,7 @@ cromwell::private::export_conformance_variables() {
     CROMWELL_BUILD_CWL_TEST_OUTPUT="${CROMWELL_BUILD_LOG_DIRECTORY}/cwl_conformance_test.out.txt"
     CROMWELL_BUILD_CWL_TEST_PARALLELISM=10 # Set too high will cause false negatives due to cromwell server timeouts.
 
+    export CROMWELL_BUILD_CWL_RUNNER_MODE
     export CROMWELL_BUILD_CWL_TOOL_VERSION
     export CROMWELL_BUILD_CWL_TEST_VERSION
     export CROMWELL_BUILD_CWL_TEST_COMMIT
@@ -375,12 +502,58 @@ cromwell::private::export_conformance_variables() {
     export CROMWELL_BUILD_CWL_TEST_PARALLELISM
 }
 
+cromwell::private::verify_secure_build() {
+    case "${CROMWELL_BUILD_PROVIDER}" in
+        "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
+            if [[ "${CROMWELL_BUILD_IS_SECURE}" != "true" ]] && \
+                [[ "${CROMWELL_BUILD_REQUIRES_SECURE}" == "true" ]]; then
+                echo "********************************************************"
+                echo "********************************************************"
+                echo "**                                                    **"
+                echo "**  WARNING: Encrypted keys are unavailable. Exiting. **"
+                echo "**                                                    **"
+                echo "********************************************************"
+                echo "********************************************************"
+                exit 0
+            fi
+            ;;
+        *)
+            ;;
+    esac
+}
+
+cromwell::private::verify_pull_request_build() {
+    case "${CROMWELL_BUILD_PROVIDER}" in
+        "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
+            if [[ "${CROMWELL_BUILD_IS_PULL_REQUEST}" != "true" ]] && \
+                [[ "${CROMWELL_BUILD_REQUIRES_PULL_REQUEST}" == "true" ]]; then
+                echo "**************************************************"
+                echo "**************************************************"
+                echo "**                                              **"
+                echo "**  NOTE: Build is not a Pull Request. Exiting. **"
+                echo "**                                              **"
+                echo "**************************************************"
+                echo "**************************************************"
+                exit 0
+            fi
+            ;;
+        *)
+            ;;
+    esac
+}
+
 cromwell::private::exec_test_script() {
     local upper_build_type
     upper_build_type="$(tr '[:lower:]' '[:upper:]' <<< "${CROMWELL_BUILD_TYPE:0:1}")${CROMWELL_BUILD_TYPE:1}"
     exec "${CROMWELL_BUILD_SCRIPTS_DIRECTORY}/test${upper_build_type}.sh"
 }
 
+cromwell::private::stop_travis_defaults() {
+  # https://stackoverflow.com/questions/27382295/how-to-stop-services-on-travis-ci-running-by-default#answer-27410479
+  sudo /etc/init.d/mysql stop
+  sudo /etc/init.d/postgresql stop
+}
+
 cromwell::private::delete_boto_config() {
     # https://github.com/travis-ci/travis-ci/issues/7940#issuecomment-310759657
     sudo rm -f /etc/boto.cfg
@@ -417,18 +590,65 @@ cromwell::private::install_wait_for_it() {
     chmod +x "$CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT"
 }
 
-cromwell::private::create_mysql_cromwell_test() {
+cromwell::private::start_docker() {
+    local docker_image
+    local docker_cid_file
+    docker_image="${1:?foo called without a docker image}"; shift
+    docker_cid_file="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/$(echo "${docker_image}" | tr "/" "_" | tr ":" "-").cid.$$"
+
+    docker run --cidfile="${docker_cid_file}" --detach "$@" "${docker_image}"
+
+    cromwell::private::add_exit_function docker rm --force --volumes "$(cat "${docker_cid_file}")"
+    cromwell::private::add_exit_function rm "${docker_cid_file}"
+}
+
+cromwell::private::start_docker_mysql() {
     if cromwell::private::is_xtrace_enabled; then
-        cromwell::private::exec_silent_function cromwell::private::create_mysql_cromwell_test
-    else
-        "$CROMWELL_BUILD_WAIT_FOR_IT_SCRIPT" -t 120 "${CROMWELL_BUILD_MYSQL_HOSTNAME}:${CROMWELL_BUILD_MYSQL_PORT}"
-        mysql \
-            --host="${CROMWELL_BUILD_MYSQL_HOSTNAME}" \
-            --port="${CROMWELL_BUILD_MYSQL_PORT}" \
-            --user="${CROMWELL_BUILD_MYSQL_USERNAME}" \
-            --password="${CROMWELL_BUILD_MYSQL_PASSWORD}" \
-            --execute \
-            "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES'; CREATE DATABASE IF NOT EXISTS ${CROMWELL_BUILD_MYSQL_SCHEMA};"
+        cromwell::private::exec_silent_function cromwell::private::start_docker_mysql
+
+    elif [[ -n "${CROMWELL_BUILD_MYSQL_DOCKER_TAG:+set}" ]]; then
+        cromwell::private::start_docker \
+            mysql:"${CROMWELL_BUILD_MYSQL_DOCKER_TAG}" \
+            --env MYSQL_ROOT_PASSWORD=private \
+            --env MYSQL_USER="${CROMWELL_BUILD_MYSQL_USERNAME}" \
+            --env MYSQL_PASSWORD="${CROMWELL_BUILD_MYSQL_PASSWORD}" \
+            --env MYSQL_DATABASE="${CROMWELL_BUILD_MYSQL_SCHEMA}" \
+            --publish "${CROMWELL_BUILD_MYSQL_PORT}":3306 \
+            --volume "${CROMWELL_BUILD_DOCKER_DIRECTORY}"/mysql-conf.d:/etc/mysql/conf.d \
+
+    fi
+}
+
+cromwell::private::start_docker_mariadb() {
+    if cromwell::private::is_xtrace_enabled; then
+        cromwell::private::exec_silent_function cromwell::private::start_docker_mariadb
+
+    elif [[ -n "${CROMWELL_BUILD_MARIADB_DOCKER_TAG:+set}" ]]; then
+        cromwell::private::start_docker \
+            mariadb:"${CROMWELL_BUILD_MARIADB_DOCKER_TAG}" \
+            --env MYSQL_ROOT_PASSWORD=private \
+            --env MYSQL_USER="${CROMWELL_BUILD_MARIADB_USERNAME}" \
+            --env MYSQL_PASSWORD="${CROMWELL_BUILD_MARIADB_PASSWORD}" \
+            --env MYSQL_DATABASE="${CROMWELL_BUILD_MARIADB_SCHEMA}" \
+            --publish "${CROMWELL_BUILD_MARIADB_PORT}":3306 \
+            --volume "${CROMWELL_BUILD_DOCKER_DIRECTORY}"/mariadb-conf.d:/etc/mysql/conf.d \
+
+    fi
+}
+
+cromwell::private::start_docker_postgresql() {
+    if cromwell::private::is_xtrace_enabled; then
+        cromwell::private::exec_silent_function cromwell::private::start_docker_postgresql
+
+    elif [[ -n "${CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG:+set}" ]]; then
+        cromwell::private::start_docker \
+            postgres:"${CROMWELL_BUILD_POSTGRESQL_DOCKER_TAG}" \
+            --env POSTGRES_USER="${CROMWELL_BUILD_POSTGRESQL_USERNAME}" \
+            --env POSTGRES_PASSWORD="${CROMWELL_BUILD_POSTGRESQL_PASSWORD}" \
+            --env POSTGRES_DB="${CROMWELL_BUILD_POSTGRESQL_SCHEMA}" \
+            --publish "${CROMWELL_BUILD_POSTGRESQL_PORT}":5432 \
+            --volume "${CROMWELL_BUILD_DOCKER_DIRECTORY}"/postgresql-initdb.d:/docker-entrypoint-initdb.d \
+
     fi
 }
 
@@ -498,7 +718,7 @@ cromwell::private::vault_login() {
                 # shellcheck disable=SC2015
                 docker run --rm \
                     -v "${CROMWELL_BUILD_HOME_DIRECTORY}:/root:rw" \
-                    broadinstitute/dsde-toolbox \
+                    broadinstitute/dsde-toolbox:dev \
                     vault auth "${vault_token}" < /dev/null > /dev/null && echo vault auth success \
                 || true
                 ;;
@@ -576,7 +796,7 @@ cromwell::private::calculate_prior_version_tag() {
         )"
 
     # This function should only ever run on PR builds.
-    if [[ -z "${CROMWELL_BUILD_BRANCH_PULL_REQUEST}" ]]; then
+    if [[ -z "${CROMWELL_BUILD_BRANCH_PULL_REQUEST-}" ]]; then
        echo "Error: the CROMWELL_BUILD_BRANCH_PULL_REQUEST variable is not set. calculate_prior_version_tag expects to only run on Travis Pull Request builds in which this variable is set." >&2
        exit 1
     fi
@@ -626,7 +846,8 @@ cromwell::private::exists_cromwell_jar() {
 cromwell::private::assemble_jars() {
     # CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND allows for an override of the default `assembly` command for assembly.
     # This can be useful to reduce time and memory that might otherwise be spent assembling unused subprojects.
-    CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=error sbt coverage ${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND:-assembly} -error
+    # shellcheck disable=SC2086
+    CROMWELL_SBT_ASSEMBLY_LOG_LEVEL=error sbt coverage ${CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND} -error
 }
 
 cromwell::private::generate_code_coverage() {
@@ -737,11 +958,13 @@ cromwell::private::run_exit_functions() {
 }
 
 # Adds the function to the list of functions to run on exit.
-# Requires one positional parameter, the function to run.
+# Requires at least one positional parameter, the function to run.
 cromwell::private::add_exit_function() {
-    local exit_function
-    exit_function="${1:?add_exit_function called without a function}"; shift
-    echo "${exit_function}" >> "${CROMWELL_BUILD_EXIT_FUNCTIONS}"
+    if [[ "$#" -eq 0 ]]; then
+        echo "Error: add_exit_function called without a function" >&2
+        exit 1
+    fi
+    echo "$@" >> "${CROMWELL_BUILD_EXIT_FUNCTIONS}"
     trap cromwell::private::run_exit_functions TERM EXIT
 }
 
@@ -823,25 +1046,29 @@ cromwell::build::setup_common_environment() {
     cromwell::private::check_debug
     cromwell::private::create_build_variables
     cromwell::private::echo_build_variables
+    cromwell::private::create_database_variables
     cromwell::private::verify_secure_build
+    cromwell::private::verify_pull_request_build
     cromwell::private::make_build_directories
     cromwell::private::setup_secure_resources
 
     case "${CROMWELL_BUILD_PROVIDER}" in
         "${CROMWELL_BUILD_PROVIDER_TRAVIS}")
+            cromwell::private::stop_travis_defaults
             cromwell::private::delete_boto_config
             cromwell::private::delete_sbt_boot
             cromwell::private::upgrade_pip
             cromwell::private::pull_common_docker_images
             cromwell::private::install_wait_for_it
-            cromwell::private::create_mysql_cromwell_test
+            cromwell::private::start_docker_mysql
+            cromwell::private::start_docker_mariadb
+            cromwell::private::start_docker_postgresql
             ;;
         "${CROMWELL_BUILD_PROVIDER_JENKINS}")
             cromwell::private::delete_boto_config
             cromwell::private::delete_sbt_boot
             cromwell::private::upgrade_pip
             cromwell::private::install_wait_for_it
-            cromwell::private::create_mysql_cromwell_test
             ;;
         *)
             cromwell::private::pull_common_docker_images
@@ -851,14 +1078,10 @@ cromwell::build::setup_common_environment() {
 
 cromwell::build::setup_centaur_environment() {
     cromwell::private::create_centaur_variables
-    if [[ "${CROMWELL_BUILD_CENTAUR_TYPE}" == "${CROMWELL_BUILD_CENTAUR_TYPE_ENGINE_UPGRADE}" ]]; then
-        cromwell::private::setup_prior_version_resources
-    elif [[ "${CROMWELL_BUILD_CENTAUR_TYPE}" == "${CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE}" ]]; then
+    if [[ "${CROMWELL_BUILD_CENTAUR_TYPE}" == "${CROMWELL_BUILD_CENTAUR_TYPE_ENGINE_UPGRADE}" ]] || \
+        [[ "${CROMWELL_BUILD_CENTAUR_TYPE}" == "${CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE}" ]]; then
         cromwell::private::setup_prior_version_resources
-        export CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v1_v2_upgrade_application.conf"
-    elif [[ "${CROMWELL_BUILD_CENTAUR_TYPE}" == "${CROMWELL_BUILD_CENTAUR_TYPE_PAPI_UPGRADE_NEW_WORKFLOWS}" ]]; then
-        export CROMWELL_BUILD_CROMWELL_CONFIG="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_v1_v2_upgrade_application.conf"
-    fi;
+    fi
     cromwell::private::start_build_heartbeat
     cromwell::private::start_cromwell_log_tail
     cromwell::private::start_centaur_log_tail
@@ -871,11 +1094,8 @@ cromwell::build::setup_centaur_environment() {
 }
 
 cromwell::build::setup_conformance_environment() {
-    # Override of the default sbt assembly command which is just `assembly`.
-    # The conformance runs only need these two subprojects so save a couple of minutes and skip the rest.
-    export CROMWELL_BUILD_SBT_ASSEMBLY_COMMAND="server/assembly centaurCwlRunner/assembly"
-
-    cromwell::private::export_conformance_variables
+    cromwell::private::create_centaur_variables
+    cromwell::private::create_conformance_variables
     if [[ "${CROMWELL_BUILD_IS_CI}" == "true" ]]; then
         cromwell::private::install_cwltest
     fi
@@ -936,10 +1156,6 @@ cromwell::build::run_centaur() {
 }
 
 cromwell::build::run_conformance() {
-    CROMWELL_BUILD_CWL_RUNNER_MODE="${CROMWELL_BUILD_BACKEND_TYPE}"
-
-    export CROMWELL_BUILD_CWL_RUNNER_MODE
-
     cromwell::private::start_conformance_cromwell
     cromwell::private::add_exit_function cromwell::private::kill_conformance_cromwell
 
diff --git a/src/ci/bin/testCentaurEngineUpgradeLocal.sh b/src/ci/bin/testCentaurEngineUpgradeLocal.sh
index 8bffc195fb2..9da358220df 100755
--- a/src/ci/bin/testCentaurEngineUpgradeLocal.sh
+++ b/src/ci/bin/testCentaurEngineUpgradeLocal.sh
@@ -2,19 +2,16 @@
 
 set -o errexit -o nounset -o pipefail
 export CROMWELL_BUILD_OPTIONAL_SECURE=true
+export CROMWELL_BUILD_REQUIRES_PULL_REQUEST=true
 # import in shellcheck / CI / IntelliJ compatible ways
 # shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 
-if [ "${CROMWELL_BUILD_PROVIDER}" = "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ] && [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
+cromwell::build::setup_common_environment
 
-  cromwell::build::setup_common_environment
+cromwell::build::setup_centaur_environment
 
-  cromwell::build::setup_centaur_environment
+cromwell::build::assemble_jars
 
-  cromwell::build::assemble_jars
-
-  cromwell::build::run_centaur \
-      -s "centaur.EngineUpgradeTestCaseSpec"
-
-fi
+cromwell::build::run_centaur \
+  -s "centaur.EngineUpgradeTestCaseSpec"
diff --git a/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2.sh b/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2.sh
index d21d2c2c992..09a2c0010db 100755
--- a/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2.sh
+++ b/src/ci/bin/testCentaurHoricromtalEngineUpgradePapiV2.sh
@@ -2,51 +2,48 @@
 
 set -o errexit -o nounset -o pipefail
 export CROMWELL_BUILD_REQUIRES_SECURE=true
+export CROMWELL_BUILD_REQUIRES_PULL_REQUEST=true
 # import in shellcheck / CI / IntelliJ compatible ways
 # shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 
-if [ "${CROMWELL_BUILD_PROVIDER}" = "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ] && [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
-  cromwell::build::setup_common_environment
+cromwell::build::setup_common_environment
 
-  prior_version=$(cromwell::private::calculate_prior_version_tag)
-  export TEST_CROMWELL_PRIOR_VERSION_TAG="${prior_version}"
-  WOULD_BE_PRIOR_VERSION_CONF="papi_v2_${prior_version}_application.conf"
-  if [[ -f "$CROMWELL_BUILD_RESOURCES_DIRECTORY/$WOULD_BE_PRIOR_VERSION_CONF" ]]; then
-    export TEST_CROMWELL_PRIOR_VERSION_CONF="$WOULD_BE_PRIOR_VERSION_CONF"
-  else
-    export TEST_CROMWELL_PRIOR_VERSION_CONF="papi_v2_application.conf"
-  fi
-  # This is the Docker tag that will be applied to the Docker image that is created for the code being built. This image
-  # will *not* be pushed to Docker Hub or any other repo, it only lives local to the build.
-  export TEST_CROMWELL_TAG=just-testing-horicromtal
-  export TEST_CROMWELL_CONF="papi_v2_application.conf"
-  export CROMWELL_BUILD_MYSQL_USERNAME=travis
-
-  cromwell::build::setup_centaur_environment
-
-  cromwell::build::assemble_jars
-
-  GOOGLE_AUTH_MODE="service-account"
-  GOOGLE_REFRESH_TOKEN_PATH="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_refresh_token.txt"
+cromwell::build::setup_centaur_environment
 
-  # Export variables used in conf files
-  export GOOGLE_AUTH_MODE
-  export GOOGLE_REFRESH_TOKEN_PATH
-  export TEST_CROMWELL_COMPOSE_FILE="${CROMWELL_BUILD_ROOT_DIRECTORY}/scripts/docker-compose-mysql/docker-compose-horicromtal.yml"
-
-  # Copy rendered files
-  mkdir -p "${CROMWELL_BUILD_CENTAUR_TEST_RENDERED}"
-  cp \
-      "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/private_docker_papi_v2_usa.options" \
-      "${TEST_CROMWELL_COMPOSE_FILE}" \
-      "${CROMWELL_BUILD_CENTAUR_TEST_RENDERED}"
-
-  docker image ls -q broadinstitute/cromwell:"${TEST_CROMWELL_TAG}" | grep . || \
-  CROMWELL_SBT_DOCKER_TAGS="${TEST_CROMWELL_TAG}" sbt server/docker
-
-  cromwell::build::run_centaur \
-      -s "centaur.EngineUpgradeTestCaseSpec" \
-      -e localdockertest
+cromwell::build::assemble_jars
 
+prior_version=$(cromwell::private::calculate_prior_version_tag)
+export TEST_CROMWELL_PRIOR_VERSION_TAG="${prior_version}"
+WOULD_BE_PRIOR_VERSION_CONF="papi_v2_${prior_version}_application.conf"
+if [[ -f "$CROMWELL_BUILD_RESOURCES_DIRECTORY/$WOULD_BE_PRIOR_VERSION_CONF" ]]; then
+    export TEST_CROMWELL_PRIOR_VERSION_CONF="$WOULD_BE_PRIOR_VERSION_CONF"
+else
+    export TEST_CROMWELL_PRIOR_VERSION_CONF="papi_v2_application.conf"
 fi
+# This is the Docker tag that will be applied to the Docker image that is created for the code being built. This image
+# will *not* be pushed to Docker Hub or any other repo, it only lives local to the build.
+export TEST_CROMWELL_TAG=just-testing-horicromtal
+export TEST_CROMWELL_CONF="papi_v2_application.conf"
+
+GOOGLE_AUTH_MODE="service-account"
+GOOGLE_REFRESH_TOKEN_PATH="${CROMWELL_BUILD_RESOURCES_DIRECTORY}/papi_refresh_token.txt"
+
+# Export variables used in conf files
+export GOOGLE_AUTH_MODE
+export GOOGLE_REFRESH_TOKEN_PATH
+export TEST_CROMWELL_COMPOSE_FILE="${CROMWELL_BUILD_ROOT_DIRECTORY}/scripts/docker-compose-mysql/docker-compose-horicromtal.yml"
+
+# Copy rendered files
+mkdir -p "${CROMWELL_BUILD_CENTAUR_TEST_RENDERED}"
+cp \
+    "${CROMWELL_BUILD_RESOURCES_DIRECTORY}/private_docker_papi_v2_usa.options" \
+    "${TEST_CROMWELL_COMPOSE_FILE}" \
+    "${CROMWELL_BUILD_CENTAUR_TEST_RENDERED}"
+
+docker image ls -q broadinstitute/cromwell:"${TEST_CROMWELL_TAG}" | grep . || \
+CROMWELL_SBT_DOCKER_TAGS="${TEST_CROMWELL_TAG}" sbt server/docker
+
+cromwell::build::run_centaur \
+    -s "centaur.EngineUpgradeTestCaseSpec" \
+    -e localdockertest
diff --git a/src/ci/bin/testCentaurHoricromtalPapiV2.sh b/src/ci/bin/testCentaurHoricromtalPapiV2.sh
index 3607e599184..a386a3739ad 100755
--- a/src/ci/bin/testCentaurHoricromtalPapiV2.sh
+++ b/src/ci/bin/testCentaurHoricromtalPapiV2.sh
@@ -10,7 +10,6 @@ source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 # There should probably be more indirections in CI scripts but that can wait.
 export TEST_CROMWELL_TAG=just-testing-horicromtal
 export TEST_CROMWELL_CONF=papi_v2_horicromtal_application.conf
-export CROMWELL_BUILD_MYSQL_USERNAME=travis
 
 cromwell::build::setup_common_environment
 
diff --git a/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV1.sh b/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV1.sh
index e04fd68de81..ae1ce769ef9 100755
--- a/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV1.sh
+++ b/src/ci/bin/testCentaurPapiUpgradeNewWorkflowsPapiV1.sh
@@ -2,21 +2,18 @@
 
 set -o errexit -o nounset -o pipefail
 export CROMWELL_BUILD_REQUIRES_SECURE=true
+export CROMWELL_BUILD_REQUIRES_PULL_REQUEST=true
 # import in shellcheck / CI / IntelliJ compatible ways
 # shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 
-if [ "${CROMWELL_BUILD_PROVIDER}" = "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ] && [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
+cromwell::build::setup_common_environment
 
-  cromwell::build::setup_common_environment
+cromwell::build::setup_centaur_environment
 
-  cromwell::build::setup_centaur_environment
+cromwell::build::assemble_jars
 
-  cromwell::build::assemble_jars
-
-  cromwell::build::run_centaur \
-      -p 100 \
-      -e localdockertest \
-      -e gpu_on_papi \
-
-fi
+cromwell::build::run_centaur \
+  -p 100 \
+  -e localdockertest \
+  -e gpu_on_papi \
diff --git a/src/ci/bin/testCentaurPapiUpgradePapiV1.sh b/src/ci/bin/testCentaurPapiUpgradePapiV1.sh
index b6a83f51755..bfe2ec5101a 100755
--- a/src/ci/bin/testCentaurPapiUpgradePapiV1.sh
+++ b/src/ci/bin/testCentaurPapiUpgradePapiV1.sh
@@ -2,20 +2,17 @@
 
 set -o errexit -o nounset -o pipefail
 export CROMWELL_BUILD_REQUIRES_SECURE=true
+export CROMWELL_BUILD_REQUIRES_PULL_REQUEST=true
 # import in shellcheck / CI / IntelliJ compatible ways
 # shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 
-if [ "${CROMWELL_BUILD_PROVIDER}" = "${CROMWELL_BUILD_PROVIDER_TRAVIS}" ] && [ -n "${TRAVIS_PULL_REQUEST_BRANCH}" ]; then
+cromwell::build::setup_common_environment
 
-  cromwell::build::setup_common_environment
+cromwell::build::setup_centaur_environment
 
-  cromwell::build::setup_centaur_environment
+cromwell::build::assemble_jars
 
-  cromwell::build::assemble_jars
-
-  cromwell::build::run_centaur \
-      -s "centaur.PapiUpgradeTestCaseSpec" \
-      -e localdockertest \
-
-fi
+cromwell::build::run_centaur \
+  -s "centaur.PapiUpgradeTestCaseSpec" \
+  -e localdockertest \
diff --git a/src/ci/docker-compose/cromwell-test/docker-setup.sh b/src/ci/docker-compose/cromwell-test/docker-setup.sh
index 03b1cacf16e..fb13b6bd714 100755
--- a/src/ci/docker-compose/cromwell-test/docker-setup.sh
+++ b/src/ci/docker-compose/cromwell-test/docker-setup.sh
@@ -13,8 +13,9 @@ apt-get install -y \
     curl \
     gnupg \
     gnupg2 \
-    python-dev \
     mysql-client \
+    postgresql-client \
+    python-dev \
     software-properties-common \
     sudo \
 
diff --git a/src/ci/docker-compose/docker-compose.yml b/src/ci/docker-compose/docker-compose.yml
index a13448367e2..14c7b07afd5 100644
--- a/src/ci/docker-compose/docker-compose.yml
+++ b/src/ci/docker-compose/docker-compose.yml
@@ -19,9 +19,31 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     links:
       - mysql-db
+      - mariadb-db
+      - postgresql-db
   mysql-db:
     image: "mysql:5.7"
     environment:
-      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
-      - MYSQL_ROOT_PASSWORD=
+      - MYSQL_ROOT_PASSWORD=private
+      - MYSQL_USER=cromwell
+      - MYSQL_PASSWORD=test
       - MYSQL_DATABASE=cromwell_test
+    volumes:
+      - ./mysql-conf.d:/etc/mysql/conf.d
+  mariadb-db:
+    image: "mariadb:10.3"
+    environment:
+      - MYSQL_ROOT_PASSWORD=private
+      - MYSQL_USER=cromwell
+      - MYSQL_PASSWORD=test
+      - MYSQL_DATABASE=cromwell_test
+    volumes:
+      - ./mariadb-conf.d:/etc/mysql/conf.d
+  postgresql-db:
+    image: "postgres:11.3"
+    environment:
+      - POSTGRES_USER=cromwell
+      - POSTGRES_PASSWORD=test
+      - POSTGRES_DB=cromwell_test
+    volumes:
+      - ./postgresql-initdb.d:/docker-entrypoint-initdb.d
diff --git a/src/ci/docker-compose/mariadb-conf.d/init.cnf b/src/ci/docker-compose/mariadb-conf.d/init.cnf
new file mode 100644
index 00000000000..ae32e540b7f
--- /dev/null
+++ b/src/ci/docker-compose/mariadb-conf.d/init.cnf
@@ -0,0 +1,3 @@
+[mysqld]
+max_connections=300
+sql_mode=STRICT_ALL_TABLES
diff --git a/src/ci/docker-compose/mysql-conf.d/init.cnf b/src/ci/docker-compose/mysql-conf.d/init.cnf
new file mode 100644
index 00000000000..ae32e540b7f
--- /dev/null
+++ b/src/ci/docker-compose/mysql-conf.d/init.cnf
@@ -0,0 +1,3 @@
+[mysqld]
+max_connections=300
+sql_mode=STRICT_ALL_TABLES
diff --git a/src/ci/docker-compose/postgresql-initdb.d/init.sh b/src/ci/docker-compose/postgresql-initdb.d/init.sh
new file mode 100755
index 00000000000..f861d9c4551
--- /dev/null
+++ b/src/ci/docker-compose/postgresql-initdb.d/init.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+cat << EOF >> /var/lib/postgresql/data/postgresql.conf
+max_connections = 300
+EOF
diff --git a/src/ci/docker-compose/postgresql-initdb.d/init.sql b/src/ci/docker-compose/postgresql-initdb.d/init.sql
new file mode 100644
index 00000000000..8cff561ceee
--- /dev/null
+++ b/src/ci/docker-compose/postgresql-initdb.d/init.sql
@@ -0,0 +1 @@
+create extension lo;
diff --git a/src/ci/resources/build_application.inc.conf b/src/ci/resources/build_application.inc.conf
index 9dad320b135..fc87166fc11 100644
--- a/src/ci/resources/build_application.inc.conf
+++ b/src/ci/resources/build_application.inc.conf
@@ -2,7 +2,7 @@ akka.http.host-connection-pool.max-open-requests = 1024
 
 workflow-options {
   base64-encryption-key = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
-  base64-encryption-key = ${?CROMWELL_BUILD_RANDOM_256_BITS_BASE64}
+  base64-encryption-key = ${?CROMWELL_BUILD_CENTAUR_256_BITS_KEY}
   encrypted-fields = [
     "docker_credentials_key_name",
     "docker_credentials_token",
diff --git a/src/ci/resources/cromwell_database.inc.conf b/src/ci/resources/cromwell_database.inc.conf
index c726df7f091..4c93bb2635b 100644
--- a/src/ci/resources/cromwell_database.inc.conf
+++ b/src/ci/resources/cromwell_database.inc.conf
@@ -1,17 +1,14 @@
 database {
+  profile = "slick.jdbc.MySQLProfile$"
+  profile = ${?CROMWELL_BUILD_CENTAUR_SLICK_PROFILE}
   db {
-    hostname = localhost
-    hostname = ${?CROMWELL_BUILD_MYSQL_HOSTNAME}
-    port = 3306
-    port = ${?CROMWELL_BUILD_MYSQL_PORT}
-    schema = cromwell_test
-    schema = ${?CROMWELL_BUILD_MYSQL_SCHEMA}
-    url = "jdbc:mysql://"${database.db.hostname}":"${database.db.port}"/"${database.db.schema}"?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
-    user = root
-    user = ${?CROMWELL_BUILD_MYSQL_USERNAME}
-    password = ""
-    password = ${?CROMWELL_BUILD_MYSQL_PASSWORD}
     driver = "com.mysql.cj.jdbc.Driver"
+    driver = ${?CROMWELL_BUILD_CENTAUR_JDBC_DRIVER}
+    url = "jdbc:mysql://localhost:3306/cromwell_test?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    url = ${?CROMWELL_BUILD_CENTAUR_JDBC_URL}
+    user = "cromwell"
+    user = ${?CROMWELL_BUILD_CENTAUR_JDBC_USERNAME}
+    password = "test"
+    password = ${?CROMWELL_BUILD_CENTAUR_JDBC_PASSWORD}
   }
-  profile = "slick.jdbc.MySQLProfile$"
 }

From 49d111ac4c77616182bb704bc049d9bec6cce725 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Thu, 13 Jun 2019 15:53:29 -0400
Subject: [PATCH 11/28] Fixup perf configuration around statsd instrumentation
 (#5024)

---
 scripts/perf/vm_scripts/cromwell/cromwell.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/perf/vm_scripts/cromwell/cromwell.conf b/scripts/perf/vm_scripts/cromwell/cromwell.conf
index 5c1df82c341..1747f25ed5d 100644
--- a/scripts/perf/vm_scripts/cromwell/cromwell.conf
+++ b/scripts/perf/vm_scripts/cromwell/cromwell.conf
@@ -72,7 +72,7 @@ services {
   Instrumentation {
     # StatsD - Send metrics to a StatsD server over UDP
     class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
-    config.statsd {
+    config {
       hostname = statsd-proxy
       port = 9125
       prefix = ${?CROMWELL_STATSD_PREFIX} # can be used to prefix all metrics with an api key for example

From 05c9dbfd9b03410640082e2da6718605e5e02030 Mon Sep 17 00:00:00 2001
From: Ruchi <rmunshi@broadinstitute.org>
Date: Fri, 14 Jun 2019 08:32:12 -0400
Subject: [PATCH 12/28] added server logging for error code 10 [BA-5703]
 (#5030)

* added server logging for error code 10
---
 .../PipelinesApiAsyncBackendJobExecutionActor.scala   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala
index a2051b98e38..01fa8aebcc0 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiAsyncBackendJobExecutionActor.scala
@@ -66,6 +66,7 @@ object PipelinesApiAsyncBackendJobExecutionActor {
   val JesPreemption = 14
 
   val PapiFailedPreConditionErrorCode = 9
+  val PapiMysteriouslyCrashedErrorCode = 10
 
   // If the JES code is 2 (UNKNOWN), this sub-string indicates preemption:
   val FailedToStartDueToPreemptionSubstring = "failed to start due to preemption"
@@ -409,7 +410,7 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta
            * In CWL-land we tend to be aggressive in pre-fetching the size in order to be able to evaluate JS expressions,
            * but less in WDL as we can get it last minute and on demand because size is a WDL function, whereas in CWL
            * we don't inspect the JS to know if size is called and therefore always pre-fetch it.
-           * 
+           *
            * We could decide to call withSize before in which case we would retrieve the size for all files and have
            * a guaranteed more accurate total size, but there might be performance impacts ?
            */
@@ -602,7 +603,13 @@ class PipelinesApiAsyncBackendJobExecutionActor(override val standardParams: Sta
           && errorMsg.contains("Execution failed")
           && (errorMsg.contains("Localization") || errorMsg.contains("Delocalization"))) {
         s"Please check the log file for more details: $jesLogPath."
-      } else errorMsg
+      }
+      //If error code 10, add some extra messaging to the server logging
+      else if (runStatus.errorCode.getCode.value == PapiMysteriouslyCrashedErrorCode) {
+        jobLogger.info(s"Job Failed with Error Code 10 for a machine where Preemptible is set to $preemptible")
+        errorMsg
+      }
+      else errorMsg
     }
 
     // Inner function: Handles a 'Failed' runStatus (or Preempted if preemptible was false)

From 8bf12a8b88f0ae776e9301e84613818597711b1a Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Fri, 14 Jun 2019 10:25:31 -0400
Subject: [PATCH 13/28] Allow includeKey and excludeKey non exclusively
 [BA-5671] (#5021)

---
 CHANGELOG.md                                  | 11 ++-
 .../slick/MetadataSlickDatabase.scala         | 28 ++-----
 .../slick/tables/MetadataEntryComponent.scala | 73 ++++++++-----------
 .../database/sql/MetadataSqlDatabase.scala    | 11 +--
 docs/api/RESTAPI.md                           |  6 +-
 .../src/main/resources/swagger/cromwell.yaml  |  9 ++-
 .../routes/MetadataRouteSupport.scala         | 13 +---
 .../routes/CromwellApiServiceSpec.scala       | 36 +++++----
 .../routes/MetadataRouteSupportSpec.scala     | 39 ++++------
 .../services/metadata/MetadataService.scala   |  7 +-
 .../impl/MetadataDatabaseAccess.scala         | 33 ++++-----
 .../metadata/impl/ReadMetadataActor.scala     |  2 +-
 .../impl/WriteMetadataActorSpec.scala         | 34 +++------
 13 files changed, 132 insertions(+), 170 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73ea7d40e65..4420f5a1dd0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,12 +7,21 @@
 Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). 
 Learn more on how to configure [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
 
-### StatsD Instrumentation Config change
+### Configuration Changes
+
+#### StatsD Instrumentation
 
 There is a small change in StatsD's configuration path. Originally, the path to the config was `services.Instrumentation.config.statsd`
 which now has been updated to `services.Instrumentation.config`. More info on its configuration can be found
 [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
 
+### API Changes
+
+#### Workflow Metadata
+
+* It is now possible to use `includeKey` and `excludeKey` at the same time. If so, the metadata key must match the `includeKey` **and not** match the `excludeKey` to be included.
+* It is now possible to use "`calls`" as one of your `excludeKey`s, to request that only workflow metadata gets returned.
+
 ## 42 Release Notes
 
 ### Womtool endpoint
diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index 91dbc44c3a0..2bb2f46ff8f 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -2,7 +2,6 @@ package cromwell.database.slick
 
 import java.sql.Timestamp
 
-import cats.data.NonEmptyList
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.database.slick.tables.MetadataDataAccessComponent
 import cromwell.database.sql.MetadataSqlDatabase
@@ -83,29 +82,18 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
     runTransaction(action)
   }
 
-  override def queryMetadataEntriesLikeMetadataKeys(workflowExecutionUuid: String,
-                                                    metadataKeys: NonEmptyList[String],
+  override def queryMetadataEntryWithKeyConstraints(workflowExecutionUuid: String,
+                                                    metadataKeysToFilterFor: List[String],
+                                                    metadataKeysToFilterOut: List[String],
                                                     metadataJobQueryValue: MetadataJobQueryValue)
                                                    (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
     val action = metadataJobQueryValue match {
       case CallQuery(callFqn, jobIndex, jobAttempt) =>
-        dataAccess.metadataEntriesLikeMetadataKeysWithJob(workflowExecutionUuid, metadataKeys, callFqn, jobIndex, jobAttempt).result
-      case WorkflowQuery => dataAccess.metadataEntriesLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = true).result
-      case CallOrWorkflowQuery => dataAccess.metadataEntriesLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = false).result
-    }
-
-    runTransaction(action)
-  }
-
-  override def queryMetadataEntryNotLikeMetadataKeys(workflowExecutionUuid: String,
-                                                     metadataKeys: NonEmptyList[String],
-                                                     metadataJobQueryValue: MetadataJobQueryValue)
-                                                    (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]] = {
-    val action = metadataJobQueryValue match {
-      case CallQuery(callFqn, jobIndex, jobAttempt) =>
-        dataAccess.metadataEntriesNotLikeMetadataKeysWithJob(workflowExecutionUuid, metadataKeys, callFqn, jobIndex, jobAttempt).result
-      case WorkflowQuery => dataAccess.metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = true).result
-      case CallOrWorkflowQuery => dataAccess.metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid, metadataKeys, requireEmptyJobKey = false).result
+        dataAccess.metadataEntriesForJobWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, callFqn, jobIndex, jobAttempt).result
+      case WorkflowQuery =>
+        dataAccess.metadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = true).result
+      case CallOrWorkflowQuery =>
+        dataAccess.metadataEntriesWithKeyConstraints(workflowExecutionUuid, metadataKeysToFilterFor, metadataKeysToFilterOut, requireEmptyJobKey = false).result
     }
     runTransaction(action)
   }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
index 711cde84f56..9e8106aebd4 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
@@ -2,7 +2,6 @@ package cromwell.database.slick.tables
 
 import java.sql.{Clob, Timestamp}
 
-import cats.data.NonEmptyList
 import cromwell.database.sql.tables.MetadataEntry
 
 trait MetadataEntryComponent {
@@ -152,12 +151,14 @@ trait MetadataEntryComponent {
     * If requireEmptyJobKey is true, only workflow level keys are returned, otherwise both workflow and call level
     * keys are returned.
     */
-  def metadataEntriesLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
-                                      requireEmptyJobKey: Boolean) = {
+  def metadataEntriesWithKeyConstraints(workflowExecutionUuid: String,
+                                        metadataKeysToFilterFor: List[String],
+                                        metadataKeysToFilterOut: List[String],
+                                        requireEmptyJobKey: Boolean) = {
     (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
-      if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
+      if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeysToFilterFor, metadataKeysToFilterOut)
       if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey)
     } yield metadataEntry).sortBy(_.metadataTimestamp)
   }
@@ -166,59 +167,43 @@ trait MetadataEntryComponent {
     * Returns metadata entries that are "like" metadataKeys for the specified call.
     * If jobAttempt has no value, all metadata keys for all attempts are returned.
     */
-  def metadataEntriesLikeMetadataKeysWithJob(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
-                                             callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int]) = {
+  def metadataEntriesForJobWithKeyConstraints(workflowExecutionUuid: String,
+                                              metadataKeysToFilterFor: List[String],
+                                              metadataKeysToFilterOut: List[String],
+                                              callFqn: String,
+                                              jobIndex: Option[Int],
+                                              jobAttempt: Option[Int]) = {
     (for {
       metadataEntry <- metadataEntries
       if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
-      if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
+      if metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeysToFilterFor, metadataKeysToFilterOut)
       if metadataEntry.callFullyQualifiedName === callFqn
       if hasSameIndex(metadataEntry, jobIndex)
       // Assume that every metadata entry for a call should have a non null attempt value
-      // Because of that, if the jobAttempt paramater is Some(_), make sure it matches, otherwise take all entries
+      // Because of that, if the jobAttempt parameter is Some(_), make sure it matches, otherwise take all entries
       // regardless of the attempt
       if (metadataEntry.jobAttempt === jobAttempt) || jobAttempt.isEmpty
     } yield metadataEntry).sortBy(_.metadataTimestamp)
   }
 
-  /**
-    * Returns metadata entries that are NOT "like" metadataKeys for the specified workflow.
-    * If requireEmptyJobKey is true, only workflow level keys are returned, otherwise both workflow and call level
-    * keys are returned.
-    */
-  def metadataEntriesNotLikeMetadataKeys(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
-                                         requireEmptyJobKey: Boolean) = {
-    (for {
-      metadataEntry <- metadataEntries
-      if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
-      if !metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
-      if metadataEntryHasEmptyJobKey(metadataEntry, requireEmptyJobKey)
-    } yield metadataEntry).sortBy(_.metadataTimestamp)
-  }
+  private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries,
+                                                     metadataKeysToFilterFor: List[String],
+                                                     metadataKeysToFilterOut: List[String]): Rep[Boolean] = {
 
-  /**
-    * Returns metadata entries that are NOT "like" metadataKeys for the specified call.
-    * If jobIndex (resp. jobAttempt) has no value, all metadata keys for all indices (resp. attempt)
-    * are returned.
-    */
-  def metadataEntriesNotLikeMetadataKeysWithJob(workflowExecutionUuid: String, metadataKeys: NonEmptyList[String],
-                                                callFqn: String, jobIndex: Option[Int], jobAttempt: Option[Int]) = {
-    (for {
-      metadataEntry <- metadataEntries
-      if metadataEntry.workflowExecutionUuid === workflowExecutionUuid
-      if !metadataEntryHasMetadataKeysLike(metadataEntry, metadataKeys)
-      if metadataEntry.callFullyQualifiedName === callFqn
-      if hasSameIndex(metadataEntry, jobIndex)
-        // Assume that every metadata entry for a call should have a non null attempt value
-        // Because of that, if the jobAttempt parameter is Some(_), make sure it matches, otherwise take all entries
-        // regardless of the attempt
-      if (metadataEntry.jobAttempt === jobAttempt) || jobAttempt.isEmpty
-    } yield metadataEntry).sortBy(_.metadataTimestamp)
-  }
+    def containsKey(key: String): Rep[Boolean] = metadataEntry.metadataKey like key
 
-  private[this] def metadataEntryHasMetadataKeysLike(metadataEntry: MetadataEntries,
-                                                     metadataKeys: NonEmptyList[String]): Rep[Boolean] = {
-    metadataKeys.toList.map(metadataEntry.metadataKey like _).reduce(_ || _)
+    val positiveFilter: Option[Rep[Boolean]] = metadataKeysToFilterFor.map(containsKey).reduceOption(_ || _)
+    val negativeFilter: Option[Rep[Boolean]] = metadataKeysToFilterOut.map(containsKey).reduceOption(_ || _)
+
+    (positiveFilter, negativeFilter) match {
+      case (Some(pf), Some(nf)) => pf && !nf
+      case (Some(pf), None) => pf
+      case (None, Some(nf)) => !nf
+
+      // We should never get here, but there's no reason not to handle it:
+      // ps: is there a better literal "true" in slick?
+      case (None, None) => true: Rep[Boolean]
+    }
   }
 
   private[this] def hasSameIndex(metadataEntry: MetadataEntries, jobIndex: Rep[Option[Int]]) = {
diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
index 30e07f94d37..eaf883fa2bf 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
@@ -2,7 +2,6 @@ package cromwell.database.sql
 
 import java.sql.Timestamp
 
-import cats.data.NonEmptyList
 import cromwell.database.sql.joins.MetadataJobQueryValue
 import cromwell.database.sql.tables.{MetadataEntry, WorkflowMetadataSummaryEntry}
 
@@ -51,16 +50,12 @@ trait MetadataSqlDatabase extends SqlDatabase {
                            jobAttempt: Option[Int])
                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
-  def queryMetadataEntriesLikeMetadataKeys(workflowExecutionUuid: String,
-                                           metadataKeys: NonEmptyList[String],
+  def queryMetadataEntryWithKeyConstraints(workflowExecutionUuid: String,
+                                           metadataKeysToFilterFor: List[String],
+                                           metadataKeysToFilterAgainst: List[String],
                                            metadataJobQueryValue: MetadataJobQueryValue)
                                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
 
-  def queryMetadataEntryNotLikeMetadataKeys(workflowExecutionUuid: String,
-                                            metadataKeys: NonEmptyList[String],
-                                            metadataJobQueryValue: MetadataJobQueryValue)
-                                           (implicit ec: ExecutionContext): Future[Seq[MetadataEntry]]
-
   /**
     * Retrieves next summarizable block of metadata satisfying the specified criteria.
     *
diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md
index 380bdc15a63..62d9cda44fc 100644
--- a/docs/api/RESTAPI.md
+++ b/docs/api/RESTAPI.md
@@ -1,5 +1,5 @@
 <!--
-This file was generated by `sbt generateRestApiDocs` on Mon, 13 May 2019 16:35:13 -0400
+This file was generated by `sbt generateRestApiDocs` on Mon, 10 Jun 2019 18:49:29 -0400
 
 !!! DO NOT CHANGE THIS FILE DIRECTLY !!!
 
@@ -460,9 +460,9 @@ GET /api/workflows/{version}/{id}/metadata
 |---|---|---|---|---|
 |**Path**|**id**  <br>*required*|A workflow ID|string||
 |**Path**|**version**  <br>*required*|Cromwell API Version|string|`"v1"`|
-|**Query**|**excludeKey**  <br>*optional*|When specified key(s) to exclude from the metadata. Matches any key starting with the value. May not be used with includeKey. This applies to all keys in the response, including within nested blocks.|< string > array(multi)||
+|**Query**|**excludeKey**  <br>*optional*|When specified, filters metadata to not return any field with a name which begins with this value. This key is used relative to the root of the response *and* relative to each call's metadata fields. Use 'calls' to filter out all call level metadata.|< string > array(multi)||
 |**Query**|**expandSubWorkflows**  <br>*optional*|When true, metadata for sub workflows will be fetched and inserted automatically in the metadata response.|boolean|`"false"`|
-|**Query**|**includeKey**  <br>*optional*|When specified key(s) to include from the metadata. Matches any key starting with the value. May not be used with excludeKey. This applies to all keys in the response, including within nested blocks.|< string > array(multi)||
+|**Query**|**includeKey**  <br>*optional*|When specified, filters metadata to only return fields with names which begins with this value. This key is used relative to the root of the response *and* relative to each call's metadata fields.|< string > array(multi)||
 
 
 #### Responses
diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml
index b6d117675c9..63a99d488de 100644
--- a/engine/src/main/resources/swagger/cromwell.yaml
+++ b/engine/src/main/resources/swagger/cromwell.yaml
@@ -511,8 +511,8 @@ paths:
         - $ref: '#/parameters/singleId'
         - name: includeKey
           description: >
-            When specified key(s) to include from the metadata. Matches any key starting with the value. May not be
-            used with excludeKey. This applies to all keys in the response, including within nested blocks.
+            When specified, filters metadata to only return fields with names which begins with this value.
+            This key is used relative to the root of the response *and* relative to each call's metadata fields.
           required: false
           type: array
           items:
@@ -521,8 +521,9 @@ paths:
           in: query
         - name: excludeKey
           description: >
-            When specified key(s) to exclude from the metadata. Matches any key starting with the value. May not be
-            used with includeKey. This applies to all keys in the response, including within nested blocks.
+            When specified, filters metadata to not return any field with a name which begins with this value.
+            This key is used relative to the root of the response *and* relative to each call's metadata fields.
+            Use 'calls' to filter out all call level metadata.
           required: false
           type: array
           items:
diff --git a/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala b/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala
index dcb10522d1d..96a87d14844 100644
--- a/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala
+++ b/engine/src/main/scala/cromwell/webservice/routes/MetadataRouteSupport.scala
@@ -69,15 +69,10 @@ trait MetadataRouteSupport extends HttpInstrumentation {
             val excludeKeysOption = NonEmptyList.fromList(excludeKeys.toList)
             val expandSubWorkflows = expandSubWorkflowsOption.getOrElse(false)
 
-            (includeKeysOption, excludeKeysOption) match {
-              case (Some(_), Some(_)) =>
-                val e = new IllegalArgumentException("includeKey and excludeKey may not be specified together")
-                e.failRequest(StatusCodes.BadRequest)
-              case (_, _) =>
-                metadataLookup(possibleWorkflowId,
-                  (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, includeKeysOption, excludeKeysOption, expandSubWorkflows),
-                  serviceRegistryActor, metadataBuilderRegulatorActor)
-            }
+            metadataLookup(possibleWorkflowId,
+              (w: WorkflowId) => GetSingleWorkflowMetadataAction(w, includeKeysOption, excludeKeysOption, expandSubWorkflows),
+              serviceRegistryActor,
+              metadataBuilderRegulatorActor)
           }
         }
       }
diff --git a/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala
index 3c61b0d6451..a805774e1a6 100644
--- a/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala
@@ -6,6 +6,7 @@ import akka.http.scaladsl.model.ContentTypes._
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
 import akka.stream.ActorMaterializer
+import com.typesafe.scalalogging.StrictLogging
 import common.util.VersionUtil
 import cromwell.core._
 import cromwell.core.abort.{WorkflowAbortFailureResponse, WorkflowAbortRequestedResponse, WorkflowAbortedResponse}
@@ -15,6 +16,7 @@ import cromwell.engine.workflow.workflowstore.WorkflowStoreActor._
 import cromwell.engine.workflow.workflowstore.WorkflowStoreEngineActor.{WorkflowOnHoldToSubmittedFailure, WorkflowOnHoldToSubmittedSuccess}
 import cromwell.engine.workflow.workflowstore.WorkflowStoreSubmitActor.{WorkflowSubmittedToStore, WorkflowsBatchSubmittedToStore}
 import cromwell.services.healthmonitor.ProtoHealthMonitorServiceActor.{GetCurrentStatus, StatusCheckResponse, SubsystemStatus}
+import cromwell.services.instrumentation.InstrumentationService.InstrumentationServiceMessage
 import cromwell.services.metadata.MetadataService._
 import cromwell.services.metadata._
 import cromwell.services.womtool.WomtoolServiceMessages.{DescribeFailure, DescribeRequest, DescribeSuccess}
@@ -472,7 +474,6 @@ class CromwellApiServiceSpec extends AsyncFlatSpec with ScalatestRouteTest with
           assertResult("<html>") {
             responseAs[String].substring(0, 6)
           }
-          assert(responseAs[String].contains("var metadataJson = {\"testKey1b\":\"myValue1b\",\"calls\":{},\"testKey1a\":\"myValue1a\",\"id\":\"c4c6339c-8cc9-47fb-acc5-b5cb8d2809f5\",\"testKey2a\":\"myValue2a\"};"))
         }
     }
 
@@ -533,14 +534,20 @@ object CromwellApiServiceSpec {
   }
 
   object MockServiceRegistryActor {
-    def fullMetadataResponse(workflowId: WorkflowId) = {
-      List(MetadataEvent(MetadataKey(workflowId, None, "testKey1"), MetadataValue("myValue1", MetadataString)),
-        MetadataEvent(MetadataKey(workflowId, None, "testKey2"), MetadataValue("myValue2", MetadataString)))
-    }
-    def filteredMetadataResponse(workflowId: WorkflowId) = {
-      List(MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a", MetadataString)),
+
+    private def fullMetadataResponse(workflowId: WorkflowId) = {
+      List(
+        MetadataEvent(MetadataKey(workflowId, None, "testKey1a"), MetadataValue("myValue1a", MetadataString)),
         MetadataEvent(MetadataKey(workflowId, None, "testKey1b"), MetadataValue("myValue1b", MetadataString)),
-        MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a", MetadataString)))
+        MetadataEvent(MetadataKey(workflowId, None, "testKey2a"), MetadataValue("myValue2a", MetadataString))
+      )
+    }
+
+    def responseMetadataValues(workflowId: WorkflowId, withKeys: List[String], withoutKeys: List[String]) = {
+      def keyFilter(keys: List[String])(m: MetadataEvent) = keys.exists(k => m.key.key.startsWith(k))
+      fullMetadataResponse(workflowId)
+        .filter(m => withKeys.isEmpty || keyFilter(withKeys)(m))
+        .filter(m => withoutKeys.isEmpty || !keyFilter(withoutKeys)(m))
     }
 
     def metadataQuery(workflowId: WorkflowId) =
@@ -554,8 +561,9 @@ object CromwellApiServiceSpec {
     }
   }
 
-  class MockServiceRegistryActor extends Actor {
+  class MockServiceRegistryActor extends Actor with StrictLogging {
     import MockServiceRegistryActor._
+
     override def receive = {
       case WorkflowQuery(parameters) =>
         val labels: Option[Map[String, String]] = {
@@ -589,9 +597,10 @@ object CromwellApiServiceSpec {
         val event = Vector(MetadataEvent(MetadataKey(id, None, "outputs:test.hello.salutation"), MetadataValue("Hello foo!", MetadataString)))
         sender ! WorkflowOutputsResponse(id, event)
       case GetLogs(id) => sender ! LogsResponse(id, logsEvents(id))
-      case GetSingleWorkflowMetadataAction(id, None, None, _) => sender ! MetadataLookupResponse(metadataQuery(id), fullMetadataResponse(id))
-      case GetSingleWorkflowMetadataAction(id, Some(_), None, _) => sender ! MetadataLookupResponse(metadataQuery(id), filteredMetadataResponse(id))
-      case GetSingleWorkflowMetadataAction(id, None, Some(_), _) => sender ! MetadataLookupResponse(metadataQuery(id), filteredMetadataResponse(id))
+      case GetSingleWorkflowMetadataAction(id, withKeys, withoutKeys, _) =>
+        val withKeysList = withKeys.map(_.toList).getOrElse(List.empty)
+        val withoutKeysList = withoutKeys.map(_.toList).getOrElse(List.empty)
+        sender ! MetadataLookupResponse(metadataQuery(id), responseMetadataValues(id, withKeysList, withoutKeysList))
       case PutMetadataActionAndRespond(events, _, _) =>
         events.head.key.workflowId match {
           case CromwellApiServiceSpec.ExistingWorkflowId => sender ! MetadataWriteSuccess(events)
@@ -617,7 +626,8 @@ object CromwellApiServiceSpec {
 
             sender ! DescribeSuccess(description = WorkflowDescription(valid = true, errors = readBack, validWorkflow = true))
         }
-
+      case _: InstrumentationServiceMessage => // Do nothing.
+      case m => logger.error("Unexpected message received by MockServiceRegistryActor: {}", m)
     }
   }
 
diff --git a/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala b/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala
index de0e075ab2c..db76f89fd50 100644
--- a/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala
+++ b/engine/src/test/scala/cromwell/webservice/routes/MetadataRouteSupportSpec.scala
@@ -1,13 +1,11 @@
 package cromwell.webservice.routes
 
 import akka.actor.{ActorSystem, Props}
-import akka.http.scaladsl.coding.{Decoder, Gzip}
 import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 import akka.http.scaladsl.model.headers.{HttpEncodings, `Accept-Encoding`}
 import akka.http.scaladsl.model.{ContentTypes, HttpEntity, StatusCodes}
 import akka.http.scaladsl.server.Route
 import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest}
-import akka.http.scaladsl.unmarshalling.Unmarshal
 import cromwell.core.WorkflowMetadataKeys
 import cromwell.webservice.routes.CromwellApiServiceSpec.MockServiceRegistryActor
 import cromwell.webservice.routes.MetadataRouteSupportSpec.MockMetadataRouteSupport
@@ -144,10 +142,11 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
-        result.fields.keys should contain allOf("testKey1", "testKey2")
+        result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
         result.fields.keys shouldNot contain("testKey3")
-        result.fields("testKey1") should be(JsString("myValue1"))
-        result.fields("testKey2") should be(JsString("myValue2"))
+        result.fields("testKey1a") should be(JsString("myValue1a"))
+        result.fields("testKey1b") should be(JsString("myValue1b"))
+        result.fields("testKey2a") should be(JsString("myValue2a"))
       }
   }
 
@@ -182,35 +181,29 @@ class MetadataRouteSupportSpec extends AsyncFlatSpec with ScalatestRouteTest wit
   }
 
   it should "return with excluded metadata from the metadata route" in {
-    Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?excludeKey=testKey2b&excludeKey=testKey3") ~>
+    Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?excludeKey=testKey2&excludeKey=testKey3") ~>
       akkaHttpService.metadataRoutes ~>
       check {
         status should be(StatusCodes.OK)
         val result = responseAs[JsObject]
-        result.fields.keys should contain allOf("testKey1a", "testKey1b", "testKey2a")
-        result.fields.keys should contain noneOf("testKey2b", "testKey3")
+        result.fields.keys should contain allOf("testKey1a", "testKey1b")
+        result.fields.keys should contain noneOf("testKey2a", "testKey3")
         result.fields("testKey1a") should be(JsString("myValue1a"))
         result.fields("testKey1b") should be(JsString("myValue1b"))
-        result.fields("testKey2a") should be(JsString("myValue2a"))
       }
   }
 
-  it should "return an error when included and excluded metadata requested from the metadata route" in {
-    Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey2") ~>
+  it should "correctly include and exclude metadata keys in workflow details requests" in {
+    Get(s"/workflows/$version/${CromwellApiServiceSpec.ExistingWorkflowId}/metadata?includeKey=testKey1&excludeKey=testKey1a") ~>
       akkaHttpService.metadataRoutes ~>
       check {
-        assertResult(StatusCodes.BadRequest) {
-          status
-        }
-
-        val decoder: Decoder = Gzip
-        Unmarshal(decoder.decodeMessage(response)).to[String] map { r =>
-          assertResult(
-            s"""{
-               |  "status": "fail",
-               |  "message": "includeKey and excludeKey may not be specified together"
-               |}""".stripMargin
-          ) { r }
+        val r = responseAs[String]
+        withClue(s"From response $r") {
+          status should be(StatusCodes.OK)
+          val result = responseAs[JsObject]
+          result.fields.keys should contain allElementsOf(List("testKey1b"))
+          result.fields.keys should contain noneOf("testKey1a", "testKey2")
+          result.fields("testKey1b") should be(JsString("myValue1b"))
         }
       }
   }
diff --git a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
index ce9ae225de8..9fb078de984 100644
--- a/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
+++ b/services/src/main/scala/cromwell/services/metadata/MetadataService.scala
@@ -82,9 +82,10 @@ object MetadataService {
 
   final case object ListenToMetadataWriteActor extends MetadataServiceAction with ListenToMessage
 
-  final case class GetSingleWorkflowMetadataAction(workflowId: WorkflowId, includeKeysOption: Option[NonEmptyList[String]],
-                                             excludeKeysOption: Option[NonEmptyList[String]],
-                                             expandSubWorkflows: Boolean)
+  final case class GetSingleWorkflowMetadataAction(workflowId: WorkflowId,
+                                                   includeKeysOption: Option[NonEmptyList[String]],
+                                                   excludeKeysOption: Option[NonEmptyList[String]],
+                                                   expandSubWorkflows: Boolean)
     extends ReadAction
   final case class GetMetadataQueryAction(key: MetadataQuery) extends ReadAction
   final case class GetStatus(workflowId: WorkflowId) extends ReadAction
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index ba306af18fd..9a6061f4fad 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -115,6 +115,9 @@ trait MetadataDatabaseAccess {
   }
 
   def queryMetadataEvents(query: MetadataQuery)(implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = {
+
+    def listKeyRequirements(keyRequirementsInput: Option[NonEmptyList[String]]): List[String] = keyRequirementsInput.map(_.toList).toList.flatten.map(_ + "%")
+
     val uuid = query.workflowId.id.toString
 
     val futureMetadata: Future[Seq[MetadataEntry]] = query match {
@@ -124,21 +127,13 @@ trait MetadataDatabaseAccess {
         metadataDatabaseInterface.queryMetadataEntries(uuid, jobKey.callFqn, jobKey.index, jobKey.attempt)
       case MetadataQuery(_, Some(jobKey), Some(key), None, None, _) =>
         metadataDatabaseInterface.queryMetadataEntries(uuid, key, jobKey.callFqn, jobKey.index, jobKey.attempt)
-      case MetadataQuery(_, None, None, Some(includeKeys), None, _) =>
-        metadataDatabaseInterface.
-          queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), CallOrWorkflowQuery)
-      case MetadataQuery(_, Some(MetadataQueryJobKey(callFqn, index, attempt)), None, Some(includeKeys), None, _) =>
-        metadataDatabaseInterface.
-          queryMetadataEntriesLikeMetadataKeys(uuid, includeKeys.map(_ + "%"), CallQuery(callFqn, index, attempt))
-      case MetadataQuery(_, None, None, None, Some(excludeKeys), _) =>
-        metadataDatabaseInterface.
-          queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), CallOrWorkflowQuery)
-      case MetadataQuery(_, Some(MetadataQueryJobKey(callFqn, index, attempt)), None, None, Some(excludeKeys), _) =>
-        metadataDatabaseInterface.
-          queryMetadataEntryNotLikeMetadataKeys(uuid, excludeKeys.map(_ + "%"), CallQuery(callFqn, index, attempt))
-      case MetadataQuery(_, None, None, Some(includeKeys), Some(excludeKeys), _) => Future.failed(
-        new IllegalArgumentException(
-          s"Include/Exclude keys may not be mixed: include = $includeKeys, exclude = $excludeKeys"))
+      case MetadataQuery(_, None, None, includeKeys, excludeKeys, _) =>
+        val excludeKeyRequirements = listKeyRequirements(excludeKeys)
+        val queryType = if (excludeKeyRequirements.contains("calls%")) WorkflowQuery else CallOrWorkflowQuery
+
+        metadataDatabaseInterface.queryMetadataEntryWithKeyConstraints(uuid, listKeyRequirements(includeKeys), excludeKeyRequirements, queryType)
+      case MetadataQuery(_, Some(MetadataQueryJobKey(callFqn, index, attempt)), None, includeKeys, excludeKeys, _) =>
+        metadataDatabaseInterface.queryMetadataEntryWithKeyConstraints(uuid, listKeyRequirements(includeKeys), listKeyRequirements(excludeKeys), CallQuery(callFqn, index, attempt))
       case _ => Future.failed(new IllegalArgumentException(s"Invalid MetadataQuery: $query"))
     }
 
@@ -148,8 +143,8 @@ trait MetadataDatabaseAccess {
   def queryWorkflowOutputs(id: WorkflowId)
                           (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = {
     val uuid = id.id.toString
-    metadataDatabaseInterface.queryMetadataEntriesLikeMetadataKeys(
-      uuid, NonEmptyList.of(s"${WorkflowMetadataKeys.Outputs}:%"), WorkflowQuery).
+    metadataDatabaseInterface.queryMetadataEntryWithKeyConstraints(
+      uuid, List(s"${WorkflowMetadataKeys.Outputs}:%"), List.empty, WorkflowQuery).
       map(metadataToMetadataEvents(id))
   }
 
@@ -157,8 +152,8 @@ trait MetadataDatabaseAccess {
                (implicit ec: ExecutionContext): Future[Seq[MetadataEvent]] = {
     import cromwell.services.metadata.CallMetadataKeys._
 
-    val keys = NonEmptyList.of(Stdout, Stderr, BackendLogsPrefix + ":%")
-    metadataDatabaseInterface.queryMetadataEntriesLikeMetadataKeys(id.id.toString, keys, CallOrWorkflowQuery) map
+    val keys = List(Stdout, Stderr, BackendLogsPrefix + ":%")
+    metadataDatabaseInterface.queryMetadataEntryWithKeyConstraints(id.id.toString, keys, List.empty, CallOrWorkflowQuery) map
       metadataToMetadataEvents(id)
   }
 
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
index 52e049ae128..91ec1729e12 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/ReadMetadataActor.scala
@@ -21,7 +21,7 @@ class ReadMetadataActor extends Actor with ActorLogging with MetadataDatabaseAcc
   def receive = {
     case GetSingleWorkflowMetadataAction(workflowId, includeKeysOption, excludeKeysOption, expandSubWorkflows) =>
       val includeKeys = if (expandSubWorkflows) {
-        includeKeysOption map { _.::(CallMetadataKeys.SubWorkflowId) }
+        includeKeysOption map { _ :+ CallMetadataKeys.SubWorkflowId }
       } else includeKeysOption
       queryAndRespond(MetadataQuery(workflowId, None, None, includeKeys, excludeKeysOption, expandSubWorkflows))
     case GetMetadataQueryAction(query@MetadataQuery(_, _, _, _, _, _)) => queryAndRespond(query)
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
index ad43eb9f0c6..81801bed672 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/WriteMetadataActorSpec.scala
@@ -4,7 +4,7 @@ import java.sql.{Connection, Timestamp}
 
 import akka.actor.ActorRef
 import akka.testkit.{TestFSMRef, TestProbe}
-import cats.data.{NonEmptyList, NonEmptyVector}
+import cats.data.NonEmptyVector
 import com.typesafe.config.ConfigFactory
 import cromwell.core.{TestKitSuite, WorkflowId}
 import cromwell.database.sql.joins.MetadataJobQueryValue
@@ -13,14 +13,14 @@ import cromwell.database.sql.{MetadataSqlDatabase, SqlDatabase}
 import cromwell.services.metadata.MetadataService.{MetadataWriteAction, MetadataWriteFailure, MetadataWriteSuccess, PutMetadataAction, PutMetadataActionAndRespond}
 import cromwell.services.metadata.impl.WriteMetadataActorSpec.BatchSizeCountingWriteMetadataActor
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
+import org.scalatest.concurrent.Eventually
+import org.scalatest.time.{Millis, Seconds, Span}
 import org.scalatest.{FlatSpecLike, Matchers}
 
 import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
-import scala.util.{Failure, Success}
 import scala.util.control.NoStackTrace
-import org.scalatest.concurrent.Eventually
-import org.scalatest.time.{Millis, Seconds, Span}
+import scala.util.{Failure, Success}
 
 class WriteMetadataActorSpec extends TestKitSuite with FlatSpecLike with Matchers with Eventually {
 
@@ -151,34 +151,24 @@ class WriteMetadataActorSpec extends TestKitSuite with FlatSpecLike with Matcher
                                      (implicit ec: ExecutionContext): Nothing = notImplemented()
 
     override def queryMetadataEntries(workflowExecutionUuid: String,
-                                      metadataKey: String)(implicit ec: ExecutionContext): Nothing = {
-      notImplemented()
-    }
+                                      metadataKey: String)(implicit ec: ExecutionContext): Nothing = notImplemented()
 
     override def queryMetadataEntries(workflowExecutionUuid: String,
                                       callFullyQualifiedName: String,
                                       jobIndex: Option[Int],
-                                      jobAttempt: Option[Int])(implicit ec: ExecutionContext): Nothing = {
-      notImplemented()
-    }
+                                      jobAttempt: Option[Int])(implicit ec: ExecutionContext): Nothing = notImplemented()
 
     override def queryMetadataEntries(workflowUuid: String,
                                       metadataKey: String,
                                       callFullyQualifiedName: String,
                                       jobIndex: Option[Int],
-                                      jobAttempt: Option[Int])(implicit ec: ExecutionContext): Nothing = {
-      notImplemented()
-    }
-
-    override def queryMetadataEntriesLikeMetadataKeys(workflowExecutionUuid: String,
-                                                      metadataKeys: NonEmptyList[String],
-                                                      metadataJobQueryValue: MetadataJobQueryValue)
-                                                     (implicit ec: ExecutionContext): Nothing = notImplemented()
+                                      jobAttempt: Option[Int])(implicit ec: ExecutionContext): Nothing = notImplemented()
 
-    override def queryMetadataEntryNotLikeMetadataKeys(workflowExecutionUuid: String,
-                                                       metadataKeys: NonEmptyList[String],
-                                                       metadataJobQueryValue: MetadataJobQueryValue)
-                                                      (implicit ec: ExecutionContext): Nothing = notImplemented()
+    override def queryMetadataEntryWithKeyConstraints(workflowExecutionUuid: String,
+                                             metadataKeysToFilterFor: List[String],
+                                             metadataKeysToFilterAgainst: List[String],
+                                             metadataJobQueryValue: MetadataJobQueryValue)
+                                            (implicit ec: ExecutionContext): Nothing = notImplemented()
 
     override def summarizeIncreasing(summaryNameIncreasing: String,
                                      startMetadataKey: String,

From ef7bc24a0a2fd045bc8a15749d4b169434599c5a Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Fri, 14 Jun 2019 10:28:00 -0400
Subject: [PATCH 14/28]  Fix the dollar re-interpolation behavior in draft-2
 [BA-4928] (#5026)

---
 .../cromwell/backend/ReadLikeFunctions.scala  |  2 +-
 .../standardTestCases/dollars_in_strings.test | 14 +++++++
 .../dollars_in_strings/dollars_in_strings.wdl | 32 ++++++++++++++
 .../model/expression/ValueEvaluator.scala     | 42 +++++++++----------
 .../scala/wdl/AdvancedInterpolationSpec.scala | 13 +++++-
 5 files changed, 79 insertions(+), 24 deletions(-)
 create mode 100644 centaur/src/main/resources/standardTestCases/dollars_in_strings.test
 create mode 100644 centaur/src/main/resources/standardTestCases/dollars_in_strings/dollars_in_strings.wdl

diff --git a/backend/src/main/scala/cromwell/backend/ReadLikeFunctions.scala b/backend/src/main/scala/cromwell/backend/ReadLikeFunctions.scala
index 0b0c59831e6..7f718978435 100644
--- a/backend/src/main/scala/cromwell/backend/ReadLikeFunctions.scala
+++ b/backend/src/main/scala/cromwell/backend/ReadLikeFunctions.scala
@@ -8,7 +8,7 @@ import scala.concurrent.Future
 import scala.util.Try
 
 trait ReadLikeFunctions extends PathFactory with IoFunctionSet with AsyncIoFunctions {
-  
+
   override def readFile(path: String, maxBytes: Option[Int], failOnOverflow: Boolean): Future[String] =
     Future.fromTry(Try(buildPath(path))) flatMap { p => asyncIo.contentAsStringAsync(p, maxBytes, failOnOverflow) }
 
diff --git a/centaur/src/main/resources/standardTestCases/dollars_in_strings.test b/centaur/src/main/resources/standardTestCases/dollars_in_strings.test
new file mode 100644
index 00000000000..c2838e6dfb5
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/dollars_in_strings.test
@@ -0,0 +1,14 @@
+name: dollars_in_strings
+testFormat: workflowsuccess
+
+files {
+  workflow: dollars_in_strings/dollars_in_strings.wdl
+}
+
+metadata {
+  workflowName: read_dollared_strings
+  status: Succeeded
+  "outputs.read_dollared_strings.s1": "${BLAH}"
+  "outputs.read_dollared_strings.s2": "${BLAH}"
+  "outputs.read_dollared_strings.s3": "oops ${BLAH}"
+}
diff --git a/centaur/src/main/resources/standardTestCases/dollars_in_strings/dollars_in_strings.wdl b/centaur/src/main/resources/standardTestCases/dollars_in_strings/dollars_in_strings.wdl
new file mode 100644
index 00000000000..f98eb1b0e6f
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/dollars_in_strings/dollars_in_strings.wdl
@@ -0,0 +1,32 @@
+workflow read_dollared_strings {
+
+  call dollars_in_strings
+
+  String dollar = "$"
+
+  output {
+    String s1 = "${dollar}{BLAH}"
+    String s2 = s1
+
+    String s3 = dollars_in_strings.s3
+  }
+}
+
+
+task dollars_in_strings {
+  String dollar = "$"
+  command <<<
+    cat > foo.txt << 'EOF'
+      oops ${dollar}{BLAH}
+    EOF
+  >>>
+
+  output {
+    File x = "foo.txt"
+    String s3 = read_string(x)
+  }
+
+  runtime {
+    docker: "ubuntu:latest"
+  }
+}
diff --git a/wdl/model/draft2/src/main/scala/wdl/draft2/model/expression/ValueEvaluator.scala b/wdl/model/draft2/src/main/scala/wdl/draft2/model/expression/ValueEvaluator.scala
index 413774b6da9..e57ead11abb 100644
--- a/wdl/model/draft2/src/main/scala/wdl/draft2/model/expression/ValueEvaluator.scala
+++ b/wdl/model/draft2/src/main/scala/wdl/draft2/model/expression/ValueEvaluator.scala
@@ -21,34 +21,34 @@ case class ValueEvaluator(override val lookup: String => WomValue, override val
 
   private val InterpolationTagPattern = "\\$\\{\\s*([^\\}]*)\\s*\\}".r
 
-  private def replaceInterpolationTag(string: Try[WomString], tag: String): Try[WomString] = {
-    val expr = WdlExpression.fromString(tag.substring(2, tag.length - 1))
-    (expr.evaluate(lookup, functions), string) match {
-      case (Success(value), Success(str)) =>
-        value match {
-          case s: WomString if InterpolationTagPattern.anchored.findFirstIn(s.valueString).isDefined =>
-            replaceInterpolationTag(Success(WomString(str.value.replace(tag, s.valueString))), s.valueString)
-          case v => Success(WomString(str.value.replace(tag, v.valueString)))
-        }
-      case (Failure(ex), _) => Failure(ex)
-      case (_, Failure(ex)) => Failure(ex)
+
+  private def interpolate(strToProcess: String, resultSoFar: Try[WomString] = Success(WomString(""))): Try[WomString] = {
+
+    def evaluateTag(tag: String): Try[WomString] = {
+      val expr = WdlExpression.fromString(tag.substring(2, tag.length - 1))
+      expr.evaluate(lookup, functions).map(result => WomString(result.valueString))
     }
-  }
 
-  private def interpolate(str: String): Try[WomString] = {
-    InterpolationTagPattern.findAllIn(str).foldLeft(Try(WomString(str)))(replaceInterpolationTag)
-  }
+    InterpolationTagPattern.findFirstIn(strToProcess) match {
+      case Some(tag) =>
+        val prefix = strToProcess.substring(0, strToProcess.indexOf(tag))
+        val suffix = strToProcess.substring(strToProcess.indexOf(tag) + tag.length, strToProcess.length)
+
+        val newResultSoFar = for {
+          rsf <- resultSoFar
+          interpolatedTag <- evaluateTag(tag)
+        } yield WomString(rsf.value + prefix + interpolatedTag.value)
 
-  private def interpolate(value: WomValue): Try[WomValue] = {
-    value match {
-      case s: WomString => interpolate(s.valueString)
-      case _ => Try(value)
+        interpolate(suffix, newResultSoFar)
+
+      case None => resultSoFar.map(ws => WomString(ws.value + strToProcess))
     }
+
   }
 
   override def evaluate(ast: AstNode): Try[WomValue] = {
     ast match {
-      case t: Terminal if t.getTerminalStr == "identifier" => Try(lookup(t.getSourceString)).flatMap(interpolate)
+      case t: Terminal if t.getTerminalStr == "identifier" => Try(lookup(t.getSourceString))
       case t: Terminal if t.getTerminalStr == "integer" => Success(WomInteger(t.getSourceString.toInt))
       case t: Terminal if t.getTerminalStr == "float" => Success(WomFloat(t.getSourceString.toDouble))
       case t: Terminal if t.getTerminalStr == "boolean" => Success(WomBoolean(t.getSourceString == "true"))
@@ -136,7 +136,7 @@ case class ValueEvaluator(override val lookup: String => WomValue, override val
         a.getAttribute("rhs") match {
           case rhs:Terminal if rhs.getTerminalStr == "identifier" =>
             val memberAccessAsString = s"${a.getAttribute("lhs").sourceString}.${a.getAttribute("rhs").sourceString}"
-            Try(lookup(memberAccessAsString)).flatMap(interpolate).recoverWith {
+            Try(lookup(memberAccessAsString)).recoverWith {
               case _ =>
                 evaluate(a.getAttribute("lhs")).flatMap {
                   case o: WomObjectLike =>
diff --git a/wdl/model/draft2/src/test/scala/wdl/AdvancedInterpolationSpec.scala b/wdl/model/draft2/src/test/scala/wdl/AdvancedInterpolationSpec.scala
index 03605eed60f..0d0617b688a 100644
--- a/wdl/model/draft2/src/test/scala/wdl/AdvancedInterpolationSpec.scala
+++ b/wdl/model/draft2/src/test/scala/wdl/AdvancedInterpolationSpec.scala
@@ -31,12 +31,21 @@ class AdvancedInterpolationSpec extends FlatSpec with Matchers {
 
   val namespace = WdlNamespaceWithWorkflow.load(wdl, Seq.empty).get
 
-  it should "be able to resolve interpolated strings within interpolated strings" in {
+  it should "not interpolate into input strings which are not WDL literals" in {
     val testCall = namespace.workflow.taskCalls.find(_.unqualifiedName == "test") getOrElse {
       fail("call 'test' not found")
     }
     val inputs = testCall.evaluateTaskInputs(Map("testWF.test.eval_this" -> WomString("${var}")), NoFunctions)
     val testCmd = testCall.task.instantiateCommand(inputs.get, NoFunctions).toTry.get.head
-    testCmd.commandString shouldEqual "echo 'inside inside inside'"
+    testCmd.commandString shouldEqual "echo '${var} ${var} inside'"
+  }
+
+  it should "interpolate into input values which are WDL literals" in {
+    val test2Call = namespace.workflow.taskCalls.find(_.unqualifiedName == "test2") getOrElse {
+      fail("call 'test2' not found")
+    }
+    val inputs2 = test2Call.evaluateTaskInputs(Map.empty, NoFunctions)
+    val test2Cmd = test2Call.task.instantiateCommand(inputs2.get, NoFunctions).toTry.get.head
+    test2Cmd.commandString shouldEqual "echo 'outside outside inside'"
   }
 }

From 72af68d5297893d221d9a41565789a47ed27d119 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Fri, 14 Jun 2019 11:40:15 -0400
Subject: [PATCH 15/28] Allow public key retrieval in centaur test connection
 string (#5025)

---
 src/ci/resources/cromwell_database.inc.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ci/resources/cromwell_database.inc.conf b/src/ci/resources/cromwell_database.inc.conf
index 4c93bb2635b..62ce15c9345 100644
--- a/src/ci/resources/cromwell_database.inc.conf
+++ b/src/ci/resources/cromwell_database.inc.conf
@@ -4,7 +4,7 @@ database {
   db {
     driver = "com.mysql.cj.jdbc.Driver"
     driver = ${?CROMWELL_BUILD_CENTAUR_JDBC_DRIVER}
-    url = "jdbc:mysql://localhost:3306/cromwell_test?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
+    url = "jdbc:mysql://localhost:3306/cromwell_test?allowPublicKeyRetrieval=true&useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
     url = ${?CROMWELL_BUILD_CENTAUR_JDBC_URL}
     user = "cromwell"
     user = ${?CROMWELL_BUILD_CENTAUR_JDBC_USERNAME}

From 756b0fe0b9004c7da31ae6f5b6f18e059a136b1c Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Wed, 19 Jun 2019 13:18:20 -0400
Subject: [PATCH 16/28] Stackdriver in Cromwell Perf [BA-5729] (#5036)

---
 scripts/perf/helper.inc.sh                    | 15 ---------------
 scripts/perf/jenkins_script.sh                | 14 +++++++-------
 scripts/perf/startup_script.sh                |  5 -----
 .../perf/vm_scripts/cromwell/cromwell.conf    | 13 +++++++------
 scripts/perf/vm_scripts/docker-compose.yml    | 19 +------------------
 ...ackdriverInstrumentationServiceActor.scala |  3 ++-
 6 files changed, 17 insertions(+), 52 deletions(-)

diff --git a/scripts/perf/helper.inc.sh b/scripts/perf/helper.inc.sh
index cd99337ab2a..1cdf6c50b9f 100644
--- a/scripts/perf/helper.inc.sh
+++ b/scripts/perf/helper.inc.sh
@@ -78,27 +78,12 @@ export_logs() {
     gsutil -h "Content-Type:text/plain" cp $(docker inspect --format='{{.LogPath}}' cromwell) "${REPORT_URL}/cromwell.log" || true
     # Copy the docker daemon logs
     gsutil -h "Content-Type:text/plain" cp /var/log/daemon.log "${REPORT_URL}/daemon.log" || true
-    # Copy the statsd logs
-    gsutil -h "Content-Type:text/plain" cp "${STATSD_PROXY_APP_DIR}/statsd.log" "${REPORT_URL}/statsd.log" || true
     # Copy centaur log
     gsutil -h "Content-Type:text/plain" cp "${CROMWELL_ROOT}/centaur.log" "${REPORT_URL}/centaur.log" || true
     # Copy test_rc.log
     echo $TEST_RC > test_rc.txt && gsutil -h "Content-Type:text/plain" cp test_rc.txt "${REPORT_URL}/test_rc.txt" || true
 }
 
-prepare_statsd_proxy() {
-    # Build the image
-    cd ${CROMWELL_ROOT}
-    export CROMWELL_SBT_DOCKER_TAGS=perf
-    sbt statsDProxy/docker
-    
-    # Export configuration variables
-    export STATSD_PROXY_APP_DIR="${PERF_ROOT}/vm_scripts/statsd-proxy"
-    # Hard code those for now as there's no obvious reason to make them configurable
-    export PROXY_HOST=0.0.0.0
-    export PROXY_PORT=9125
-}
-
 clean_up() {
     if [ "${CLEAN_UP}" = true ]
     then
diff --git a/scripts/perf/jenkins_script.sh b/scripts/perf/jenkins_script.sh
index 0c102c7d6c7..6a65f28d88e 100755
--- a/scripts/perf/jenkins_script.sh
+++ b/scripts/perf/jenkins_script.sh
@@ -7,6 +7,8 @@ VAULT_TOKEN=$(cat /etc/vault-token-dsde)
 
 DOCKER_ETC_PATH=/usr/share/etc
 
+CROMWELL_PERF_PROJECT=broad-dsde-cromwell-perf
+
 mkdir -p mnt
 
 #TODO: need this in the google cloud docker in order to auth
@@ -27,18 +29,18 @@ CLOUD_SQL_INSTANCE=cromwell-db-perf-test-$BUILD_TAG
 # By doing it here we can run the jenkins jobs sequentially and ensure the database is cloned once at a time as well
 SQL_OPERATION=$(docker run --name perf_sql_create_gcloud_$BUILD_NUMBER -v "$(pwd)"/mnt:$DOCKER_ETC_PATH --rm google/cloud-sdk:slim /bin/bash -c "\
     gcloud auth activate-service-account --key-file $DOCKER_ETC_PATH/sa.json 2> /dev/null &&\
-    gcloud --project broad-dsde-cromwell-perf sql instances clone --async cromwell-perf-testing-base-09-24-18 ${CLOUD_SQL_INSTANCE} --format='value(name)'")
+    gcloud --project $CROMWELL_PERF_PROJECT sql instances clone --async cromwell-perf-testing-base-09-24-18 ${CLOUD_SQL_INSTANCE} --format='value(name)'")
 
 # The cloning itself sometimes takes a long time and the clone command errors out when that happens.
 # Instead use the --async flag in the clone command above and then explicitly wait for the operation to be done. Timeout 15 minutes
 docker run --name perf_sql_create_gcloud_$BUILD_NUMBER -v "$(pwd)"/mnt:$DOCKER_ETC_PATH -e DOCKER_ETC_PATH --rm google/cloud-sdk:slim /bin/bash -c "\
     gcloud auth activate-service-account --key-file $DOCKER_ETC_PATH/sa.json &&\
-    gcloud beta sql operations wait --timeout=900 --project broad-dsde-cromwell-perf $SQL_OPERATION"
+    gcloud beta sql operations wait --timeout=900 --project $CROMWELL_PERF_PROJECT $SQL_OPERATION"
 
 # Add a cromwell user to the cloned database
 docker run --name perf_sql_user_gcloud_$BUILD_NUMBER -v "$(pwd)"/mnt:$DOCKER_ETC_PATH --rm google/cloud-sdk:slim /bin/bash -c "\
     gcloud auth activate-service-account --key-file $DOCKER_ETC_PATH/sa.json &&\
-    gcloud --project broad-dsde-cromwell-perf sql users create cromwell --instance=${CLOUD_SQL_INSTANCE} --password=${DB_PASS}"
+    gcloud --project $CROMWELL_PERF_PROJECT sql users create cromwell --instance=${CLOUD_SQL_INSTANCE} --password=${DB_PASS}"
 
 metadata=(
   "BUILD_NUMBER=$BUILD_NUMBER"
@@ -48,10 +50,8 @@ metadata=(
   "CROMWELL_DB_USER=cromwell"
   "CROMWELL_DB_PASS=$DB_PASS"
   "CROMWELL_DOCKER_IMAGE=$CROMWELL_DOCKER_IMAGE"
-  "CROMWELL_PROJECT=broad-dsde-cromwell-perf"
+  "CROMWELL_PROJECT=$CROMWELL_PERF_PROJECT"
   "CROMWELL_BUCKET=$GCS_BUCKET"
-  "CROMWELL_STATSD_HOST=10.128.0.4"
-  "CROMWELL_STATSD_PORT=8125"
   "CROMWELL_PERF_SCRIPTS_BRANCH=$CROMWELL_PERF_SCRIPTS_BRANCH"
   "GCS_REPORT_BUCKET=$GCS_REPORT_BUCKET"
   "GCS_REPORT_PATH=$GCS_REPORT_PATH"
@@ -62,7 +62,7 @@ docker run --name perf_gcloud_$BUILD_NUMBER -v "$(pwd)"/mnt:$DOCKER_ETC_PATH --r
     gcloud auth activate-service-account --key-file $DOCKER_ETC_PATH/sa.json &&\
 gcloud \
     --verbosity info \
-    --project broad-dsde-cromwell-perf \
+    --project $CROMWELL_PERF_PROJECT \
     compute \
     instances \
     create $BUILD_TAG \
diff --git a/scripts/perf/startup_script.sh b/scripts/perf/startup_script.sh
index 8194811fa30..be699afb10e 100644
--- a/scripts/perf/startup_script.sh
+++ b/scripts/perf/startup_script.sh
@@ -28,15 +28,11 @@ addVar CLOUD_SQL_INSTANCE=$(extract_metadata CLOUD_SQL_INSTANCE)
 addVar CROMWELL_DOCKER_IMAGE=$(extract_metadata CROMWELL_DOCKER_IMAGE)
 addVar CROMWELL_PROJECT=$(extract_metadata CROMWELL_PROJECT)
 addVar CROMWELL_EXECUTION_ROOT=$(extract_metadata CROMWELL_BUCKET)
-addVar CROMWELL_STATSD_HOST=$(extract_metadata CROMWELL_STATSD_HOST)
-addVar CROMWELL_STATSD_PORT=$(extract_metadata CROMWELL_STATSD_PORT)
 addVar GCS_REPORT_BUCKET=$(extract_metadata GCS_REPORT_BUCKET)
 addVar GCS_REPORT_PATH=$(extract_metadata GCS_REPORT_PATH)
 addVar BUILD_ID=$(extract_metadata BUILD_TAG)
 addVar CLEAN_UP=$(extract_metadata CLEAN_UP)
 
-# Use the instance name as statsd prefix to avoid metrics collisions
-addVar CROMWELL_STATSD_PREFIX=${BUILD_ID}
 addVar REPORT_BUCKET=cromwell-perf-test-reporting
 
 addVar CROMWELL_ROOT=/app
@@ -82,7 +78,6 @@ then
 fi
 
 # Start cromwell and cloud sql proxy
-prepare_statsd_proxy
 docker-compose -f ${PERF_ROOT}/vm_scripts/docker-compose.yml up -d
 
 if [ -n "$CENTAUR_TEST_FILE" ]
diff --git a/scripts/perf/vm_scripts/cromwell/cromwell.conf b/scripts/perf/vm_scripts/cromwell/cromwell.conf
index 1747f25ed5d..528f404be3b 100644
--- a/scripts/perf/vm_scripts/cromwell/cromwell.conf
+++ b/scripts/perf/vm_scripts/cromwell/cromwell.conf
@@ -70,13 +70,14 @@ backend {
 
 services {
   Instrumentation {
-    # StatsD - Send metrics to a StatsD server over UDP
-    class = "cromwell.services.instrumentation.impl.statsd.StatsDInstrumentationServiceActor"
+    # Stackdriver - Send metrics to Google's monitoring API
+    class = "cromwell.services.instrumentation.impl.stackdriver.StackdriverInstrumentationServiceActor"
     config {
-      hostname = statsd-proxy
-      port = 9125
-      prefix = ${?CROMWELL_STATSD_PREFIX} # can be used to prefix all metrics with an api key for example
-      flush-rate = 5 seconds # rate at which aggregated metrics will be sent to statsd
+      auth = "application-default"
+      google-project = ${?CROMWELL_PROJECT}
+      # rate at which aggregated metrics will be sent to Stackdriver API, must be 1 minute or more.
+      flush-rate = 1 minute
+      cromwell-perf-test-case = ${?TEST_CASE_DIRECTORY}
     }
   }
 }
diff --git a/scripts/perf/vm_scripts/docker-compose.yml b/scripts/perf/vm_scripts/docker-compose.yml
index 254c51bb2a0..a6e05ced58c 100644
--- a/scripts/perf/vm_scripts/docker-compose.yml
+++ b/scripts/perf/vm_scripts/docker-compose.yml
@@ -10,35 +10,18 @@ services:
     environment:
       - CROMWELL_PROJECT
       - CROMWELL_EXECUTION_ROOT
-      - CROMWELL_STATSD_PREFIX
       - CLOUD_SQL_DB_USER
       - CLOUD_SQL_DB_PASSWORD
+      - TEST_CASE_DIRECTORY
       - JAVA_OPTS=-Dconfig.file=/conf/cromwell.conf
     command: ["server"]
     depends_on:
-      - statsd-proxy
       - cloudsql-db
     ports:
       - "8000:8000"
     networks:
       mysql_net:
     container_name: cromwell
-  statsd-proxy:
-    image: "broadinstitute/statsd-proxy:perf"
-    volumes:
-      - ${STATSD_PROXY_APP_DIR}:/conf
-    environment:
-      - PROXY_HOST
-      - PROXY_PORT
-      - CROMWELL_STATSD_HOST
-      - CROMWELL_STATSD_PORT
-      - JAVA_OPTS=-Dconfig.file=/conf/proxy.conf
-    ports:
-      - "8125:8125/udp"
-      - "9125:9125/udp"
-    container_name: statsd-proxy
-    networks:
-      mysql_net:
   cloudsql-db:
     image: "gcr.io/cloudsql-docker/gce-proxy:1.11"
     ports:
diff --git a/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
index 1dd5462f62d..9de1e035c91 100644
--- a/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/instrumentation/impl/stackdriver/StackdriverInstrumentationServiceActor.scala
@@ -165,7 +165,8 @@ object StackdriverInstrumentationServiceActor {
     /**
       * Transforms a CromwellBucket to a Stackdriver path
       */
-    def toStackdriverString = (CromwellMetricPrefix ++ cromwellBucket.prefix ++ cromwellBucket.path.toList).mkString("/").replace(" ", "_")
+    def toStackdriverString = (CromwellMetricPrefix ++ cromwellBucket.prefix ++
+        cromwellBucket.path.toList).mkString("/").replaceAll(" ", "_").replaceAll("\\[|\\]", "")
   }
 }
 

From 4f19570d2c61d9e0c7d4e66234dbc406a7179061 Mon Sep 17 00:00:00 2001
From: mcovarr <mcovarr@users.noreply.github.com>
Date: Wed, 19 Jun 2019 14:54:40 -0400
Subject: [PATCH 17/28]  BIGINT the CALL_CACHING_HASH_ENTRY PK [develop
 edition] (#5037)

BIGINT the CALL_CACHING_HASH_ENTRY PK
---
 CHANGELOG.md                                  | 13 ++++++++++
 .../src/main/resources/changelog.xml          |  1 +
 .../enlarge_call_caching_hash_entry_id.xml    | 26 +++++++++++++++++++
 .../CallCachingHashEntryComponent.scala       |  6 ++---
 .../sql/tables/CallCachingHashEntry.scala     |  2 +-
 docs/developers/Arch.md                       | 14 +++++-----
 6 files changed, 51 insertions(+), 11 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4420f5a1dd0..b2ea6c415fa 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,19 @@
 
 ## 43 Release Notes
 
+### Call caching database refactoring
+
+Cromwell's `CALL_CACHING_HASH_ENTRY` primary key has been refactored to use a `BIGINT` datatype in place of the previous
+`INT` datatype. Cromwell will not be usable during the time the Liquibase migration for this refactor is running.
+In the Google Cloud SQL with SSD environment this migration runs at a rate of approximately 100,000 `CALL_CACHING_HASH_ENTRY`
+rows per second. In deployments with millions or billions of `CALL_CACHING_HASH_ENTRY` rows the migration may require  
+a significant amount of downtime so please plan accordingly. The following SQL could be used to estimate the number of
+rows in this table:
+
+```
+select max(CALL_CACHING_HASH_ENTRY_ID) from CALL_CACHING_HASH_ENTRY
+```
+
 ### Stackdriver Instrumentation
 
 Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). 
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 477309766cf..5b755a12cba 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -76,6 +76,7 @@
     <include file="changesets/add_workflow_url_in_workflow_store_entry.xml" relativeToChangelogFile="true" />
     <include file="changesets/change_max_size_workflow_url.xml" relativeToChangelogFile="true" />
     <include file="changesets/docker_hash_store_add_size_column.xml" relativeToChangelogFile="true" />
+    <include file="changesets/enlarge_call_caching_hash_entry_id.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
new file mode 100644
index 00000000000..d2159d41bdc
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="mcovarr" id="enlarge_call_caching_hash_entry_id">
+        <comment>
+            Larger integer data type for the PK since > 2 billion rows.
+        </comment>
+        <modifyDataType
+                tableName="CALL_CACHING_HASH_ENTRY"
+                columnName="CALL_CACHING_HASH_ENTRY_ID"
+                newDataType="BIGINT"/>
+    </changeSet>
+
+    <changeSet author="mcovarr" id="restore_auto_increment_call_caching_hash_entry_id">
+        <comment>
+            Factored into a separate changeset from the above to allow for handling various RDBMS implementations differently.
+        </comment>
+        <addAutoIncrement columnName="CALL_CACHING_HASH_ENTRY_ID"
+                          columnDataType="BIGINT"
+                          incrementBy="1"
+                          tableName="CALL_CACHING_HASH_ENTRY"/>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
index 017a8149f68..1a8e3e772f1 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingHashEntryComponent.scala
@@ -9,7 +9,7 @@ trait CallCachingHashEntryComponent {
   import driver.api._
 
   class CallCachingHashEntries(tag: Tag) extends Table[CallCachingHashEntry](tag, "CALL_CACHING_HASH_ENTRY") {
-    def callCachingHashEntryId = column[Int]("CALL_CACHING_HASH_ENTRY_ID", O.PrimaryKey, O.AutoInc)
+    def callCachingHashEntryId = column[Long]("CALL_CACHING_HASH_ENTRY_ID", O.PrimaryKey, O.AutoInc)
 
     def hashKey = column[String]("HASH_KEY", O.Length(255))
 
@@ -36,9 +36,9 @@ trait CallCachingHashEntryComponent {
     * Find all hashes for a CALL_CACHING_ENTRY_ID
     */
   val callCachingHashEntriesForCallCachingEntryId = Compiled(
-    (callCachingHashEntryId: Rep[Int]) => for {
+    (callCachingEntryId: Rep[Int]) => for {
       callCachingHashEntry <- callCachingHashEntries
-      if callCachingHashEntry.callCachingEntryId === callCachingHashEntryId
+      if callCachingHashEntry.callCachingEntryId === callCachingEntryId
     } yield callCachingHashEntry
   )
 }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala
index 91bb72da747..41926ff2057 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingHashEntry.scala
@@ -5,5 +5,5 @@ case class CallCachingHashEntry
   hashKey: String,
   hashValue: String,
   callCachingEntryId: Option[Int] = None,
-  callCachingHashEntryId: Option[Int] = None
+  callCachingHashEntryId: Option[Long] = None
 )
diff --git a/docs/developers/Arch.md b/docs/developers/Arch.md
index 3044ad57fae..eda0fc4664b 100644
--- a/docs/developers/Arch.md
+++ b/docs/developers/Arch.md
@@ -55,19 +55,19 @@
 
 | Field                      | Type         | Null | Key | Default | Extra          |
 |----------------------------|--------------|------|-----|---------|----------------|
-| CALL_CACHING_HASH_ENTRY_ID | int(11)      | NO   | PRI | NULL    | auto_increment |
+| CALL_CACHING_HASH_ENTRY_ID | bigint(20)   | NO   | PRI | NULL    | auto_increment |
 | HASH_KEY                   | varchar(255) | NO   |     | NULL    |                |
 | HASH_VALUE                 | varchar(255) | NO   |     | NULL    |                |
 | CALL_CACHING_ENTRY_ID      | int(11)      | YES  | MUL | NULL    |                |
 
 ## Call Caching Simpleton Entry
 
-| Field                      | Type         | Null | Key | Default | Extra          |
-|----------------------------|--------------|------|-----|---------|----------------|
-| CALL_CACHING_HASH_ENTRY_ID | int(11)      | NO   | PRI | NULL    | auto_increment |
-| HASH_KEY                   | varchar(255) | NO   |     | NULL    |                |
-| HASH_VALUE                 | varchar(255) | NO   |     | NULL    |                |
-| CALL_CACHING_ENTRY_ID      | int(11)      | YES  | MUL | NULL    |                |
+| Field                           | Type         | Null | Key | Default | Extra          |
+|---------------------------------|--------------|------|-----|---------|----------------|
+| CALL_CACHING_SIMPLETON_ENTRY_ID | int(11)      | NO   | PRI | NULL    | auto_increment |
+| HASH_KEY                        | varchar(255) | NO   |     | NULL    |                |
+| HASH_VALUE                      | varchar(255) | NO   |     | NULL    |                |
+| CALL_CACHING_ENTRY_ID           | int(11)      | YES  | MUL | NULL    |                |
 
 ## Custom Labels
 

From c45b96306ee7a5a418767e731652a8875a31b1af Mon Sep 17 00:00:00 2001
From: Adam Nichols <aednichols@gmail.com>
Date: Thu, 20 Jun 2019 12:15:13 -0400
Subject: [PATCH 18/28] Catch and log workflow log copy failures [BA-4916]
 (#5022)

---
 .../cromwell/engine/EngineFilesystems.scala   |  8 +-
 .../engine/workflow/WorkflowActor.scala       | 13 ++-
 .../MaterializeWorkflowDescriptorActor.scala  | 10 ++-
 .../drs/DrsPathBuilderFactory.scala           |  2 +-
 .../engine/workflow/WorkflowActorSpec.scala   | 87 +++++++++++++------
 5 files changed, 84 insertions(+), 36 deletions(-)

diff --git a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
index 8f7a102e6d3..296d227d39f 100644
--- a/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
+++ b/engine/src/main/scala/cromwell/engine/EngineFilesystems.scala
@@ -18,12 +18,14 @@ object EngineFilesystems {
       .filter(_ => config.as[Boolean]("engine.filesystems.local.enabled"))
       .toMap
 
-  private val pathBuilderFactories = {
+  private val pathBuilderFactories: Map[String, PathBuilderFactory] = {
     CromwellFileSystems.instance.factoriesFromConfig(config.as[Config]("engine"))
       .unsafe("Failed to instantiate engine filesystem") ++ defaultFileSystemFactory
   }
 
-  def pathBuildersForWorkflow(workflowOptions: WorkflowOptions)(implicit as: ActorSystem): Future[List[PathBuilder]] = {
-    PathBuilderFactory.instantiatePathBuilders(pathBuilderFactories.values.toList, workflowOptions)
+  def configuredPathBuilderFactories: List[PathBuilderFactory] = pathBuilderFactories.values.toList
+
+  def pathBuildersForWorkflow(workflowOptions: WorkflowOptions, factories: List[PathBuilderFactory])(implicit as: ActorSystem): Future[List[PathBuilder]] = {
+    PathBuilderFactory.instantiatePathBuilders(factories, workflowOptions)
   }
 }
diff --git a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
index 687781defe4..56b05f1c596 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/WorkflowActor.scala
@@ -12,7 +12,7 @@ import cromwell.core.WorkflowOptions.FinalWorkflowLogDir
 import cromwell.core.WorkflowProcessingEvents.DescriptionEventValue.Finished
 import cromwell.core._
 import cromwell.core.logging.{WorkflowLogger, WorkflowLogging}
-import cromwell.core.path.{PathBuilder, PathFactory}
+import cromwell.core.path.{PathBuilder, PathBuilderFactory, PathFactory}
 import cromwell.engine._
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.instrumentation.WorkflowInstrumentation
@@ -217,6 +217,8 @@ class WorkflowActor(workflowToStart: WorkflowToStart,
   private val workflowDockerLookupActor = context.actorOf(
     WorkflowDockerLookupActor.props(workflowId, dockerHashActor, initialStartableState.restarted), s"WorkflowDockerLookupActor-$workflowId")
 
+  protected val pathBuilderFactories: List[PathBuilderFactory] = EngineFilesystems.configuredPathBuilderFactories
+
   startWith(WorkflowUnstartedState, WorkflowActorData(initialStartableState))
 
   pushCurrentStateToMetadataService(workflowId, WorkflowUnstartedState.workflowState)
@@ -431,7 +433,10 @@ class WorkflowActor(workflowToStart: WorkflowToStart,
         def bruteForceWorkflowOptions: WorkflowOptions = WorkflowOptions.fromJsonString(sources.workflowOptionsJson).getOrElse(WorkflowOptions.fromJsonString("{}").get)
         val system = context.system
         val ec = context.system.dispatcher
-        def bruteForcePathBuilders: Future[List[PathBuilder]] = EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions)(system)
+        def bruteForcePathBuilders: Future[List[PathBuilder]] = {
+          // Protect against path builders that may throw an exception instead of returning a failed future
+          Future(EngineFilesystems.pathBuildersForWorkflow(bruteForceWorkflowOptions, pathBuilderFactories)(system))(ec).flatten
+        }
 
         val (workflowOptions, pathBuilders) = stateData.workflowDescriptor match {
           case Some(wd) => (wd.backendDescriptor.workflowOptions, Future.successful(wd.pathBuilders))
@@ -440,7 +445,9 @@ class WorkflowActor(workflowToStart: WorkflowToStart,
 
         workflowOptions.get(FinalWorkflowLogDir).toOption match {
           case Some(destinationDir) =>
-            pathBuilders.map(pb => workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pb)))(ec)
+            pathBuilders
+              .map(pb => workflowLogCopyRouter ! CopyWorkflowLogsActor.Copy(workflowId, PathFactory.buildPath(destinationDir, pb)))(ec)
+              .recover { case e => log.error(e, "Failed to copy workflow log") }(ec)
           case None => workflowLogger.close(andDelete = WorkflowLogger.isTemporary) match {
             case Failure(f) => log.error(f, "Failed to delete workflow log")
             case _ =>
diff --git a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala
index eb3d016665c..ffc6356bb58 100644
--- a/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala
+++ b/engine/src/main/scala/cromwell/engine/workflow/lifecycle/materialization/MaterializeWorkflowDescriptorActor.scala
@@ -13,7 +13,7 @@ import cats.syntax.either._
 import cats.syntax.traverse._
 import cats.syntax.validated._
 import com.typesafe.config.Config
-import com.typesafe.scalalogging.LazyLogging
+import com.typesafe.scalalogging.StrictLogging
 import common.exception.{AggregatedMessageException, MessageAggregation}
 import common.validation.Checked._
 import common.validation.ErrorOr._
@@ -26,7 +26,7 @@ import cromwell.core.callcaching._
 import cromwell.core.io.AsyncIo
 import cromwell.core.labels.{Label, Labels}
 import cromwell.core.logging.WorkflowLogging
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.{PathBuilder, PathBuilderFactory}
 import cromwell.engine._
 import cromwell.engine.backend.CromwellBackends
 import cromwell.engine.workflow.lifecycle.EngineLifecycleActorAbortCommand
@@ -137,7 +137,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
                                          workflowId: WorkflowId,
                                          cromwellBackends: => CromwellBackends,
                                          importLocalFilesystem: Boolean,
-                                         ioActorProxy: ActorRef) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, Unit] with LazyLogging with WorkflowLogging {
+                                         ioActorProxy: ActorRef) extends LoggingFSM[MaterializeWorkflowDescriptorActorState, Unit] with StrictLogging with WorkflowLogging {
 
   import MaterializeWorkflowDescriptorActor._
   val tag = self.path.name
@@ -148,6 +148,8 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
   val iOExecutionContext = context.system.dispatchers.lookup("akka.dispatchers.io-dispatcher")
   implicit val ec = context.dispatcher
 
+  protected val pathBuilderFactories: List[PathBuilderFactory] = EngineFilesystems.configuredPathBuilderFactories
+
   startWith(ReadyToMaterializeState, ())
 
   when(ReadyToMaterializeState) {
@@ -221,7 +223,7 @@ class MaterializeWorkflowDescriptorActor(serviceRegistryActor: ActorRef,
   private def workflowOptionsAndPathBuilders(sourceFiles: WorkflowSourceFilesCollection): ErrorOr[(WorkflowOptions, Future[List[PathBuilder]])] = {
     val workflowOptionsValidation = validateWorkflowOptions(sourceFiles.workflowOptionsJson)
     workflowOptionsValidation map { workflowOptions =>
-      val pathBuilders = EngineFilesystems.pathBuildersForWorkflow(workflowOptions)(context.system)
+      val pathBuilders = EngineFilesystems.pathBuildersForWorkflow(workflowOptions, pathBuilderFactories)(context.system)
       (workflowOptions, pathBuilders)
     }
   }
diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
index 3f4689ca588..54891849238 100644
--- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
+++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
@@ -82,7 +82,7 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single
     )
     val authCredentials = googleAuthMode.credentials(options.get(_).get, marthaScopes)
 
-    Future.successful(DrsPathBuilder(new DrsCloudNioFileSystemProvider(singletonConfig.config, authCredentials, httpClientBuilder, drsReadInterpreter)))
+    Future(DrsPathBuilder(new DrsCloudNioFileSystemProvider(singletonConfig.config, authCredentials, httpClientBuilder, drsReadInterpreter)))
   }
 }
 
diff --git a/server/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala b/server/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
index fa948815654..d9f2095cc86 100644
--- a/server/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
+++ b/server/src/test/scala/cromwell/engine/workflow/WorkflowActorSpec.scala
@@ -3,14 +3,14 @@ package cromwell.engine.workflow
 import java.time.OffsetDateTime
 import java.util.concurrent.atomic.AtomicInteger
 
-import akka.actor.{Actor, ActorRef, Props}
-import akka.testkit.{TestActorRef, TestFSMRef, TestProbe}
+import akka.actor.{Actor, ActorRef, ActorSystem, Props}
+import akka.testkit.{EventFilter, TestActorRef, TestFSMRef, TestProbe}
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell._
 import cromwell.backend.{AllBackendInitializationData, JobExecutionMap}
 import cromwell.core._
-import cromwell.core.path.DefaultPathBuilder
-import cromwell.engine.EngineWorkflowDescriptor
+import cromwell.core.path.{DefaultPathBuilder, PathBuilder, PathBuilderFactory}
+import cromwell.engine.{EngineFilesystems, EngineWorkflowDescriptor}
 import cromwell.engine.backend.BackendSingletonCollection
 import cromwell.engine.workflow.WorkflowActor._
 import cromwell.engine.workflow.lifecycle.EngineLifecycleActorAbortCommand
@@ -24,10 +24,16 @@ import cromwell.util.SampleWdl.ThreeStep
 import org.scalatest.BeforeAndAfter
 import org.scalatest.concurrent.Eventually
 
+import scala.concurrent.{ExecutionContext, Future}
 import scala.concurrent.duration._
 
 class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorBuilder with BeforeAndAfter with Eventually {
-  override implicit val actorSystem = system
+
+  // https://doc.akka.io/docs/akka/current/testing.html#expecting-log-messages
+  override implicit val actorSystem = ActorSystem(
+    "testsystem",
+    ConfigFactory.parseString("""akka.loggers = ["akka.testkit.TestEventListener"]""")
+  )
 
   val mockServiceRegistryActor = TestActorRef(new Actor {
     override def receive = {
@@ -57,9 +63,9 @@ class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorB
 
   private val workflowHeartbeatConfig = WorkflowHeartbeatConfig(ConfigFactory.load())
 
-  private def createWorkflowActor(state: WorkflowActorState) = {
+  private def createWorkflowActor(state: WorkflowActorState, extraPathBuilderFactory: Option[PathBuilderFactory] = None) = {
     val actor = TestFSMRef(
-      factory = new MockWorkflowActor(
+      factory = new WorkflowActorWithTestAddons(
         finalizationProbe = finalizationProbe,
         workflowId = currentWorkflowId,
         startState = Submitted,
@@ -76,7 +82,8 @@ class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorB
         jobTokenDispenserActor = TestProbe().ref,
         workflowStoreActor = system.actorOf(Props.empty),
         workflowHeartbeatConfig = workflowHeartbeatConfig,
-        totalJobsByRootWf = initialJobCtByRootWf
+        totalJobsByRootWf = initialJobCtByRootWf,
+        extraPathBuilderFactory = extraPathBuilderFactory
       ),
       supervisor = supervisorProbe.ref)
     actor.setState(stateName = state, stateData = WorkflowActorData(Option(currentLifecycleActor.ref), Option(descriptor),
@@ -179,26 +186,51 @@ class WorkflowActorSpec extends CromwellTestKitWordSpec with WorkflowDescriptorB
       finalizationProbe.expectNoMessage(AwaitAlmostNothing)
       deathwatch.expectTerminated(actor)
     }
+
+    "log an error when a path builder factory initialization fails" in {
+      EventFilter.error(start = "Failed to copy workflow log", pattern = ".*Failing as requested.*", occurrences = 1).intercept {
+        val _ = createWorkflowActor(WorkflowSucceededState, Option(new FailingPathBuilderFactory()))
+      }
+    }
+
+    "log an error when a path builder factory initialization throws" in {
+      EventFilter.error(start = "Failed to copy workflow log", pattern = ".*Throwing as requested.*", occurrences = 1).intercept {
+        val _ = createWorkflowActor(WorkflowSucceededState, Option(new ThrowingPathBuilderFactory()))
+      }
+    }
   }
 }
 
-class MockWorkflowActor(val finalizationProbe: TestProbe,
-                        workflowId: WorkflowId,
-                        startState: StartableState,
-                        workflowSources: WorkflowSourceFilesCollection,
-                        conf: Config,
-                        ioActor: ActorRef,
-                        serviceRegistryActor: ActorRef,
-                        workflowLogCopyRouter: ActorRef,
-                        jobStoreActor: ActorRef,
-                        subWorkflowStoreActor: ActorRef,
-                        callCacheReadActor: ActorRef,
-                        callCacheWriteActor: ActorRef,
-                        dockerHashActor: ActorRef,
-                        jobTokenDispenserActor: ActorRef,
-                        workflowStoreActor: ActorRef,
-                        workflowHeartbeatConfig: WorkflowHeartbeatConfig,
-                        totalJobsByRootWf: AtomicInteger) extends WorkflowActor(
+class FailingPathBuilderFactory() extends PathBuilderFactory {
+  override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = {
+    Future(throw new Exception("Failing as requested"))
+  }
+}
+
+class ThrowingPathBuilderFactory() extends PathBuilderFactory {
+  override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] = {
+    throw new Exception("Throwing as requested")
+  }
+}
+
+class WorkflowActorWithTestAddons(val finalizationProbe: TestProbe,
+                                  workflowId: WorkflowId,
+                                  startState: StartableState,
+                                  workflowSources: WorkflowSourceFilesCollection,
+                                  conf: Config,
+                                  ioActor: ActorRef,
+                                  serviceRegistryActor: ActorRef,
+                                  workflowLogCopyRouter: ActorRef,
+                                  jobStoreActor: ActorRef,
+                                  subWorkflowStoreActor: ActorRef,
+                                  callCacheReadActor: ActorRef,
+                                  callCacheWriteActor: ActorRef,
+                                  dockerHashActor: ActorRef,
+                                  jobTokenDispenserActor: ActorRef,
+                                  workflowStoreActor: ActorRef,
+                                  workflowHeartbeatConfig: WorkflowHeartbeatConfig,
+                                  totalJobsByRootWf: AtomicInteger,
+                                  extraPathBuilderFactory: Option[PathBuilderFactory]) extends WorkflowActor(
   workflowToStart = WorkflowToStart(id = workflowId,
     submissionTime = OffsetDateTime.now,
     state = startState,
@@ -221,5 +253,10 @@ class MockWorkflowActor(val finalizationProbe: TestProbe,
   fileHashCacheActor = None,
   blacklistCache = None) {
 
+  override val pathBuilderFactories: List[PathBuilderFactory] = extraPathBuilderFactory match {
+    case Some(pbf) => EngineFilesystems.configuredPathBuilderFactories :+ pbf
+    case None => EngineFilesystems.configuredPathBuilderFactories
+  }
+
   override def makeFinalizationActor(workflowDescriptor: EngineWorkflowDescriptor, jobExecutionMap: JobExecutionMap, worfklowOutputs: CallOutputs) = finalizationProbe.ref
 }

From 0ded9d67e719ecb3a8ef4377d8b18122c4e812c5 Mon Sep 17 00:00:00 2001
From: Jeff Gentry <geoffjentry@gmail.com>
Date: Thu, 20 Jun 2019 16:22:45 -0400
Subject: [PATCH 19/28] Add BigQuery scopes to GCP [BA-5716] (#5023)

---
 CHANGELOG.md                                                  | 4 ++++
 .../backend/google/pipelines/v2alpha1/GenomicsFactory.scala   | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b2ea6c415fa..cb5bce63313 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,10 @@ select max(CALL_CACHING_HASH_ENTRY_ID) from CALL_CACHING_HASH_ENTRY
 Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). 
 Learn more on how to configure [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
 
+### BigQuery in PAPI
+
+Cromwell now allows a user to specify BigQuery jobs when using the PAPIv2 backend
+
 ### Configuration Changes
 
 #### StatsD Instrumentation
diff --git a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
index 6e970aa03aa..381e76cf133 100644
--- a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
+++ b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
@@ -4,6 +4,7 @@ import java.net.URL
 
 import cats.effect.IO
 import com.google.api.client.http.{HttpRequest, HttpRequestInitializer, HttpResponse}
+import com.google.api.services.bigquery.BigqueryScopes
 import com.google.api.services.cloudresourcemanager.CloudResourceManager
 import com.google.api.services.compute.ComputeScopes
 import com.google.api.services.genomics.v2alpha1.model._
@@ -154,6 +155,8 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
             Oauth2Scopes.USERINFO_PROFILE,
             // Monitoring scope as POC
             GenomicsFactory.MonitoringWrite,
+            // Allow read/write with BigQuery
+            BigqueryScopes.BIGQUERY
           ).asJava
         )
 

From 5142bf1f0dfe2bd90dc20ef75a0fc6ac151263d5 Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Fri, 21 Jun 2019 21:16:51 +0200
Subject: [PATCH 20/28] Add cached-copy localization strategy. (#4900)

---
 CHANGELOG.md                                  |  5 ++
 .../cached_copy/cached_copy.wdl               | 55 ++++++++++++
 docs/backends/HPC.md                          | 11 ++-
 .../backend/sfs/SharedFileSystem.scala        | 88 +++++++++++++++++++
 ...haredFileSystemJobCachingActorHelper.scala |  8 +-
 .../backend/sfs/SharedFileSystemSpec.scala    | 48 ++++++++++
 6 files changed, 213 insertions(+), 2 deletions(-)
 create mode 100644 centaur/src/main/resources/standardTestCases/cached_copy/cached_copy.wdl

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cb5bce63313..8625d4fa70d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,11 @@ There is a small change in StatsD's configuration path. Originally, the path to
 which now has been updated to `services.Instrumentation.config`. More info on its configuration can be found
 [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/).
 
+#### cached-copy
+
+A new experimental feature, the `cached-copy` localization strategy is available for the shared filesystem. 
+More information can be found in the [documentation on localization](https://cromwell.readthedocs.io/en/stable/backends/HPC).
+
 ### API Changes
 
 #### Workflow Metadata
diff --git a/centaur/src/main/resources/standardTestCases/cached_copy/cached_copy.wdl b/centaur/src/main/resources/standardTestCases/cached_copy/cached_copy.wdl
new file mode 100644
index 00000000000..a683b01df0e
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/cached_copy/cached_copy.wdl
@@ -0,0 +1,55 @@
+version 1.0
+
+workflow cached_inputs {
+    Array[Int] one_to_ten = [1,2,3,4,5,6,7,8,9,10]
+
+    call ten_lines
+
+    scatter (x in one_to_ten) {
+        call read_line {
+            input:
+                file=ten_lines.text,
+                line_number=x
+        }
+    }
+    output {
+        Array[String] lines = read_line.line
+    }
+}
+
+task ten_lines {
+    command {
+        echo "Line 1
+        Line 2
+        Line 3
+        Line 4
+        Line 5
+        Line 6
+        Line 7
+        Line 8
+        Line 9
+        Line 10" > outfile.txt
+    }
+    output {
+        File text = "outfile.txt"
+    }
+    runtime {
+      docker: "ubuntu:latest"
+    }
+}
+
+task read_line {
+    input {
+        File file
+        Int line_number
+    }
+    command {
+        sed -n ~{line_number}p ~{file}
+    }
+    output {
+        String line = read_string(stdout())
+    }
+    runtime {
+      docker: "ubuntu:latest"
+    }
+}
\ No newline at end of file
diff --git a/docs/backends/HPC.md b/docs/backends/HPC.md
index 80c0969e344..ef4de8d040e 100644
--- a/docs/backends/HPC.md
+++ b/docs/backends/HPC.md
@@ -18,11 +18,20 @@ Cromwell is configured with a root execution directory which is set in the confi
 When Cromwell runs a workflow, it first creates a directory `<cromwell_root>/<workflow_uuid>`.  This is called the `workflow_root` and it is the root directory for all activity in this workflow.
 
 Each `call` has its own subdirectory located at `<workflow_root>/call-<call_name>`.  This is the `<call_dir>`.
-Any input files to a call need to be localized into the `<call_dir>/inputs` directory. There are different localization strategies that Cromwell will try until one works. Below is the default order specified in `reference.conf` but it can be overridden:
+Any input files to a call need to be localized into the `<call_dir>/inputs` directory. There are different localization strategies that Cromwell will try until one works:
 
 * `hard-link` - This will create a hard link to the file
 * `soft-link` - Create a symbolic link to the file. This strategy is not applicable for tasks which specify a Docker image and will be ignored.
 * `copy` - Make a copy the file
+* `cached-copy` An experimental feature. This copies files to a file cache in 
+`<workflow_root>/cached-inputs` and then hard links them in the `<call_dir>/inputs` directory. 
+
+`cached-copy` is intended for a shared filesystem that runs on multiple physical disks, where docker containers are used. 
+Hard-links don't work between different physical disks and soft-links don't work with docker. Copying uses a lot of
+space if a multitude of tasks use the same input. `cached-copy` copies the file only once to the physical disk containing
+the `<workflow_root>` and then uses hard links for every task that needs the input file. This can save a lot of space.
+
+The default order in `reference.conf` is `hard-link`, `soft-link`, `copy`
 
 Shared filesystem localization is defined in the `config` section of each backend. The default stanza for the Local and HPC backends looks like this:
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index 70a3ab37507..850e2e5871d 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -18,6 +18,7 @@ import wom.WomFileMapper
 import wom.values._
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 import scala.concurrent.Await
 import scala.concurrent.duration.Duration
 import scala.language.postfixOps
@@ -92,6 +93,14 @@ object SharedFileSystem extends StrictLogging {
       TryUtil.sequence(attempts, s"Could not $description $source -> $dest").void
     }
   }
+
+  private lazy val beingCopied: mutable.Map[Path, Boolean] = mutable.Map[Path, Boolean]()
+
+  private def waitOnCopy(path: Path, lockFile: Path): Unit = {
+    while (beingCopied.getOrElse(path, false) || lockFile.exists)  {
+      Thread.sleep(1)
+    }
+  }
 }
 
 trait SharedFileSystem extends PathFactory {
@@ -99,6 +108,82 @@ trait SharedFileSystem extends PathFactory {
 
   def sharedFileSystemConfig: Config
 
+  lazy val cachedCopyDir: Option[Path] = None
+
+  private def localizePathViaCachedCopy(originalPath: Path, executionPath: Path, docker: Boolean): Try[Unit] = {
+    val action = Try {
+      createParentDirectory(executionPath, docker)
+      // Hash the parent. This will make sure bamfiles and their indexes stay in the same dir. This is not ideal. But should work.
+      // There should be no file collisions because two files with the same name cannot exist in the same parent dir.
+      // use .get . This strategy should not be used when there is no cachedCopyDir
+      val cachedCopySubDir: Path = cachedCopyDir.get.createChild(originalPath.toAbsolutePath.parent.hashCode.toString, asDirectory = true)
+
+      // By prepending the modtime we prevent collisions in the cache from files that have changed in between.
+      // Md5 is safer but much much slower and way too CPU intensive for big files.
+      val pathAndModTime: String = originalPath.lastModifiedTime.toEpochMilli.toString + originalPath.name
+      val cachedCopyPath: Path = cachedCopySubDir./(pathAndModTime)
+      val cachedCopyPathLockFile: Path = cachedCopyPath.plusSuffix(".lock")
+
+      if (!cachedCopyPath.exists) {
+        // This variable is used so we can release the lock before we start with the copying.
+        var shouldCopy = false
+
+        // LOCK: This block should only be accessed by one thread at a time. Otherwise multiple threads
+        // will copy the same cache file. The performance impact should be minimal as it is only a few
+        // decisions which are not time consuming.
+        SharedFileSystem.synchronized {
+
+          // We check again if cachedCopyPath is there. It may have been created while waiting on the lock.
+          // If it is not there, is it already being copied by another thread?
+          // if not copied by another thread, is it copied by another cromwell process? (Lock file present)
+          if (!cachedCopyPath.exists &&
+            !SharedFileSystem.beingCopied.getOrElse(cachedCopyPath, false) &&
+            !cachedCopyPathLockFile.exists) {
+            // Create a lock file so other cromwell processes know copying has started
+            try {
+              cachedCopyPathLockFile.touch()
+              // Create an entry in the dictionary so other threads in this process know copying
+              // has started (filesystem can be too slow if multiple threads need the same file at
+              // exactly the same time)
+              SharedFileSystem.beingCopied(cachedCopyPath) = true
+              // Set should copy to true for *this* thread. The lock can now be released.
+              // this thread will do the copying. The other threads and/or processes will wait for the copying to
+              // be completed.
+              shouldCopy = true
+            } catch {
+              case e: Exception =>
+                // In case any error happens, make sure that all locks are removed. Otherwise cromwell will hang in the
+                // future.
+                SharedFileSystem.beingCopied.remove(cachedCopyPath)
+                cachedCopyPathLockFile.delete(true)
+                throw e
+            }
+          }
+        } // LOCK RELEASE
+        if (shouldCopy) {
+          try {
+            val cachedCopyTmpPath = cachedCopyPath.plusExt("tmp")
+            originalPath.copyTo(cachedCopyTmpPath, overwrite = true).moveTo(cachedCopyPath)
+          } catch {
+            case e: Exception => throw e
+          }
+          finally {
+            // Always remove the locks after copying. Even if there is an exception.
+            // We remove the key! Not set it to false. We don't want this map being flooded with
+            // keys if the cromwell process is active for months in server mode. (Memory leak!)
+            SharedFileSystem.beingCopied.remove(cachedCopyPath)
+            cachedCopyPathLockFile.delete()
+          }
+        }
+      }
+      // If the file does exist, it may still be copied/moved and therefore not all the bits might
+      // be there. Therefore we always wait until the locks are cleared.
+      SharedFileSystem.waitOnCopy(cachedCopyPath, cachedCopyPathLockFile)
+      cachedCopyPath.linkTo(executionPath)
+    }.void
+    logOnFailure(action, "cached copy file")
+  }
+
   implicit def actorContext: ActorContext
 
   lazy val DefaultStrategies = Seq("hard-link", "soft-link", "copy")
@@ -120,8 +205,10 @@ trait SharedFileSystem extends PathFactory {
 
   private def createStrategies(configStrategies: Seq[String], docker: Boolean): Seq[DuplicationStrategy] = {
     // If localizing for a docker job, remove soft-link as an option
+    // If no cachedCopyDir is defined, cached-copy can not be used and is removed.
     val filteredConfigStrategies = configStrategies filter {
       case "soft-link" if docker => false
+      case "cached-copy" if cachedCopyDir.isEmpty => false
       case _ => true
     }
 
@@ -130,6 +217,7 @@ trait SharedFileSystem extends PathFactory {
       case "hard-link" => localizePathViaHardLink _
       case "soft-link" => localizePathViaSymbolicLink _
       case "copy" => localizePathViaCopy _
+      case "cached-copy" => localizePathViaCachedCopy _
       case unsupported => throw new UnsupportedOperationException(s"Strategy $unsupported is not recognized")
     }
 
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
index 6fce3777bef..2897d8a1ac1 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystemJobCachingActorHelper.scala
@@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorContext}
 import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.backend.standard.StandardCachingActorHelper
 import cromwell.core.logging.JobLogging
-import cromwell.core.path.PathBuilder
+import cromwell.core.path.{Path, PathBuilder}
 import net.ceedubs.ficus.Ficus._
 
 trait SharedFileSystemJobCachingActorHelper extends StandardCachingActorHelper {
@@ -19,5 +19,11 @@ trait SharedFileSystemJobCachingActorHelper extends StandardCachingActorHelper {
     override lazy val sharedFileSystemConfig: Config = {
       configurationDescriptor.backendConfig.as[Option[Config]]("filesystems.local").getOrElse(ConfigFactory.empty())
     }
+
+    // cachedCopyDir should be on the same physical filesystem as the execution root.
+    // WDL workflow names may not contain '-' so using 'cached-inputs' will certainly
+    // not collide with any workflows in the root directory.
+    override lazy val cachedCopyDir: Option[Path] = Option(
+      workflowPaths.executionRoot.createChild("cached-inputs", asDirectory = true))
   }
 }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
index e2ef7b06bea..6a4f000dc9b 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
@@ -10,6 +10,8 @@ import org.scalatest.{FlatSpec, Matchers}
 import org.specs2.mock.Mockito
 import wom.values.WomSingleFile
 
+import scala.io.Source
+
 class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with TableDrivenPropertyChecks with BackendSpec {
 
   behavior of "SharedFileSystem"
@@ -17,6 +19,7 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
   val defaultLocalization = ConfigFactory.parseString(""" localization: [copy, hard-link, soft-link] """)
   val hardLinkLocalization = ConfigFactory.parseString(""" localization: [hard-link] """)
   val softLinkLocalization = ConfigFactory.parseString(""" localization: [soft-link] """)
+  val cachedCopyLocalization = ConfigFactory.parseString(""" localization: [cached-copy] """)
   val localPathBuilder = List(DefaultPathBuilder)
 
 
@@ -25,14 +28,21 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
                        fileInCallDir: Boolean = false,
                        fileAlreadyExists: Boolean = false,
                        symlink: Boolean = false,
+                       cachedCopy: Boolean = false,
                        linkNb: Int = 1) = {
     val callDir = DefaultPathBuilder.createTempDirectory("SharedFileSystem")
     val orig = if (fileInCallDir) callDir.createChild("inputFile") else DefaultPathBuilder.createTempFile("inputFile")
     val dest = if (fileInCallDir) orig else callDir./(orig.parent.pathAsString.hashCode.toString())./(orig.name)
+    val testText =
+      """This is a simple text to check if the localization
+        | works correctly for the file contents.
+        |""".stripMargin
     orig.touch()
+    orig.writeText(testText)
     if (fileAlreadyExists) {
       dest.parent.createPermissionedDirectories()
       dest.touch()
+      dest.writeText(testText)
     }
 
     val inputs = fqnWdlMapToDeclarationMap(Map("input" -> WomSingleFile(orig.pathAsString)))
@@ -40,7 +50,10 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
       override val pathBuilders = localPathBuilder
       override val sharedFileSystemConfig = config
       override implicit def actorContext: ActorContext = null
+      override lazy val cachedCopyDir = Some(DefaultPathBuilder.createTempDirectory("cached-copy"))
     }
+    val cachedFile: Option[Path] = sharedFS.cachedCopyDir.map(
+      _./(orig.parent.pathAsString.hashCode.toString)./(orig.lastModifiedTime.toEpochMilli.toString + orig.name))
     val localizedinputs = Map(inputs.head._1 -> WomSingleFile(dest.pathAsString))
     val result = sharedFS.localizeInputs(callDir, docker = docker)(inputs)
 
@@ -48,9 +61,11 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
     result.get.toList should contain theSameElementsAs localizedinputs
 
     dest.exists shouldBe true
+    Source.fromFile(dest.toFile).mkString shouldBe testText
     countLinks(dest) should be(linkNb)
     isSymLink(dest) should be(symlink)
 
+    cachedFile.foreach(_.exists should be(cachedCopy))
     orig.delete(swallowIOExceptions = true)
     dest.delete(swallowIOExceptions = true)
   }
@@ -79,6 +94,11 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
     localizationTest(softLinkLocalization, docker = false, symlink = true)
   }
 
+  it should "localize a file via cached copy" in {
+    localizationTest(cachedCopyLocalization, docker = false, cachedCopy = true, linkNb = 2)
+    localizationTest(cachedCopyLocalization, docker = true, cachedCopy = true, linkNb = 2)
+  }
+
   it should "throw a fatal exception if localization fails" in {
     val callDir = DefaultPathBuilder.createTempDirectory("SharedFileSystem")
     val orig = DefaultPathBuilder.get("/made/up/origin")
@@ -94,6 +114,34 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
     result.failed.get.isInstanceOf[CromwellFatalExceptionMarker] shouldBe true
   }
 
+  it should "cache only one file if copied multiple times via cached copy" in {
+    val callDirs: List[Path] = List(1,2,3).map(x => DefaultPathBuilder.createTempDirectory("SharedFileSystem"))
+    val orig = DefaultPathBuilder.createTempFile("inputFile")
+    val dests = callDirs.map(_./(orig.parent.pathAsString.hashCode.toString())./(orig.name))
+    orig.touch()
+    val inputs = fqnWdlMapToDeclarationMap(Map("input" -> WomSingleFile(orig.pathAsString)))
+    val sharedFS = new SharedFileSystem {
+      override val pathBuilders = localPathBuilder
+      override val sharedFileSystemConfig = cachedCopyLocalization
+      override implicit def actorContext: ActorContext = null
+      override lazy val cachedCopyDir = Some(DefaultPathBuilder.createTempDirectory("cached-copy"))
+    }
+    val cachedFile: Option[Path] = sharedFS.cachedCopyDir.map(
+      _./(orig.parent.pathAsString.hashCode.toString)./(orig.lastModifiedTime.toEpochMilli.toString + orig.name))
+
+    val results = callDirs.map(sharedFS.localizeInputs(_, docker = true)(inputs))
+
+    results.foreach(_.isSuccess shouldBe true)
+    dests.foreach(_.exists shouldBe true)
+    dests.foreach(countLinks(_) shouldBe 4)
+
+    cachedFile.foreach(_.exists shouldBe true)
+    cachedFile.foreach(countLinks(_) shouldBe 4)
+    orig.delete(swallowIOExceptions = true)
+    dests.foreach(_.delete(swallowIOExceptions = true))
+  }
+
+
   private[this] def countLinks(file: Path): Int = file.getAttribute("unix:nlink").asInstanceOf[Int]
 
   private[this] def isSymLink(file: Path): Boolean = file.isSymbolicLink

From 778b201ff51e4f1d5b8dd6c38395a264c2385285 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Fri, 21 Jun 2019 16:00:28 -0400
Subject: [PATCH 21/28] Yaml DOS checks

---
 CHANGELOG.md                                  |   6 +
 .../standardTestCases/empty_inputs_file.test  |   2 +-
 .../scala/centaur/cwl/CloudPreprocessor.scala |   5 +-
 .../test/scala/CloudPreprocessorSpec.scala    |   5 +-
 cwl/src/main/scala/cwl/CommandLineTool.scala  |   4 +-
 .../scala/cwl/CwlExecutableValidation.scala   |   5 +-
 .../cwl/preprocessor/CwlPreProcessor.scala    |   4 +-
 docs/Configuring.md                           |  20 +-
 .../webservice/PartialWorkflowSources.scala   |  11 +-
 project/Dependencies.scala                    |  10 +-
 wom/src/main/resources/reference.conf         |   4 +
 wom/src/main/scala/wom/util/YamlUtils.scala   | 105 +++++++++++
 .../test/scala/wom/util/YamlUtilsSpec.scala   | 176 ++++++++++++++++++
 13 files changed, 334 insertions(+), 23 deletions(-)
 create mode 100644 wom/src/main/resources/reference.conf
 create mode 100644 wom/src/main/scala/wom/util/YamlUtils.scala
 create mode 100644 wom/src/test/scala/wom/util/YamlUtilsSpec.scala

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8625d4fa70d..5f1ca5382c6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -37,6 +37,12 @@ which now has been updated to `services.Instrumentation.config`. More info on it
 A new experimental feature, the `cached-copy` localization strategy is available for the shared filesystem. 
 More information can be found in the [documentation on localization](https://cromwell.readthedocs.io/en/stable/backends/HPC).
 
+#### Yaml node limits
+
+Yaml parsing now checks for cycles, and limits the maximum number of parsed nodes to a configurable value. See
+[the documentation on configuring Yaml](https://cromwell.readthedocs.io/en/stable/Configuring/#yaml) for more
+information.
+
 ### API Changes
 
 #### Workflow Metadata
diff --git a/centaur/src/main/resources/standardTestCases/empty_inputs_file.test b/centaur/src/main/resources/standardTestCases/empty_inputs_file.test
index dbda16cdcab..d9cb0fb972d 100644
--- a/centaur/src/main/resources/standardTestCases/empty_inputs_file.test
+++ b/centaur/src/main/resources/standardTestCases/empty_inputs_file.test
@@ -10,6 +10,6 @@ submit {
   statusCode: 400
   message: """{
   "status": "fail",
-  "message": "Error(s): Input file is not a valid yaml or json. Inputs data: ''. Error: MatchError: null."
+  "message": "Error(s): Input file is not a valid yaml or json. Inputs data: ''. Error: ParsingFailure: null."
 }"""
 }
diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala
index 4e8b90e1316..68e79b9b701 100644
--- a/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala
+++ b/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala
@@ -1,6 +1,7 @@
 package centaur.cwl
 import better.files.File
 import com.typesafe.config.Config
+import common.util.StringUtil._
 import common.validation.IOChecked.IOChecked
 import cwl.preprocessor.CwlPreProcessor
 import io.circe.optics.JsonPath
@@ -8,7 +9,7 @@ import io.circe.optics.JsonPath._
 import io.circe.yaml.Printer.StringStyle
 import io.circe.{Json, yaml}
 import net.ceedubs.ficus.Ficus._
-import common.util.StringUtil._
+import wom.util.YamlUtils
 
 /**
   * Tools to pre-process the CWL workflows and inputs before feeding them to Cromwell so they can be executed on PAPI.
@@ -39,7 +40,7 @@ class CloudPreprocessor(config: Config, prefixConfigPath: String) {
 
   // Parse value, apply f to it, and print it back to String using the printer
   private def process(value: String, f: Json => Json, printer: Json => String) = {
-    yaml.parser.parse(value) match {
+    YamlUtils.parse(value) match {
       case Left(error) => throw new Exception(error.getMessage)
       case Right(json) => printer(f(json))
     }
diff --git a/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala b/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala
index b8eb3d47818..513efa07407 100644
--- a/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala
+++ b/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala
@@ -1,6 +1,7 @@
 import centaur.cwl.CloudPreprocessor
 import com.typesafe.config.ConfigFactory
 import org.scalatest.{FlatSpec, Matchers}
+import wom.util.YamlUtils
 
 class CloudPreprocessorSpec extends FlatSpec with Matchers {
   behavior of "PAPIPreProcessor"
@@ -8,8 +9,8 @@ class CloudPreprocessorSpec extends FlatSpec with Matchers {
   val pAPIPreprocessor = new CloudPreprocessor(ConfigFactory.load(), "papi.default-input-gcs-prefix")
   
   def validate(result: String, expectation: String) = {
-    val parsedResult = io.circe.yaml.parser.parse(result).right.get
-    val parsedExpectation = io.circe.yaml.parser.parse(expectation).right.get
+    val parsedResult = YamlUtils.parse(result).right.get
+    val parsedExpectation = YamlUtils.parse(expectation).right.get
 
     // This is an actual Json comparison from circe
     parsedResult shouldBe parsedExpectation
diff --git a/cwl/src/main/scala/cwl/CommandLineTool.scala b/cwl/src/main/scala/cwl/CommandLineTool.scala
index e720ebe62b8..21382cc1b99 100644
--- a/cwl/src/main/scala/cwl/CommandLineTool.scala
+++ b/cwl/src/main/scala/cwl/CommandLineTool.scala
@@ -17,6 +17,7 @@ import wom.callable.{Callable, CallableTaskDefinition, ContainerizedInputExpress
 import wom.expression.{IoFunctionSet, ValueAsAnExpression, WomExpression}
 import wom.graph.GraphNodePort.OutputPort
 import wom.types.{WomArrayType, WomIntegerType, WomOptionalType}
+import wom.util.YamlUtils
 import wom.values.{WomArray, WomEvaluatedCallInputs, WomGlobFile, WomInteger, WomString, WomValue}
 import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys}
 
@@ -167,8 +168,9 @@ case class CommandLineTool private(
 
     // Parse content as json and return output values for each output port
     def parseContent(content: String): EvaluatedOutputs = {
+      val yaml = YamlUtils.parse(content)
       for {
-        parsed <- io.circe.yaml.parser.parse(content).flatMap(_.as[Map[String, Json]]).leftMap(error => NonEmptyList.one(error.getMessage))
+        parsed <- yaml.flatMap(_.as[Map[String, Json]]).leftMap(error => NonEmptyList.one(error.getMessage))
         jobOutputsMap <- jsonToOutputs(parsed)
       } yield jobOutputsMap.toMap
     }
diff --git a/cwl/src/main/scala/cwl/CwlExecutableValidation.scala b/cwl/src/main/scala/cwl/CwlExecutableValidation.scala
index 8df69890ba7..f6f8237c5ff 100644
--- a/cwl/src/main/scala/cwl/CwlExecutableValidation.scala
+++ b/cwl/src/main/scala/cwl/CwlExecutableValidation.scala
@@ -3,18 +3,19 @@ package cwl
 import common.Checked
 import common.validation.Checked._
 import io.circe.Json
-import io.circe.yaml
 import wom.callable.{ExecutableCallable, TaskDefinition}
 import wom.executable.Executable
 import wom.executable.Executable.{InputParsingFunction, ParsedInputMap}
 import wom.expression.IoFunctionSet
+import wom.util.YamlUtils
 
 object CwlExecutableValidation {
 
   // Decodes the input file, and build the ParsedInputMap
   private val inputCoercionFunction: InputParsingFunction =
     inputFile => {
-      yaml.parser.parse(inputFile).flatMap(_.as[Map[String, Json]]) match {
+      val yaml = YamlUtils.parse(inputFile)
+      yaml.flatMap(_.as[Map[String, Json]]) match {
         case Left(error) => error.getMessage.invalidNelCheck[ParsedInputMap]
         case Right(inputValue) => inputValue.map({ case (key, value) => key -> value.foldWith(CwlJsonToDelayedCoercionFunction) }).validNelCheck
       }
diff --git a/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala b/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala
index 010296c01a2..3444259f742 100644
--- a/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala
+++ b/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala
@@ -14,6 +14,7 @@ import io.circe.optics.JsonPath._
 import io.circe.{Json, JsonNumber, JsonObject}
 import mouse.all._
 import org.slf4j.LoggerFactory
+import wom.util.YamlUtils
 
 import scala.concurrent.ExecutionContext
 
@@ -218,7 +219,8 @@ object CwlPreProcessor {
   }
 
   private [preprocessor] def parseYaml(in: String): IOChecked[Json] = {
-   io.circe.yaml.parser.parse(in).leftMap(error => NonEmptyList.one(error.message)).toIOChecked
+    val yaml = YamlUtils.parse(in)
+    yaml.leftMap(error => NonEmptyList.one(error.message)).toIOChecked
   }
 
   /**
diff --git a/docs/Configuring.md b/docs/Configuring.md
index 49eb9adaf3d..45bdec17872 100644
--- a/docs/Configuring.md
+++ b/docs/Configuring.md
@@ -297,7 +297,7 @@ url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true&serverTimezone=U
 
 Using this option does not alter your database's underlying timezone; rather, it causes Cromwell to "speak UTC" when communicating with the DB, and the DB server performs the conversion for you. 
 
-## Abort
+### Abort
 
 **Control-C (SIGINT) abort handler**
 
@@ -514,7 +514,7 @@ Cromwell writes one batch of workflow heartbeats at a time. While the internal q
 a configurable threshold then [instrumentation](developers/Instrumentation.md) may send a metric signal that the
 heartbeat load is above normal.
 
-This threshold may be configured the configuration value:
+This threshold may be configured via the configuration value:
 
 ```hocon
 system.workflow-heartbeats {
@@ -523,3 +523,19 @@ system.workflow-heartbeats {
 ```
 
 The default threshold value is 100, just like the default for the heartbeat batch size.
+
+### YAML
+
+Cromwell will throw an error when detecting cyclic loops in Yaml inputs. However one can craft small acyclic YAML
+documents that consume significant amounts of memory or cpu. To limit the amount of processing during parsing, there is
+a limit on the number of nodes parsed per YAML document.
+
+This limit may be configured via the configuration value:
+
+```hocon
+yaml {
+  max-nodes = 1000000
+}
+```
+
+The default limit is 1,000,000 nodes.
diff --git a/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala
index d96ec1f6770..7c2ae6e2e11 100644
--- a/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala
+++ b/engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala
@@ -2,7 +2,6 @@ package cromwell.webservice
 
 import java.net.URL
 
-import _root_.io.circe.yaml
 import akka.util.ByteString
 import cats.data.NonEmptyList
 import cats.data.Validated._
@@ -21,6 +20,7 @@ import org.slf4j.LoggerFactory
 import spray.json._
 import wdl.draft2.model.WorkflowJson
 import wom.core._
+import wom.util.YamlUtils
 
 import scala.util.{Failure, Success, Try}
 
@@ -141,7 +141,7 @@ object PartialWorkflowSources {
       val onHold: ErrorOr[Boolean] = getBooleanValue(workflowOnHoldKey).getOrElse(false.validNel)
 
       (unrecognized, workflowSourceFinal, workflowInputs, workflowInputsAux, workflowDependenciesFinal, onHold) mapN {
-        case (_, source, inputs, aux, dep, onHold) => PartialWorkflowSources(
+        case (_, source, inputs, aux, dep, onHoldActual) => PartialWorkflowSources(
           workflowSource = source,
           workflowUrl = workflowUrl,
           workflowRoot = getStringValue(WorkflowRootKey),
@@ -153,7 +153,7 @@ object PartialWorkflowSources {
           customLabels = getStringValue(labelsKey),
           zippedImports = dep,
           warnings = wdlSourceWarning.toVector ++ wdlDependenciesWarning.toVector,
-          workflowOnHold = onHold
+          workflowOnHold = onHoldActual
         )
       }
     }
@@ -169,7 +169,7 @@ object PartialWorkflowSources {
     import cats.syntax.validated._
 
     val parseInputsTry = Try {
-      yaml.parser.parse(data) match {
+      YamlUtils.parse(data) match {
         // If it's an array, treat each element as an individual input object, otherwise simply toString the whole thing
         case Right(json) => json.asArray.map(_.map(_.toString())).getOrElse(Vector(json.pretty(Printer.noSpaces))).validNel
         case Left(error) => s"Input file is not a valid yaml or json. Inputs data: '$data'. Error: ${ExceptionUtils.getMessage(error)}.".invalidNel
@@ -265,12 +265,11 @@ object PartialWorkflowSources {
 
   private def toMap(someInput: Option[String]): ErrorOr[Map[String, JsValue]] = {
     someInput match {
-      case Some(input: String) =>  {
+      case Some(input: String) =>
         Try(input.parseJson).toErrorOrWithContext(s"parse input: '$input', which is not a valid json. Please check for syntactical errors.") flatMap {
           case JsObject(inputMap) => inputMap.validNel
           case j: JsValue => s"Submitted input '$input' of type ${j.getClass.getSimpleName} is not a JSON object.".invalidNel
         }
-      }
       case None => Map.empty[String, JsValue].validNel
     }
   }
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index 54c0e1241f7..e7c23b5a7de 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -62,7 +62,7 @@ object Dependencies {
   private val paradiseV = "2.1.1"
   private val pegdownV = "1.6.0"
   private val rdf4jV = "2.4.2"
-  private val refinedV = "0.9.4"
+  private val refinedV = "0.9.8"
   private val rhinoV = "1.7.10"
   private val scalaGraphV = "1.12.5"
   private val scalaLoggingV = "3.9.2"
@@ -370,8 +370,7 @@ object Dependencies {
     "org.scalacheck" %% "scalacheck" % scalacheckV % Test,
     "com.github.mpilquist" %% "simulacrum" % simulacrumV,
     "commons-codec" % "commons-codec" % commonsCodecV,
-    "eu.timepit" %% "refined" % refinedV
-  )
+  ) ++ circeDependencies ++ refinedTypeDependenciesList
 
   val wdlDependencies = List(
     "commons-io" % "commons-io" % commonsIoV,
@@ -421,15 +420,14 @@ object Dependencies {
     "org.javadelight" % "delight-rhino-sandbox" % delightRhinoSandboxV,
     "org.scalamock" %% "scalamock" % scalamockV % Test,
     "commons-io" % "commons-io" % commonsIoV % Test
-  ) ++ circeDependencies ++ womDependencies ++ refinedTypeDependenciesList ++ betterFilesDependencies ++
-    owlApiDependencies
+  ) ++ betterFilesDependencies ++ owlApiDependencies
 
   val womtoolDependencies = catsDependencies ++ slf4jBindingDependencies
 
   val centaurCwlRunnerDependencies = List(
     "com.github.scopt" %% "scopt" % scoptV,
     "io.circe" %% "circe-optics" % circeOpticsV
-  ) ++ slf4jBindingDependencies ++ circeDependencies
+  ) ++ slf4jBindingDependencies
 
   val coreDependencies = List(
     "com.google.auth" % "google-auth-library-oauth2-http" % googleOauth2V,
diff --git a/wom/src/main/resources/reference.conf b/wom/src/main/resources/reference.conf
new file mode 100644
index 00000000000..08b0bd72e32
--- /dev/null
+++ b/wom/src/main/resources/reference.conf
@@ -0,0 +1,4 @@
+yaml {
+  # The maximum number of nodes (scalars + sequences + mappings) that will be parsed in a YAML
+  max-nodes = 1000000
+}
diff --git a/wom/src/main/scala/wom/util/YamlUtils.scala b/wom/src/main/scala/wom/util/YamlUtils.scala
new file mode 100644
index 00000000000..bd26913dd16
--- /dev/null
+++ b/wom/src/main/scala/wom/util/YamlUtils.scala
@@ -0,0 +1,105 @@
+package wom.util
+
+import java.io.StringReader
+
+import com.typesafe.config.ConfigException.BadValue
+import com.typesafe.config.{Config, ConfigFactory}
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.NonNegative
+import eu.timepit.refined.refineV
+import io.circe.{Json, ParsingFailure}
+import net.ceedubs.ficus.Ficus._
+import net.ceedubs.ficus.readers.ValueReader
+
+import scala.collection.JavaConverters._
+
+object YamlUtils {
+
+  private[util] implicit val refinedNonNegativeReader: ValueReader[Int Refined NonNegative] = {
+    (config: Config, path: String) => {
+      val int = config.getInt(path)
+      refineV[NonNegative](int) match {
+        case Left(error) => throw new BadValue(path, error)
+        case Right(refinedInt) => refinedInt
+      }
+    }
+  }
+
+  private val defaultMaxNodes = ConfigFactory.load().as[Int Refined NonNegative]("yaml.max-nodes")
+
+  /**
+    * Parses the yaml, detecting loops, and a maximum number of nodes.
+    *
+    * See: https://en.wikipedia.org/wiki/Billion_laughs_attack
+    *
+    * @param yaml     The yaml string.
+    * @param maxNodes Maximum number of yaml nodes to parse.
+    * @return The parsed yaml.
+    */
+  def parse(yaml: String, maxNodes: Int Refined NonNegative = defaultMaxNodes): Either[ParsingFailure, Json] = {
+    try {
+      val snakeYamlParser = new org.yaml.snakeyaml.Yaml()
+      val parsed = snakeYamlParser.load[AnyRef](new StringReader(yaml))
+      // Use identity comparisons to check if two nodes are the same and do NOT recurse into them checking equality
+      val identityHashMap = new java.util.IdentityHashMap[AnyRef, java.lang.Boolean]()
+      // Since we don't actually need the values, wrap the Map in a Set
+      val identitySet = java.util.Collections.newSetFromMap(identityHashMap)
+      searchForOversizedYaml(parsed, identitySet, maxNodes, new Counter)
+      io.circe.yaml.parser.parse(yaml)
+    } catch {
+      case exception: Exception =>
+        Left(ParsingFailure(exception.getMessage, exception))
+    }
+  }
+
+  /** A "pointer" reference to a mutable count. */
+  private class Counter {
+    var count = 0L
+  }
+
+  /**
+    * Looks for loops and large documents in yaml parsed by SnakeYaml.
+    *
+    * Possibly can be refactored if/when Circe switches to SnakeYaml-Engine (aka Yaml 1.2), as SY-E disables recursive
+    * key by default:
+    *
+    * - https://github.com/circe/circe-yaml/issues/46
+    * - https://bitbucket.org/asomov/snakeyaml/issues/432/aggressive-yaml-anchors-causing
+    * - https://bitbucket.org/asomov/snakeyaml-engine/commits/7573a8b4d551fc84521f4ac1234a361bfbc96698
+    *
+    * @param node        Current node to be evaluated.
+    * @param identitySet Previously seen nodes during this branch of cycle checking.
+    * @param maxNodes    The maximum number of nodes allowed to be traversed.
+    * @param counter     A counter tracking to the total number of nodes traversed during the entire traversal.
+    */
+  private def searchForOversizedYaml(node: AnyRef,
+                                     identitySet: java.util.Set[AnyRef],
+                                     maxNodes: Int Refined NonNegative,
+                                     counter: Counter): Unit = {
+    if (!identitySet.add(node)) {
+      throw new IllegalArgumentException("Loop detected")
+    }
+
+    counter.count += 1
+    if (counter.count > maxNodes.value) {
+      throw new IllegalArgumentException(s"Loop detection halted at $maxNodes nodes")
+    }
+
+    node match {
+      case iterable: java.lang.Iterable[AnyRef]@unchecked =>
+        iterable.asScala foreach {
+          searchForOversizedYaml(_, identitySet, maxNodes, counter)
+        }
+      case map: java.util.Map[AnyRef, AnyRef]@unchecked =>
+        map.asScala foreach {
+          case (key, value) =>
+            searchForOversizedYaml(key, identitySet, maxNodes, counter)
+            searchForOversizedYaml(value, identitySet, maxNodes, counter)
+        }
+      case _ => /* ignore scalars, only loop through Yaml sequences and mappings: https://yaml.org/spec/1.1/#id861435 */
+    }
+
+    identitySet.remove(node)
+    ()
+  }
+}
diff --git a/wom/src/test/scala/wom/util/YamlUtilsSpec.scala b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala
new file mode 100644
index 00000000000..e3565e14cec
--- /dev/null
+++ b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala
@@ -0,0 +1,176 @@
+package wom.util
+
+import com.typesafe.config.ConfigException.BadValue
+import com.typesafe.config.ConfigFactory
+import eu.timepit.refined.api.Refined
+import eu.timepit.refined.numeric.NonNegative
+import eu.timepit.refined.refineMV
+import io.circe.Json
+import net.ceedubs.ficus.Ficus._
+import org.scalatest.prop.TableDrivenPropertyChecks
+import org.scalatest.{EitherValues, FlatSpec, Matchers}
+
+class YamlUtilsSpec extends FlatSpec with Matchers with TableDrivenPropertyChecks with EitherValues {
+
+  behavior of "YamlUtils"
+
+  private val illegalYamlTests = Table(
+    ("description", "yaml", "maxNodes", "exceptionMessage"),
+    (
+      // https://stackoverflow.com/a/16986339
+      "a stackoverflow yaml bomb",
+      "{a: &b [*b]}",
+      refineMV[NonNegative](10),
+      "Loop detected",
+    ),
+    (
+      // https://bitbucket.org/asomov/snakeyaml/wiki/Documentation#markdown-header-aliases
+      "a recursive yaml example from the snakeyaml wiki",
+      "&A [ *A ]",
+      refineMV[NonNegative](10),
+      "Loop detected",
+    ),
+    (
+      "a recursive yaml based on mappings back to the root",
+      """|a: &b
+         |  c: *b
+         |""".stripMargin,
+      refineMV[NonNegative](10),
+      "Loop detected",
+    ),
+    (
+      // https://bitbucket.org/asomov/snakeyaml-engine/src/41b3845/src/test/resources/recursive/recursive-set-1.yaml
+      "a recursive yaml set",
+      """|# Sets are represented as a
+         |# Mapping where each key is
+         |# associated with a null value
+         |--- !!set &key
+         |? Mark McGwire
+         |? Sammy Sosa
+         |? *key
+         |""".stripMargin,
+      refineMV[NonNegative](10),
+      "Loop detected",
+    ),
+    (
+      // https://en.wikipedia.org/w/index.php?title=Billion_laughs_attack&oldid=871224525#Variations
+      "a billion laughs when limited to 10,000 nodes",
+      """|a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"]
+         |b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]
+         |c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]
+         |d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]
+         |e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]
+         |f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]
+         |g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]
+         |h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]
+         |i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]
+         |""".stripMargin,
+      refineMV[NonNegative](10000),
+      "Loop detection halted at 10000 nodes",
+    ),
+    (
+      "a null yaml",
+      null,
+      refineMV[NonNegative](0),
+      null,
+    ),
+    (
+      "an empty yaml",
+      "",
+      refineMV[NonNegative](1),
+      "null",
+    ),
+    (
+      "an empty yaml mapping when limited to zero nodes",
+      "{}",
+      refineMV[NonNegative](0),
+      "Loop detection halted at 0 nodes",
+    ),
+    (
+      "an empty yaml sequence when limited to zero nodes",
+      "[]",
+      refineMV[NonNegative](0),
+      "Loop detection halted at 0 nodes",
+    ),
+    (
+      "a yaml without a closing brace",
+      "{",
+      refineMV[NonNegative](10),
+      """|while parsing a flow node
+         | in 'reader', line 1, column 2:
+         |    {
+         |     ^
+         |expected the node content, but found '<stream end>'
+         | in 'reader', line 1, column 2:
+         |    {
+         |     ^
+         |""".stripMargin,
+    ),
+  )
+
+  private val legalYamlTests = Table(
+    ("description", "yaml", "maxNodes", "expectedJson"),
+    (
+      "an empty yaml mapping when limited to one node",
+      "{}",
+      refineMV[NonNegative](1),
+      Json.obj(),
+    ),
+    (
+      "an empty yaml sequence when limited to one node",
+      "[]",
+      refineMV[NonNegative](1),
+      Json.arr(),
+    ),
+    (
+      "a yaml with the same node for a key and value",
+      """|? &a b
+         |: *a
+         |""".stripMargin,
+      refineMV[NonNegative](3),
+      Json.obj("b" -> Json.fromString("b")),
+    ),
+    (
+      "a yaml with the same nodes in a sequence",
+      "[ &a b, *a ]",
+      refineMV[NonNegative](3),
+      Json.arr(Json.fromString("b"), Json.fromString("b")),
+    ),
+  )
+
+  forAll(illegalYamlTests) { (description, yaml, maxNodes, exceptionMessage) =>
+    it should s"fail to parse $description" in {
+      val exception: Exception = YamlUtils.parse(yaml, maxNodes).left.value
+      exception should have message exceptionMessage
+    }
+  }
+
+  forAll(legalYamlTests) { (description, yaml, maxNodes, expectedJson) =>
+    it should s"parse $description" in {
+      YamlUtils.parse(yaml, maxNodes) should be(Right(expectedJson))
+    }
+  }
+
+  it should "fail to parse a billion laughs with the default configuration" in {
+    val yaml =
+      """|a: &a ["lol","lol","lol","lol","lol","lol","lol","lol","lol"]
+         |b: &b [*a,*a,*a,*a,*a,*a,*a,*a,*a]
+         |c: &c [*b,*b,*b,*b,*b,*b,*b,*b,*b]
+         |d: &d [*c,*c,*c,*c,*c,*c,*c,*c,*c]
+         |e: &e [*d,*d,*d,*d,*d,*d,*d,*d,*d]
+         |f: &f [*e,*e,*e,*e,*e,*e,*e,*e,*e]
+         |g: &g [*f,*f,*f,*f,*f,*f,*f,*f,*f]
+         |h: &h [*g,*g,*g,*g,*g,*g,*g,*g,*g]
+         |i: &i [*h,*h,*h,*h,*h,*h,*h,*h,*h]
+         |""".stripMargin
+    val exception: Exception = YamlUtils.parse(yaml).left.value
+    exception should have message "Loop detection halted at 1000000 nodes" // <-- Updating here? Also get the docs too!
+  }
+
+  it should "not parse a config with a negative value" in {
+    import wom.util.YamlUtils.refinedNonNegativeReader
+    the[BadValue] thrownBy {
+      ConfigFactory.parseString("non-negative: -1").as[Int Refined NonNegative]("non-negative")
+    } should have message "Invalid value at 'non-negative': Predicate (-1 < 0) did not fail."
+  }
+}

From fe361736cfaeedcdfd996e084e7e571c0f304445 Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Fri, 21 Jun 2019 18:55:12 -0400
Subject: [PATCH 22/28] Virtual Private Cloud with subnetwork [BA-5676] (#5027)

---
 CHANGELOG.md                                  |  5 ++
 .../check_network_in_vpc.test                 |  3 +
 .../check_network_in_vpc.wdl                  | 16 +++-
 .../wf_zone_options.json                      |  5 ++
 docs/backends/Google.md                       |  7 +-
 src/ci/resources/papi_v2_application.conf     |  1 +
 .../PipelinesApiConfigurationAttributes.scala | 45 ++++++----
 ...elinesApiConfigurationAttributesSpec.scala | 90 ++++++++++++++++++-
 .../pipelines/v2alpha1/GenomicsFactory.scala  | 10 ++-
 9 files changed, 156 insertions(+), 26 deletions(-)
 create mode 100644 centaur/src/main/resources/standardTestCases/virtual_private_cloud/wf_zone_options.json

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8625d4fa70d..0388f17661e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,11 @@
 
 ## 43 Release Notes
 
+### Virtual Private Cloud with Subnetworks
+
+Cromwell now allows PAPIV2 jobs to run on a specific subnetwork inside a private network by adding the subnetwork key 
+`subnetwork-label-key` inside `virtual-private-cloud` in backend configuration. More info [here](https://cromwell.readthedocs.io/en/stable/backends/Google/).
+
 ### Call caching database refactoring
 
 Cromwell's `CALL_CACHING_HASH_ENTRY` primary key has been refactored to use a `BIGINT` datatype in place of the previous
diff --git a/centaur/src/main/resources/standardTestCases/check_network_in_vpc.test b/centaur/src/main/resources/standardTestCases/check_network_in_vpc.test
index 71359de57b7..2658b81d4d6 100644
--- a/centaur/src/main/resources/standardTestCases/check_network_in_vpc.test
+++ b/centaur/src/main/resources/standardTestCases/check_network_in_vpc.test
@@ -4,6 +4,7 @@ backends: [Papiv2-Virtual-Private-Cloud]
 
 files {
   workflow: virtual_private_cloud/check_network_in_vpc.wdl
+  options: virtual_private_cloud/wf_zone_options.json
 }
 
 metadata {
@@ -11,4 +12,6 @@ metadata {
   status: Succeeded
 
   "outputs.check_network_in_vpc.network_used": "cromwell-ci-vpc-network"
+  "outputs.check_network_in_vpc.subnetwork_used": "cromwell-ci-vpc-network"
+  "outputs.check_network_in_vpc.zone_used": "us-east1-c"
 }
diff --git a/centaur/src/main/resources/standardTestCases/virtual_private_cloud/check_network_in_vpc.wdl b/centaur/src/main/resources/standardTestCases/virtual_private_cloud/check_network_in_vpc.wdl
index 23b52aa9992..81111c3594f 100644
--- a/centaur/src/main/resources/standardTestCases/virtual_private_cloud/check_network_in_vpc.wdl
+++ b/centaur/src/main/resources/standardTestCases/virtual_private_cloud/check_network_in_vpc.wdl
@@ -2,12 +2,17 @@ version 1.0
 
 task get_network {
   command {
+    set -euo pipefail
+
     apt-get install --assume-yes jq > /dev/null
     INSTANCE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/name" -H "Metadata-Flavor: Google")
     ZONE=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/zone" -H "Metadata-Flavor: Google" | sed -E 's!.*/(.*)!\1!')
     TOKEN=$(gcloud auth application-default print-access-token)
     INSTANCE_METADATA=$(curl "https://www.googleapis.com/compute/v1/projects/broad-dsde-cromwell-dev/zones/$ZONE/instances/$INSTANCE" -H "Authorization: Bearer $TOKEN" -H 'Accept: application/json')
-    echo $INSTANCE_METADATA | jq -r '.networkInterfaces[0].network' | sed -E 's!.*/(.*)!\1!'
+    NETWORK_OBJECT=$(echo $INSTANCE_METADATA | jq --raw-output --exit-status '.networkInterfaces[0]')
+    echo $NETWORK_OBJECT | jq --exit-status '.network' | sed -E 's!.*/(.*)!\1!' | sed 's/"//g' > network
+    echo $NETWORK_OBJECT | jq --exit-status '.subnetwork' | sed -E 's!.*/(.*)!\1!' | sed 's/"//g' > subnetwork
+    echo $ZONE > zone
   }
 
   runtime {
@@ -16,7 +21,9 @@ task get_network {
   }
 
   output {
-    String network = read_string(stdout())
+    String networkName = read_string("network")
+    String subnetworkName = read_string("subnetwork")
+    String zone = read_string("zone")
   }
 }
 
@@ -24,6 +31,9 @@ workflow check_network_in_vpc {
   call get_network
 
    output {
-     String network_used = get_network.network
+     String network_used = get_network.networkName
+     String subnetwork_used = get_network.subnetworkName
+     String zone_used = get_network.zone
    }
 }
+
diff --git a/centaur/src/main/resources/standardTestCases/virtual_private_cloud/wf_zone_options.json b/centaur/src/main/resources/standardTestCases/virtual_private_cloud/wf_zone_options.json
new file mode 100644
index 00000000000..8343cd1484d
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/virtual_private_cloud/wf_zone_options.json
@@ -0,0 +1,5 @@
+{
+  "default_runtime_attributes": {
+    "zones": "us-east1-c"
+  }
+}
diff --git a/docs/backends/Google.md b/docs/backends/Google.md
index d8c06040715..2b4c60d87c6 100644
--- a/docs/backends/Google.md
+++ b/docs/backends/Google.md
@@ -278,6 +278,7 @@ backend {
   		...
   		virtual-private-cloud {
   	          network-label-key = "my-private-network"
+  	          subnetwork-label-key = "my-private-subnetwork"
   	          auth = "reference-to-auth-scheme"
   	        }
   	    ...
@@ -288,8 +289,10 @@ backend {
 ```
 
 
-The `network-label-key` should reference the key in the label in your project whose value is the name of your private network.
- `auth` should reference an auth scheme in the `google` stanza which should be used to the project metadata from Google Cloud.
+The `network-label-key` and `subnetwork-label-key` should reference the keys in your project's labels whose value is the name of your private network
+and subnetwork within that network respectively. `auth` should reference an auth scheme in the `google` stanza which will be used to get the project metadata from Google Cloud.
+The `subnetwork-label-key` is an optional config.
+
 For example, if your `virtual-private-cloud` config looks like the one above, and one of the labels in your project is
 
 ```
diff --git a/src/ci/resources/papi_v2_application.conf b/src/ci/resources/papi_v2_application.conf
index d639d61ab07..35b31d60d15 100644
--- a/src/ci/resources/papi_v2_application.conf
+++ b/src/ci/resources/papi_v2_application.conf
@@ -92,6 +92,7 @@ backend {
         filesystems.http {}
         virtual-private-cloud {
           network-label-key = "cromwell-ci-network"
+          subnetwork-label-key = "cromwell-ci-subnetwork"
           auth = "service_account"
         }
       }
diff --git a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributes.scala b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributes.scala
index 9ddfad147d2..b117228eb64 100644
--- a/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributes.scala
+++ b/supportedBackends/google/pipelines/common/src/main/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributes.scala
@@ -47,7 +47,7 @@ object PipelinesApiConfigurationAttributes {
     */
   case class LocalizationConfiguration(localizationAttempts: Int Refined Positive)
 
-  final case class VirtualPrivateCloudConfiguration(name: String, auth: GoogleAuthMode)
+  final case class VirtualPrivateCloudConfiguration(name: String, subnetwork: Option[String], auth: GoogleAuthMode)
   final case class BatchRequestTimeoutConfiguration(readTimeoutMillis: Option[Int Refined Positive], connectTimeoutMillis: Option[Int Refined Positive])
 
 
@@ -90,6 +90,7 @@ object PipelinesApiConfigurationAttributes {
     "default-runtime-attributes.zones",
     "virtual-private-cloud",
     "virtual-private-cloud.network-label-key",
+    "virtual-private-cloud.subnetwork-label-key",
     "virtual-private-cloud.auth"
   )
 
@@ -101,16 +102,18 @@ object PipelinesApiConfigurationAttributes {
 
   def apply(googleConfig: GoogleConfiguration, backendConfig: Config): PipelinesApiConfigurationAttributes = {
 
-    def validateVPCConfig(networkOption: Option[String], authOption: Option[String]): ErrorOr[Option[VirtualPrivateCloudConfiguration]] = {
-      (networkOption, authOption) match {
-        case (Some(network), Some(auth)) =>
-          googleConfig.auth(auth) match {
-            case Valid(validAuth) => Option(VirtualPrivateCloudConfiguration(network, validAuth)).validNel
-            case Invalid(error) => s"Auth $auth is not valid for Virtual Private Cloud configuration. Reason: $error" .invalidNel
-          }
-        case (Some(_), None) => "Auth scheme not provided for Virtual Private Cloud configuration.".invalidNel
-        case (None, Some(_)) => "Network label key not provided for Virtual Private Cloud configuration.".invalidNel
-        case (None, None) => None.validNel
+    def vpcErrorMessage(missingKeys: List[String]) = s"Virtual Private Cloud configuration is invalid. Missing keys: `${missingKeys.mkString(",")}`.".invalidNel
+
+    def validateVPCConfig(networkOption: Option[String], subnetworkOption: Option[String], authOption: Option[String]): ErrorOr[Option[VirtualPrivateCloudConfiguration]] = {
+      (networkOption,subnetworkOption, authOption) match {
+        case (Some(network), _, Some(auth)) => googleConfig.auth(auth) match {
+          case Valid(validAuth) => Option(VirtualPrivateCloudConfiguration(network, subnetworkOption, validAuth)).validNel
+          case Invalid(error) => s"Auth $auth is not valid for Virtual Private Cloud configuration. Reason: $error" .invalidNel
+        }
+        case (Some(_), _, None) => vpcErrorMessage(List("auth"))
+        case (None, _, Some(_)) => vpcErrorMessage(List("network-label-key"))
+        case (None, Some(_), None) => vpcErrorMessage(List("network-label-key", "auth"))
+        case (None, None, None) => None.validNel
       }
     }
 
@@ -154,12 +157,11 @@ object PipelinesApiConfigurationAttributes {
         .getOrElse(LocalizationConfiguration(DefaultLocalizationAttempts).validNel)
 
     val vpcNetworkLabel: ErrorOr[Option[String]] = validate { backendConfig.getAs[String]("virtual-private-cloud.network-label-key") }
+    val vpcSubnetworkLabel: ErrorOr[Option[String]] = validate { backendConfig.getAs[String]("virtual-private-cloud.subnetwork-label-key") }
     val vpcAuth: ErrorOr[Option[String]] = validate { backendConfig.getAs[String]("virtual-private-cloud.auth")}
 
     val virtualPrivateCloudConfiguration: ErrorOr[Option[VirtualPrivateCloudConfiguration]] = {
-      (vpcNetworkLabel, vpcAuth) flatMapN  {
-        case (network, auth) => validateVPCConfig(network, auth)
-      }
+      (vpcNetworkLabel, vpcSubnetworkLabel, vpcAuth) flatMapN  validateVPCConfig
     }
 
     val batchRequestsReadTimeout = readOptionalPositiveMillisecondsIntFromDuration(backendConfig, "batch-requests.timeouts.read")
@@ -201,8 +203,19 @@ object PipelinesApiConfigurationAttributes {
           )
     }
 
-    (project, executionBucket, endpointUrl, genomicsAuthName, genomicsRestrictMetadataAccess, gcsFilesystemAuthName,
-      qpsValidation, duplicationStrategy, requestWorkers, localizationConfiguration, virtualPrivateCloudConfiguration, batchRequestTimeoutConfigurationValidation) flatMapN authGoogleConfigForPapiConfigurationAttributes match {
+    (project,
+      executionBucket,
+      endpointUrl,
+      genomicsAuthName,
+      genomicsRestrictMetadataAccess,
+      gcsFilesystemAuthName,
+      qpsValidation,
+      duplicationStrategy,
+      requestWorkers,
+      localizationConfiguration,
+      virtualPrivateCloudConfiguration,
+      batchRequestTimeoutConfigurationValidation
+    ) flatMapN authGoogleConfigForPapiConfigurationAttributes match {
       case Valid(r) => r
       case Invalid(f) =>
         throw new IllegalArgumentException with MessageAggregation {
diff --git a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributesSpec.scala b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributesSpec.scala
index 1693f563559..f9fb7ecca9f 100644
--- a/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributesSpec.scala
+++ b/supportedBackends/google/pipelines/common/src/test/scala/cromwell/backend/google/pipelines/common/PipelinesApiConfigurationAttributesSpec.scala
@@ -105,6 +105,7 @@ class PipelinesApiConfigurationAttributesSpec extends FlatSpec with Matchers {
       """
         |  virtual-private-cloud {
         |    network-label-key = my-network
+        |    subnetwork-label-key = my-subnetwork
         |    auth = application-default
         |  }
       """.stripMargin
@@ -113,9 +114,38 @@ class PipelinesApiConfigurationAttributesSpec extends FlatSpec with Matchers {
 
     val pipelinesApiAttributes = PipelinesApiConfigurationAttributes(googleConfig, backendConfig)
     pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.name should be("my-network")
+    pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.subnetwork should be (Option("my-subnetwork"))
     pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.auth.name should be("application-default")
   }
 
+  it should "parse virtual-private-cloud without subnetwork key" in {
+
+    val customConfig =
+      """
+        |  virtual-private-cloud {
+        |    network-label-key = my-network
+        |    auth = application-default
+        |  }
+      """.stripMargin
+
+    val backendConfig = ConfigFactory.parseString(configString(customConfig))
+
+    val pipelinesApiAttributes = PipelinesApiConfigurationAttributes(googleConfig, backendConfig)
+    pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.name should be("my-network")
+    pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.subnetwork should be(None)
+    pipelinesApiAttributes.virtualPrivateCloudConfiguration.get.auth.name should be("application-default")
+  }
+
+  it should "parse virtual-private-cloud with empty body" in {
+
+    val customConfig = """virtual-private-cloud{ }"""
+
+    val backendConfig = ConfigFactory.parseString(configString(customConfig))
+
+    val pipelinesApiAttributes = PipelinesApiConfigurationAttributes(googleConfig, backendConfig)
+    pipelinesApiAttributes.virtualPrivateCloudConfiguration should be(None)
+  }
+
   it should "not parse invalid config" in {
     val nakedConfig =
       ConfigFactory.parseString(
@@ -153,7 +183,7 @@ class PipelinesApiConfigurationAttributesSpec extends FlatSpec with Matchers {
       PipelinesApiConfigurationAttributes(googleConfig, networkKeyOnlyConfig)
     }
     val errorsList = exception.errorMessages.toList
-    errorsList should contain("Auth scheme not provided for Virtual Private Cloud configuration.")
+    errorsList should contain("Virtual Private Cloud configuration is invalid. Missing keys: `auth`.")
   }
 
   it should "not parse invalid virtual-private-cloud config without network label key" in {
@@ -171,7 +201,63 @@ class PipelinesApiConfigurationAttributesSpec extends FlatSpec with Matchers {
       PipelinesApiConfigurationAttributes(googleConfig, authOnlyConfig)
     }
     val errorsList = exception.errorMessages.toList
-    errorsList should contain("Network label key not provided for Virtual Private Cloud configuration.")
+    errorsList should contain("Virtual Private Cloud configuration is invalid. Missing keys: `network-label-key`.")
+  }
+
+  it should "not parse invalid virtual-private-cloud config with just subnetwork label key" in {
+    val subnetworkOnlyConfig =
+      ConfigFactory.parseString(
+        """
+          |{
+          |   virtual-private-cloud {
+          |     subnetwork-label-key = "my-subnetwork"
+          |   }
+          |}
+        """.stripMargin)
+
+    val exception = intercept[IllegalArgumentException with MessageAggregation] {
+      PipelinesApiConfigurationAttributes(googleConfig, subnetworkOnlyConfig)
+    }
+    val errorsList = exception.errorMessages.toList
+    errorsList should contain("Virtual Private Cloud configuration is invalid. Missing keys: `network-label-key,auth`.")
+  }
+
+  it should "not parse invalid virtual-private-cloud config with network & subnetwork label keys" in {
+    val config =
+      ConfigFactory.parseString(
+        """
+          |{
+          |   virtual-private-cloud {
+          |     network-label-key = "my-network"
+          |     subnetwork-label-key = "my-subnetwork"
+          |   }
+          |}
+        """.stripMargin)
+
+    val exception = intercept[IllegalArgumentException with MessageAggregation] {
+      PipelinesApiConfigurationAttributes(googleConfig, config)
+    }
+    val errorsList = exception.errorMessages.toList
+    errorsList should contain("Virtual Private Cloud configuration is invalid. Missing keys: `auth`.")
+  }
+
+  it should "not parse invalid virtual-private-cloud config with subnetwork label key & auth" in {
+    val config =
+      ConfigFactory.parseString(
+        """
+          |{
+          |   virtual-private-cloud {
+          |     subnetwork-label-key = "my-subnetwork"
+          |     auth = "application-default"
+          |   }
+          |}
+        """.stripMargin)
+
+    val exception = intercept[IllegalArgumentException with MessageAggregation] {
+      PipelinesApiConfigurationAttributes(googleConfig, config)
+    }
+    val errorsList = exception.errorMessages.toList
+    errorsList should contain("Virtual Private Cloud configuration is invalid. Missing keys: `network-label-key`.")
   }
 
   def configString(customContent: String = "", genomics: String = ""): String =
diff --git a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
index 381e76cf133..49733eaa577 100644
--- a/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
+++ b/supportedBackends/google/pipelines/v2alpha1/src/main/scala/cromwell/backend/google/pipelines/v2alpha1/GenomicsFactory.scala
@@ -89,15 +89,19 @@ case class GenomicsFactory(applicationName: String, authMode: GoogleAuthMode, en
 
       def networkFromLabels(vpcConfig: VirtualPrivateCloudConfiguration, projectLabels: ProjectLabels): Network = {
         val networkLabelOption = projectLabels.labels.find(l => l._1.equals(vpcConfig.name))
+        val subnetworkLabelOption = vpcConfig.subnetwork.flatMap(s => projectLabels.labels.find(l => l._1.equals(s)))
 
         networkLabelOption match {
           case Some(networkLabel) =>
-            new Network()
+            val network = new Network()
               .setUsePrivateAddress(createPipelineParameters.runtimeAttributes.noAddress)
               .setName(VirtualPrivateCloudNetworkPath.format(createPipelineParameters.projectId, networkLabel._2))
+
+            subnetworkLabelOption foreach { case(_, subnet) => network.setSubnetwork(subnet) }
+            network
           case None =>
-            jobLogger.debug("Project `{}` does not have high security network configured. " +
-              "Project metadata does not have network label key `{}`. Falling back to running the job on default network.", createPipelineParameters.projectId: Any, vpcConfig.name: Any)
+            // Falling back to running the job on default network since the project does not provide the custom network
+            // specifying keys in its metadata
             new Network().setUsePrivateAddress(createPipelineParameters.runtimeAttributes.noAddress)
         }
       }

From 38568522a516a88ae27afe39126b37620785d7c2 Mon Sep 17 00:00:00 2001
From: Khalid Shakir <kshakir@broadinstitute.org>
Date: Mon, 24 Jun 2019 01:22:15 -0400
Subject: [PATCH 23/28] Deep Yaml DOS checks

---
 CHANGELOG.md                                  |  6 +-
 docs/Configuring.md                           | 18 ++++
 wom/src/main/resources/reference.conf         |  2 +
 wom/src/main/scala/wom/util/YamlUtils.scala   | 86 +++++++++++++++----
 .../test/scala/wom/util/YamlUtilsSpec.scala   | 14 +++
 5 files changed, 108 insertions(+), 18 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 294bf32ebb8..09dd1bd6e54 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -44,9 +44,9 @@ More information can be found in the [documentation on localization](https://cro
 
 #### Yaml node limits
 
-Yaml parsing now checks for cycles, and limits the maximum number of parsed nodes to a configurable value. See
-[the documentation on configuring Yaml](https://cromwell.readthedocs.io/en/stable/Configuring/#yaml) for more
-information.
+Yaml parsing now checks for cycles, and limits the maximum number of parsed nodes to a configurable value. It also
+limits the nesting depth of sequences and mappings. See [the documentation on configuring
+YAML](https://cromwell.readthedocs.io/en/stable/Configuring/#yaml) for more information.
 
 ### API Changes
 
diff --git a/docs/Configuring.md b/docs/Configuring.md
index 45bdec17872..08764bf3382 100644
--- a/docs/Configuring.md
+++ b/docs/Configuring.md
@@ -526,6 +526,8 @@ The default threshold value is 100, just like the default for the heartbeat batc
 
 ### YAML
 
+**Maximum number of nodes**
+
 Cromwell will throw an error when detecting cyclic loops in Yaml inputs. However one can craft small acyclic YAML
 documents that consume significant amounts of memory or cpu. To limit the amount of processing during parsing, there is
 a limit on the number of nodes parsed per YAML document.
@@ -539,3 +541,19 @@ yaml {
 ```
 
 The default limit is 1,000,000 nodes.
+
+**Maximum nesting depth**
+
+There is a limit on the maximum depth of nested YAML. If you decide to increase this value, you will likely need to also
+increase the Java Virtual Machine's thread stack size as well using
+[either `-Xss` or `-XX:ThreadStackSize`](https://docs.oracle.com/javase/8/docs/technotes/tools/unix/java.html).
+
+This limit may be configured via the configuration value:
+
+```hocon
+yaml {
+  max-depth = 1000
+}
+```
+
+The default limit is a maximum nesting depth of 1,000.
diff --git a/wom/src/main/resources/reference.conf b/wom/src/main/resources/reference.conf
index 08b0bd72e32..6073365bc31 100644
--- a/wom/src/main/resources/reference.conf
+++ b/wom/src/main/resources/reference.conf
@@ -1,4 +1,6 @@
 yaml {
   # The maximum number of nodes (scalars + sequences + mappings) that will be parsed in a YAML
   max-nodes = 1000000
+  # The maximum nested depth for nodes (sequences + mappings) that will be parsed in a YAML
+  max-depth = 1000
 }
diff --git a/wom/src/main/scala/wom/util/YamlUtils.scala b/wom/src/main/scala/wom/util/YamlUtils.scala
index bd26913dd16..72baa60b4dd 100644
--- a/wom/src/main/scala/wom/util/YamlUtils.scala
+++ b/wom/src/main/scala/wom/util/YamlUtils.scala
@@ -10,23 +10,17 @@ import eu.timepit.refined.refineV
 import io.circe.{Json, ParsingFailure}
 import net.ceedubs.ficus.Ficus._
 import net.ceedubs.ficus.readers.ValueReader
+import org.yaml.snakeyaml.composer.Composer
+import org.yaml.snakeyaml.constructor.Constructor
+import org.yaml.snakeyaml.nodes.Node
+import org.yaml.snakeyaml.parser.ParserImpl
+import org.yaml.snakeyaml.reader.StreamReader
+import org.yaml.snakeyaml.resolver.Resolver
 
 import scala.collection.JavaConverters._
 
 object YamlUtils {
 
-  private[util] implicit val refinedNonNegativeReader: ValueReader[Int Refined NonNegative] = {
-    (config: Config, path: String) => {
-      val int = config.getInt(path)
-      refineV[NonNegative](int) match {
-        case Left(error) => throw new BadValue(path, error)
-        case Right(refinedInt) => refinedInt
-      }
-    }
-  }
-
-  private val defaultMaxNodes = ConfigFactory.load().as[Int Refined NonNegative]("yaml.max-nodes")
-
   /**
     * Parses the yaml, detecting loops, and a maximum number of nodes.
     *
@@ -34,12 +28,19 @@ object YamlUtils {
     *
     * @param yaml     The yaml string.
     * @param maxNodes Maximum number of yaml nodes to parse.
+    * @param maxDepth Maximum nested depth of yaml to parse.
     * @return The parsed yaml.
     */
-  def parse(yaml: String, maxNodes: Int Refined NonNegative = defaultMaxNodes): Either[ParsingFailure, Json] = {
+  def parse(yaml: String,
+            maxNodes: Int Refined NonNegative = defaultMaxNodes,
+            maxDepth: Int Refined NonNegative = defaultMaxDepth
+           ): Either[ParsingFailure, Json] = {
     try {
-      val snakeYamlParser = new org.yaml.snakeyaml.Yaml()
-      val parsed = snakeYamlParser.load[AnyRef](new StringReader(yaml))
+      val yamlConstructor = new Constructor()
+      val yamlComposer = new MaxDepthComposer(yaml, maxDepth)
+      yamlConstructor.setComposer(yamlComposer)
+      val parsed = yamlConstructor.getSingleData(classOf[AnyRef])
+
       // Use identity comparisons to check if two nodes are the same and do NOT recurse into them checking equality
       val identityHashMap = new java.util.IdentityHashMap[AnyRef, java.lang.Boolean]()
       // Since we don't actually need the values, wrap the Map in a Set
@@ -52,11 +53,66 @@ object YamlUtils {
     }
   }
 
+  private[util] implicit val refinedNonNegativeReader: ValueReader[Int Refined NonNegative] = {
+    (config: Config, path: String) => {
+      val int = config.getInt(path)
+      refineV[NonNegative](int) match {
+        case Left(error) => throw new BadValue(path, error)
+        case Right(refinedInt) => refinedInt
+      }
+    }
+  }
+
+  private lazy val composerRecursiveNodesField = {
+    val field = classOf[Composer].getDeclaredField("recursiveNodes")
+    field.setAccessible(true)
+    field
+  }
+
+  private val yamlConfig = ConfigFactory.load().getConfig("yaml")
+  private val defaultMaxNodes = yamlConfig.as[Int Refined NonNegative]("max-nodes")
+  private val defaultMaxDepth = yamlConfig.as[Int Refined NonNegative]("max-depth")
+
+  /** Extends SnakeYaml's Composer checking for a maximum depth before a StackOverflowError occurs. */
+  private class MaxDepthComposer(yaml: String, maxDepth: Int Refined NonNegative)
+    extends Composer(
+      new ParserImpl(new StreamReader(new StringReader(yaml))),
+      new Resolver()
+    ) {
+
+    private lazy val composerRecursiveFields: java.util.Set[Node] = getRecursiveNodeSet(this)
+
+    private def checkDepth(): Unit = {
+      if (composerRecursiveFields.size > maxDepth.value)
+        throw new IllegalArgumentException(s"Parsing halted at node depth $maxDepth")
+    }
+
+    override def composeScalarNode(anchor: String): Node = {
+      checkDepth()
+      super.composeScalarNode(anchor)
+    }
+
+    override def composeSequenceNode(anchor: String): Node = {
+      checkDepth()
+      super.composeSequenceNode(anchor)
+    }
+
+    override def composeMappingNode(anchor: String): Node = {
+      checkDepth()
+      super.composeMappingNode(anchor)
+    }
+  }
+
   /** A "pointer" reference to a mutable count. */
   private class Counter {
     var count = 0L
   }
 
+  /** Use reflection to access the existing but private Set of nested nodes */
+  private def getRecursiveNodeSet(composer: Composer): java.util.Set[Node] = {
+    composerRecursiveNodesField.get(composer).asInstanceOf[java.util.Set[Node]]
+  }
+
   /**
     * Looks for loops and large documents in yaml parsed by SnakeYaml.
     *
diff --git a/wom/src/test/scala/wom/util/YamlUtilsSpec.scala b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala
index e3565e14cec..b9ac796302a 100644
--- a/wom/src/test/scala/wom/util/YamlUtilsSpec.scala
+++ b/wom/src/test/scala/wom/util/YamlUtilsSpec.scala
@@ -167,6 +167,20 @@ class YamlUtilsSpec extends FlatSpec with Matchers with TableDrivenPropertyCheck
     exception should have message "Loop detection halted at 1000000 nodes" // <-- Updating here? Also get the docs too!
   }
 
+  it should "fail to parse deeply nested yaml sequence with the default configuration" in {
+    val nesting = 1000000
+    val yaml = ("[" * nesting) + ("]" * nesting)
+    val exception: Exception = YamlUtils.parse(yaml).left.value
+    exception should have message "Parsing halted at node depth 1000" // <-- Updating here? Also get the docs too!
+  }
+
+  it should "fail to parse deeply nested yaml mapping with the default configuration" in {
+    val nesting = 1000000
+    val yaml = ("{a:" * nesting) + "b" + ("}" * nesting)
+    val exception: Exception = YamlUtils.parse(yaml).left.value
+    exception should have message "Parsing halted at node depth 1000" // <-- Updating here? Also get the docs too!
+  }
+
   it should "not parse a config with a negative value" in {
     import wom.util.YamlUtils.refinedNonNegativeReader
     the[BadValue] thrownBy {

From 63760115ff05b8880e1bcbb2194edc7513aa15b0 Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Mon, 24 Jun 2019 09:37:48 -0400
Subject: [PATCH 24/28] Add summarization metrics [Part I of BA-4781] (#5032)

---
 .../slick/MetadataSlickDatabase.scala         | 16 ++++++---
 .../database/sql/MetadataSqlDatabase.scala    |  8 ++---
 .../instrumentation/CromwellMetric.scala      |  8 +++--
 .../impl/MetadataDatabaseAccess.scala         |  8 ++---
 .../metadata/impl/MetadataServiceActor.scala  |  2 +-
 .../impl/MetadataSummaryRefreshActor.scala    | 34 ++++++++++++++-----
 .../metadata/impl/WriteMetadataActor.scala    |  1 -
 .../impl/MetadataDatabaseAccessSpec.scala     | 28 +++++++--------
 8 files changed, 65 insertions(+), 40 deletions(-)

diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index 2bb2f46ff8f..622e62aaf33 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -174,12 +174,12 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
                                    buildUpdatedSummary:
                                    (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry])
                                      => WorkflowMetadataSummaryEntry)
-                                  (implicit ec: ExecutionContext): Future[Long] = {
+                                  (implicit ec: ExecutionContext): Future[(Long, Long)] = {
     val action = for {
       previousMetadataEntryIdOption <- getSummaryStatusEntrySummaryPosition(summarizeNameIncreasing)
       previousMaxMetadataEntryId = previousMetadataEntryIdOption.getOrElse(-1L)
       nextMaxMetadataEntryId = previousMaxMetadataEntryId + limit
-      maximumMetadataEntryId <- summarizeMetadata(
+      maximumMetadataEntryIdConsidered <- summarizeMetadata(
         minMetadataEntryId = previousMaxMetadataEntryId + 1L,
         maxMetadataEntryId = nextMaxMetadataEntryId,
         startMetadataKey = startMetadataKey,
@@ -208,7 +208,12 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
           },
         summaryName = summarizeNameIncreasing
       )
-    } yield maximumMetadataEntryId
+      maximumMetadataEntryIdInTableOption <- dataAccess.metadataEntries.map(_.metadataEntryId).max.result
+      maximumMetadataEntryIdInTable = maximumMetadataEntryIdInTableOption.getOrElse {
+        // TODO: Add a logging framework to this 'database' project and log this weirdness.
+        maximumMetadataEntryIdConsidered
+      }
+    } yield (maximumMetadataEntryIdConsidered - previousMaxMetadataEntryId, maximumMetadataEntryIdInTable - maximumMetadataEntryIdConsidered)
 
     runTransaction(action)
   }
@@ -227,7 +232,7 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
                                    buildUpdatedSummary:
                                    (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry])
                                      => WorkflowMetadataSummaryEntry)
-                                  (implicit ec: ExecutionContext): Future[Long] = {
+                                  (implicit ec: ExecutionContext): Future[(Long, Long)] = {
     val action = for {
       previousExistingMetadataEntryIdOption <- getSummaryStatusEntrySummaryPosition(summaryNameDecreasing)
       previousInitializedMetadataEntryIdOption <- previousExistingMetadataEntryIdOption match {
@@ -254,7 +259,8 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
             summaryName = summaryNameDecreasing
           )
       }
-    } yield newMinimumMetadataEntryId
+      rowsProcessed = previousExistingMetadataEntryIdOption.map(_ - newMinimumMetadataEntryId).getOrElse(0L)
+    } yield (rowsProcessed, newMinimumMetadataEntryId)
 
     runTransaction(action)
   }
diff --git a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
index eaf883fa2bf..c82f78e9265 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/MetadataSqlDatabase.scala
@@ -60,7 +60,7 @@ trait MetadataSqlDatabase extends SqlDatabase {
     * Retrieves next summarizable block of metadata satisfying the specified criteria.
     *
     * @param buildUpdatedSummary Takes in the optional existing summary and the metadata, returns the new summary.
-    * @return A `Future` with the maximum metadataEntryId summarized by the invocation of this method.
+    * @return A `Future` with the number of rows summarized by the invocation, and the number of rows still to summarize.
     */
   def summarizeIncreasing(summaryNameIncreasing: String,
                           startMetadataKey: String,
@@ -75,13 +75,13 @@ trait MetadataSqlDatabase extends SqlDatabase {
                           buildUpdatedSummary:
                           (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry])
                             => WorkflowMetadataSummaryEntry)
-                         (implicit ec: ExecutionContext): Future[Long]
+                         (implicit ec: ExecutionContext): Future[(Long, Long)]
 
   /**
     * Retrieves a window of summarizable metadata satisfying the specified criteria.
     *
     * @param buildUpdatedSummary Takes in the optional existing summary and the metadata, returns the new summary.
-    * @return A `Future` with the maximum metadataEntryId summarized by the invocation of this method.
+    * @return A `Future` with the number of rows summarized by this invocation, and the number of rows still to summarize.
     */
   def summarizeDecreasing(summaryNameDecreasing: String,
                           summaryNameIncreasing: String,
@@ -97,7 +97,7 @@ trait MetadataSqlDatabase extends SqlDatabase {
                           buildUpdatedSummary:
                           (Option[WorkflowMetadataSummaryEntry], Seq[MetadataEntry])
                             => WorkflowMetadataSummaryEntry)
-                         (implicit ec: ExecutionContext): Future[Long]
+                         (implicit ec: ExecutionContext): Future[(Long, Long)]
 
   def getWorkflowStatus(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Option[String]]
 
diff --git a/services/src/main/scala/cromwell/services/instrumentation/CromwellMetric.scala b/services/src/main/scala/cromwell/services/instrumentation/CromwellMetric.scala
index 4cbf1efb3fa..21c9dba1cd8 100644
--- a/services/src/main/scala/cromwell/services/instrumentation/CromwellMetric.scala
+++ b/services/src/main/scala/cromwell/services/instrumentation/CromwellMetric.scala
@@ -25,12 +25,14 @@ sealed trait CromwellMetric
 
 object CromwellCount {
   def apply(bucket: CromwellBucket, value: Long): CromwellCount = new CromwellCount(bucket, value)
-  def unapply(count: CromwellCount): Option[(CromwellBucket, Long, Double)] = Option((count.bucket, count. value, count.sampling))
+  def unapply(count: CromwellCount): Option[(CromwellBucket, Long, Double)] = Option((count.bucket, count.value, count.sampling))
 }
 /**
   * Count occurrences of an event
   */
-class CromwellCount(val bucket: CromwellBucket, val value: Long, val sampling: Double = 1.0D) extends CromwellMetric
+class CromwellCount(val bucket: CromwellBucket, val value: Long, val sampling: Double = 1.0D) extends CromwellMetric {
+  override def toString: String = s"CromwellCount($bucket, $value, $sampling)"
+}
 
 /**
   * Convenience class to increment a count (+1)
@@ -45,4 +47,4 @@ case class CromwellTiming(bucket: CromwellBucket, value: FiniteDuration, samplin
 /**
   * Measures a gauge value
   */
-case class CromwellGauge(bucket: CromwellBucket, value: Long) extends CromwellMetric
\ No newline at end of file
+case class CromwellGauge(bucket: CromwellBucket, value: Long) extends CromwellMetric
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
index 9a6061f4fad..4f4bded7e98 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataDatabaseAccess.scala
@@ -77,7 +77,7 @@ object MetadataDatabaseAccess {
     }
   }
 
-  case class SummaryResult(increasingResult: Long, decreasingResult: Long)
+  case class SummaryResult(rowsProcessedIncreasing: Long, increasingGap: Long, rowsProcessedDecreasing: Long, decreasingGap: Long)
 }
 
 trait MetadataDatabaseAccess {
@@ -159,7 +159,7 @@ trait MetadataDatabaseAccess {
 
   def refreshWorkflowMetadataSummaries(limit: Int)(implicit ec: ExecutionContext): Future[SummaryResult] = {
     for {
-      increasing <- metadataDatabaseInterface.summarizeIncreasing(
+      (increasingProcessed, increasingGap) <- metadataDatabaseInterface.summarizeIncreasing(
         summaryNameIncreasing = WorkflowMetadataKeys.SummaryNameIncreasing,
         startMetadataKey = WorkflowMetadataKeys.StartTime,
         endMetadataKey = WorkflowMetadataKeys.EndTime,
@@ -171,7 +171,7 @@ trait MetadataDatabaseAccess {
         labelMetadataKey = WorkflowMetadataKeys.Labels,
         limit = limit,
         buildUpdatedSummary = MetadataDatabaseAccess.buildUpdatedSummary)
-      decreasing <- metadataDatabaseInterface.summarizeDecreasing(
+      (decreasingProcessed, decreasingGap) <- metadataDatabaseInterface.summarizeDecreasing(
         summaryNameDecreasing = WorkflowMetadataKeys.SummaryNameDecreasing,
         summaryNameIncreasing = WorkflowMetadataKeys.SummaryNameIncreasing,
         startMetadataKey = WorkflowMetadataKeys.StartTime,
@@ -184,7 +184,7 @@ trait MetadataDatabaseAccess {
         labelMetadataKey = WorkflowMetadataKeys.Labels,
         limit = limit,
         buildUpdatedSummary = MetadataDatabaseAccess.buildUpdatedSummary)
-    } yield SummaryResult(increasing, decreasing)
+    } yield SummaryResult(increasingProcessed, increasingGap, decreasingProcessed, decreasingGap)
   }
 
   def getWorkflowStatus(id: WorkflowId)
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
index 511de4df4ef..3dc0d84f6b9 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataServiceActor.scala
@@ -72,7 +72,7 @@ final case class MetadataServiceActor(serviceConfig: Config, globalConfig: Confi
 
   private def buildSummaryActor: Option[ActorRef] = {
     val actor = metadataSummaryRefreshInterval map {
-      _ => context.actorOf(MetadataSummaryRefreshActor.props(), "metadata-summary-actor")
+      _ => context.actorOf(MetadataSummaryRefreshActor.props(serviceRegistryActor), "metadata-summary-actor")
     }
     val message = metadataSummaryRefreshInterval match {
       case Some(interval) => s"Metadata summary refreshing every $interval."
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
index cff448109b1..d486ce23c3d 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/MetadataSummaryRefreshActor.scala
@@ -2,8 +2,11 @@ package cromwell.services.metadata.impl
 
 
 import akka.actor.{ActorRef, LoggingFSM, Props}
+import cats.data.NonEmptyList
 import cromwell.core.Dispatcher.ServiceDispatcher
+import cromwell.core.instrumentation.InstrumentationPrefixes
 import cromwell.services.MetadataServicesStore
+import cromwell.services.instrumentation.CromwellInstrumentation
 import cromwell.services.metadata.impl.MetadataSummaryRefreshActor._
 
 import scala.util.{Failure, Success}
@@ -21,7 +24,7 @@ object MetadataSummaryRefreshActor {
   case object MetadataSummarySuccess extends MetadataSummaryActorMessage
   final case class MetadataSummaryFailure(t: Throwable) extends MetadataSummaryActorMessage
 
-  def props() = Props(new MetadataSummaryRefreshActor()).withDispatcher(ServiceDispatcher)
+  def props(serviceRegistryActor: ActorRef) = Props(new MetadataSummaryRefreshActor(serviceRegistryActor)).withDispatcher(ServiceDispatcher)
 
   sealed trait SummaryRefreshState
   case object WaitingForRequest extends SummaryRefreshState
@@ -31,23 +34,38 @@ object MetadataSummaryRefreshActor {
   case object SummaryRefreshData
 }
 
-class MetadataSummaryRefreshActor()
+class MetadataSummaryRefreshActor(override val serviceRegistryActor: ActorRef)
   extends LoggingFSM[SummaryRefreshState, SummaryRefreshData.type]
-    with MetadataDatabaseAccess with MetadataServicesStore {
+    with MetadataDatabaseAccess
+    with MetadataServicesStore
+    with CromwellInstrumentation {
 
   implicit val ec = context.dispatcher
 
+  private val summaryMetricsGapsPath: NonEmptyList[String] = MetadataServiceActor.MetadataInstrumentationPrefix :+ "summarizer" :+ "gap"
+  private val summaryMetricsProcessedPath: NonEmptyList[String] = MetadataServiceActor.MetadataInstrumentationPrefix :+ "summarizer" :+ "processed"
+
+
+  val increasingGapPath = summaryMetricsGapsPath :+ "increasing"
+  val decreasingGapPath = summaryMetricsGapsPath :+ "decreasing"
+
+  val increasingProcessedPath = summaryMetricsProcessedPath :+ "increasing"
+  val decreasingProcessedPath = summaryMetricsProcessedPath :+ "decreasing"
+
+  private val instrumentationPrefix: Option[String] = InstrumentationPrefixes.ServicesPrefix
+
   startWith(WaitingForRequest, SummaryRefreshData)
 
   when (WaitingForRequest) {
     case Event(SummarizeMetadata(limit, respondTo), _) =>
       refreshWorkflowMetadataSummaries(limit) onComplete {
         case Success(summaryResult) =>
-          log.debug(
-            "Summarized metadata: increasing = {}, decreasing = {}",
-            summaryResult.increasingResult,
-            summaryResult.decreasingResult
-          )
+          sendGauge(increasingGapPath, summaryResult.increasingGap, instrumentationPrefix)
+          sendGauge(decreasingGapPath, summaryResult.decreasingGap, instrumentationPrefix)
+
+          count(increasingProcessedPath, summaryResult.rowsProcessedIncreasing, instrumentationPrefix)
+          count(decreasingProcessedPath, summaryResult.rowsProcessedDecreasing, instrumentationPrefix)
+
           respondTo ! MetadataSummarySuccess
           self ! MetadataSummaryComplete
         case Failure(t) =>
diff --git a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
index ace3ba520ee..57c47a8536c 100644
--- a/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
+++ b/services/src/main/scala/cromwell/services/metadata/impl/WriteMetadataActor.scala
@@ -80,7 +80,6 @@ class WriteMetadataActor(override val batchSize: Int,
 }
 
 object WriteMetadataActor {
-  val MetadataWritePath = MetadataServiceActor.MetadataInstrumentationPrefix.::("writes")
 
   def props(dbBatchSize: Int,
             flushRate: FiniteDuration,
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
index 1e74eb0bb49..27f112d3e1e 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
@@ -10,6 +10,7 @@ import cromwell.core.labels.{Label, Labels}
 import cromwell.database.slick.{EngineSlickDatabase, MetadataSlickDatabase}
 import cromwell.services.ServicesStore.EnhancedSqlDatabase
 import cromwell.services.metadata._
+import cromwell.services.metadata.impl.MetadataDatabaseAccess.SummaryResult
 import cromwell.services.{EngineServicesStore, MetadataServicesStore}
 import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout}
 import org.scalatest.concurrent.ScalaFutures
@@ -150,6 +151,16 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
       } yield()).futureValue
     }
 
+    def assertRowsProcessedAndSummarizationComplete(summaryResult: SummaryResult) = {
+      withClue(s"asserting correctness of $summaryResult") {
+        summaryResult.rowsProcessedIncreasing should be > 0L
+        summaryResult.rowsProcessedDecreasing should be(0L)
+
+        summaryResult.increasingGap should be(0L)
+        summaryResult.decreasingGap should be(0L)
+      }
+    }
+
     it should "create and query a workflow" taggedAs DbmsTest in {
 
       val randomIds = Seq.fill(10)(WorkflowId.randomId().toString)
@@ -176,12 +187,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         workflow2Id <- baseWorkflowMetadata(Workflow2Name, Set(testLabel2, testLabel3))
 
         // refresh the metadata
-        _ <- dataAccess.refreshWorkflowMetadataSummaries(1000) map { summaryResult =>
-          withClue("summaryResult") {
-            summaryResult.increasingResult should be > 0L
-            summaryResult.decreasingResult should be >= 0L
-          }
-        }
+        _ <- dataAccess.refreshWorkflowMetadataSummaries(1000) map assertRowsProcessedAndSummarizationComplete
 
         // Query with no filters
         (workflowQueryResult, workflowQueryResult2) <-
@@ -372,10 +378,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
         _ <- baseWorkflowMetadata(uniqueWorkflow3Name)
         _ <- baseWorkflowMetadata(uniqueWorkflow3Name)
         // refresh the metadata
-        _ <- dataAccess.refreshWorkflowMetadataSummaries(1000) map { summaryResult =>
-          summaryResult.increasingResult should be > 0L
-          summaryResult.decreasingResult should be >= 0L
-        }
+        _ <- dataAccess.refreshWorkflowMetadataSummaries(1000) map assertRowsProcessedAndSummarizationComplete
         //get totalResultsCount when page and pagesize are specified
         _ <- dataAccess.queryWorkflowSummaries(WorkflowQueryParameters(Seq(
           // name being added to the query parameters so as to exclude workflows being populated by the other tests in this spec
@@ -397,10 +400,7 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
           MetadataEvent.labelsToMetadataEvents(Labels(customLabelKey -> customLabelValue), workflowId)
         for {
           _ <- dataAccess.addMetadataEvents(metadataEvents)
-          _ <- dataAccess.refreshWorkflowMetadataSummaries(1000).map { summaryResult =>
-            summaryResult.increasingResult should be > 0L
-            summaryResult.decreasingResult should be >= 0L
-          }
+          _ <- dataAccess.refreshWorkflowMetadataSummaries(1000) map assertRowsProcessedAndSummarizationComplete
           _ <- dataAccess.getWorkflowLabels(workflowId).map(_.toList should contain(customLabelKey -> customLabelValue))
         } yield ()
       }

From d78d5adad18efe8f3e8a5eaf12ae11d7789e472c Mon Sep 17 00:00:00 2001
From: mcovarr <mcovarr@users.noreply.github.com>
Date: Mon, 24 Jun 2019 10:13:17 -0400
Subject: [PATCH 25/28] remove unnecessary and slow migration (#5040)

---
 .../enlarge_call_caching_hash_entry_id.xml     | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
index d2159d41bdc..9e69c154730 100644
--- a/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
+++ b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
@@ -3,17 +3,15 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="enlarge_call_caching_hash_entry_id">
-        <comment>
-            Larger integer data type for the PK since > 2 billion rows.
-        </comment>
-        <modifyDataType
-                tableName="CALL_CACHING_HASH_ENTRY"
-                columnName="CALL_CACHING_HASH_ENTRY_ID"
-                newDataType="BIGINT"/>
-    </changeSet>
-
+    <!-- This changeset is misleadingly named. This used to be part of a two-changeset migration that widened the PK and
+         then added autoincrement. At the time this was first written the author did not realize that `addAutoIncrement`
+         could both alter the datatype of and add autoincrement to the PK in one shot. The changeset cannot be renamed
+         as the name is part of the Liquibase key that will prevent it from running again in environments that already
+         suffered through the old two-changeset migration. -->
     <changeSet author="mcovarr" id="restore_auto_increment_call_caching_hash_entry_id">
+        <!-- This is the original comment from the two-changeset version of the code and it doesn't make a lot of sense in
+             the new one-changeset reality. Removing the comment doesn't alter the Liquibase hash but it does make the
+             resulting row in the DATABASECHANGELOG different. Keeping it for now out of an abundance of caution. -->
         <comment>
             Factored into a separate changeset from the above to allow for handling various RDBMS implementations differently.
         </comment>

From 195de29cd4f918222311047ce4272876041e3731 Mon Sep 17 00:00:00 2001
From: Ruben Vorderman <r.h.p.vorderman@lumc.nl>
Date: Tue, 25 Jun 2019 19:22:42 +0200
Subject: [PATCH 26/28] Make cached-copy localization strategy aware of the
 maximum number of hardlinks. [BA-5748] (#5043)

---
 .../cromwell.examples.conf                    |  4 +++
 .../backend/sfs/SharedFileSystem.scala        | 20 +++++++++----
 .../backend/sfs/SharedFileSystemSpec.scala    | 28 +++++++++++++++++++
 3 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf
index 4318bce04bd..c5c76e5c404 100644
--- a/cromwell.example.backends/cromwell.examples.conf
+++ b/cromwell.example.backends/cromwell.examples.conf
@@ -421,6 +421,10 @@ backend {
             localization: [
               "hard-link", "soft-link", "copy"
             ]
+            # An experimental localization strategy called "cached-copy" is also available for SFS backends.
+            # This will copy a file to a cache and then hard-link from the cache. It will copy the file to the cache again
+            # when the maximum number of hardlinks for a file is reached. The maximum number of hardlinks can be set with:
+            # max-hardlinks: 950
 
             # Call caching strategies
             caching {
diff --git a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
index 850e2e5871d..4486cae6bcf 100644
--- a/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
+++ b/supportedBackends/sfs/src/main/scala/cromwell/backend/sfs/SharedFileSystem.scala
@@ -14,6 +14,7 @@ import cromwell.backend.io.JobPaths
 import cromwell.core.CromwellFatalExceptionMarker
 import cromwell.core.path.{DefaultPath, DefaultPathBuilder, Path, PathFactory}
 import cromwell.filesystems.http.HttpPathBuilder
+import net.ceedubs.ficus.Ficus._
 import wom.WomFileMapper
 import wom.values._
 
@@ -101,13 +102,17 @@ object SharedFileSystem extends StrictLogging {
       Thread.sleep(1)
     }
   }
+
+  private def countLinks(path: Path): Int = {
+    path.getAttribute("unix:nlink").asInstanceOf[Int]
+  }
 }
 
 trait SharedFileSystem extends PathFactory {
   import SharedFileSystem._
 
   def sharedFileSystemConfig: Config
-
+  lazy val maxHardLinks: Int = sharedFileSystemConfig.getOrElse[Int]("max-hardlinks",950)  // Windows limit 1024. Keep a safe margin.
   lazy val cachedCopyDir: Option[Path] = None
 
   private def localizePathViaCachedCopy(originalPath: Path, executionPath: Path, docker: Boolean): Try[Unit] = {
@@ -124,7 +129,7 @@ trait SharedFileSystem extends PathFactory {
       val cachedCopyPath: Path = cachedCopySubDir./(pathAndModTime)
       val cachedCopyPathLockFile: Path = cachedCopyPath.plusSuffix(".lock")
 
-      if (!cachedCopyPath.exists) {
+      if (!cachedCopyPath.exists || countLinks(cachedCopyPath) >= maxHardLinks) {
         // This variable is used so we can release the lock before we start with the copying.
         var shouldCopy = false
 
@@ -133,10 +138,11 @@ trait SharedFileSystem extends PathFactory {
         // decisions which are not time consuming.
         SharedFileSystem.synchronized {
 
-          // We check again if cachedCopyPath is there. It may have been created while waiting on the lock.
-          // If it is not there, is it already being copied by another thread?
+          // We check again if cachedCopyPath is there or if the number of Hardlinks is still exceeded.
+          // The copying may have been started while waiting on the lock.
+          // If it is not there or the maxHardLinks are exceeded, is it already being copied by another thread?
           // if not copied by another thread, is it copied by another cromwell process? (Lock file present)
-          if (!cachedCopyPath.exists &&
+          if ((!cachedCopyPath.exists || countLinks(cachedCopyPath) >= maxHardLinks) &&
             !SharedFileSystem.beingCopied.getOrElse(cachedCopyPath, false) &&
             !cachedCopyPathLockFile.exists) {
             // Create a lock file so other cromwell processes know copying has started
@@ -163,7 +169,9 @@ trait SharedFileSystem extends PathFactory {
         if (shouldCopy) {
           try {
             val cachedCopyTmpPath = cachedCopyPath.plusExt("tmp")
-            originalPath.copyTo(cachedCopyTmpPath, overwrite = true).moveTo(cachedCopyPath)
+            // CachedCopyPath is overwritten. It is possible that the number of hardlinks is exceeded. In which case
+            // the file is already there.
+            originalPath.copyTo(cachedCopyTmpPath, overwrite = true).moveTo(cachedCopyPath, overwrite = true)
           } catch {
             case e: Exception => throw e
           }
diff --git a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
index 6a4f000dc9b..92bf01d2e8c 100644
--- a/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
+++ b/supportedBackends/sfs/src/test/scala/cromwell/backend/sfs/SharedFileSystemSpec.scala
@@ -20,6 +20,7 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
   val hardLinkLocalization = ConfigFactory.parseString(""" localization: [hard-link] """)
   val softLinkLocalization = ConfigFactory.parseString(""" localization: [soft-link] """)
   val cachedCopyLocalization = ConfigFactory.parseString(""" localization: [cached-copy] """)
+  val cachedCopyLocalizationMaxHardlinks = ConfigFactory.parseString("""{localization: [cached-copy], max-hardlinks: 3 }""")
   val localPathBuilder = List(DefaultPathBuilder)
 
 
@@ -141,6 +142,33 @@ class SharedFileSystemSpec extends FlatSpec with Matchers with Mockito with Tabl
     dests.foreach(_.delete(swallowIOExceptions = true))
   }
 
+it should "copy the file again when the copy-cached file has exceeded the maximum number of hardlinks" in {
+    val callDirs: IndexedSeq[Path] = 1 to 3 map { _ => DefaultPathBuilder.createTempDirectory("SharedFileSystem") }
+    val orig = DefaultPathBuilder.createTempFile("inputFile")
+    val dests = callDirs.map(_./(orig.parent.pathAsString.hashCode.toString())./(orig.name))
+    orig.touch()
+    val inputs = fqnWdlMapToDeclarationMap(Map("input" -> WomSingleFile(orig.pathAsString)))
+    val sharedFS = new SharedFileSystem {
+      override val pathBuilders = localPathBuilder
+      override val sharedFileSystemConfig = cachedCopyLocalizationMaxHardlinks
+      override implicit def actorContext: ActorContext = null
+      override lazy val cachedCopyDir = Some(DefaultPathBuilder.createTempDirectory("cached-copy"))
+    }
+    val cachedFile: Option[Path] = sharedFS.cachedCopyDir.map(
+      _./(orig.parent.pathAsString.hashCode.toString)./(orig.lastModifiedTime.toEpochMilli.toString + orig.name))
+
+    val results = callDirs.map(sharedFS.localizeInputs(_, docker = true)(inputs))
+
+    results.foreach(_.isSuccess shouldBe true)
+    dests.foreach(_.exists shouldBe true)
+    dests.foreach(countLinks(_) should be <= 3)
+
+    cachedFile.foreach(_.exists shouldBe true)
+    cachedFile.foreach(countLinks(_) should be <= 3)
+    orig.delete(swallowIOExceptions = true)
+    dests.foreach(_.delete(swallowIOExceptions = true))
+  }
+
 
   private[this] def countLinks(file: Path): Int = file.getAttribute("unix:nlink").asInstanceOf[Int]
 

From 958663944d9236828daa8bff8e991bba4de963e9 Mon Sep 17 00:00:00 2001
From: Nat Echols <nathaniel.echols@gmail.com>
Date: Mon, 29 Apr 2019 12:46:37 -0700
Subject: [PATCH 27/28] Postgresql support

Including migration changes and workarounds for auto-incremented columns and large objects.
---
 .travis.yml                                   |   3 +
 CHANGELOG.md                                  |   7 +
 core/src/test/resources/application.conf      |   4 +-
 .../cromwell.examples.conf                    |  13 +
 .../src/main/resources/changelog.xml          |   1 +
 .../add_attempt_in_call_caching_entry.xml     |   2 +-
 .../changesets/add_attempt_in_execution.xml   |   2 +-
 .../changesets/add_index_in_execution.xml     |   2 +-
 .../add_start_end_time_in_execution.xml       |   2 +-
 .../changesets/add_unique_constraints.xml     |   2 +-
 .../changesets/add_workflow_name.xml          |   2 +-
 ...d_workflow_url_in_workflow_store_entry.xml |   2 +-
 .../resources/changesets/backend_KV_Store.xml |   4 +-
 .../call_caching_aggregated_hashes.xml        |   4 +-
 .../call_caching_allow_result_reuse_fix.xml   |   6 +-
 .../changesets/call_caching_job_detritus.xml  |   6 +-
 .../changesets/call_result_caching.xml        |   6 +-
 .../main/resources/changesets/callcaching.xml |  16 +-
 .../change_execution_unique_constraint.xml    |   2 +-
 .../change_max_size_label_entry.xml           |   2 +-
 .../change_max_size_workflow_url.xml          |   2 +-
 .../changesets/custom_label_entry.xml         |   8 +-
 .../main/resources/changesets/db_schema.xml   |   6 +-
 .../changesets/docker_hash_store.xml          |   4 +-
 .../docker_hash_store_add_size_column.xml     |   2 +-
 .../drop_workflow_uri_and_local_command.xml   |   2 +-
 .../changesets/embiggen_detritus_value.xml    |   2 +-
 .../changesets/embiggen_metadata_value.xml    |   6 +-
 .../encrypt_and_clear_workflow_options.xml    |   4 +-
 .../enlarge_call_caching_hash_entry_id.xml    |   2 +-
 .../resources/changesets/events_table.xml     |   2 +-
 .../changesets/execution_backend_info.xml     |  16 +-
 .../resources/changesets/failure_table.xml    |   2 +-
 .../resources/changesets/jes_id_update.xml    |   2 +-
 .../main/resources/changesets/job_store.xml   |   6 +-
 .../changesets/job_store_simpletons.xml       |   8 +-
 .../changesets/job_store_tinyints.xml         |   8 +-
 .../changesets/lengthen_wdl_value.xml         |   2 +-
 .../changesets/local_job_allow_null.xml       |   2 +-
 .../resources/changesets/metadata_journal.xml |   8 +-
 .../metadata_journal_subsecond_timestamp.xml  |   4 +-
 .../changesets/metadata_typed_values.xml      |   2 +-
 .../move_sql_metadata_changelog.xml           |   2 +-
 .../resources/changesets/nullable_lobs.xml    |   2 +-
 .../resources/changesets/optional_sge_ids.xml |   2 +-
 .../changesets/postgresql_engine_schema.xml   | 326 ++++++++++++++++++
 .../src/main/resources/changesets/rc.xml      |   4 +-
 .../changesets/remove_pre_pbe_tables.xml      |   2 +-
 .../changesets/rename_iteration_to_index.xml  |   2 +-
 .../resources/changesets/rename_jes_id.xml    |   4 +-
 .../rename_workflow_options_in_metadata.xml   |   2 +-
 .../replace_empty_custom_labels.xml           |   2 +-
 .../restart_and_recover_migration.xml         |  12 +-
 .../changesets/runtime_attributes_table.xml   |   2 +-
 .../src/main/resources/changesets/sge.xml     |   4 +-
 .../sge_job_execution_unique_key.xml          |   2 +-
 .../changesets/standardize_column_names.xml   |   6 +-
 .../standardize_column_names_again.xml        |   2 +-
 .../standardize_column_names_patches.xml      |   2 +-
 .../changesets/sub_workflow_store.xml         |   8 +-
 .../changesets/summary_status_table.xml       |   4 +-
 .../changesets/symbol_iteration_not_null.xml  |   2 +-
 .../changesets/symbol_iteration_null.xml      |   2 +-
 .../changesets/sync_not_null_constraints.xml  |   4 +-
 .../resources/changesets/top_level_output.xml |   4 +-
 .../resources/changesets/wdl_and_inputs.xml   |   4 +-
 .../workflow_execution_aux_not_null.xml       |   2 +-
 .../changesets/workflow_metadata_summary.xml  |  10 +-
 .../resources/changesets/workflow_options.xml |   2 +-
 .../resources/changesets/workflow_store.xml   |   6 +-
 .../workflow_store_horizontal_db.xml          |   2 +-
 .../workflow_store_imports_file.xml           |   2 +-
 .../changesets/workflow_store_labels_file.xml |   2 +-
 .../workflow_store_restarted_column.xml       |   4 +-
 .../workflow_store_state_widening.xml         |   2 +-
 .../workflow_store_type_and_version.xml       |   2 +-
 .../workflow_store_workflow_root_column.xml   |   2 +-
 .../postgresql_metadata_schema.xml            | 130 +++++++
 .../main/resources/sql_metadata_changelog.xml |   1 +
 .../slick/MetadataSlickDatabase.scala         |   2 +-
 .../database/slick/SlickDatabase.scala        |  41 +++
 .../CallCachingDetritusEntryComponent.scala   |   4 +-
 .../CallCachingSimpletonEntryComponent.scala  |   4 +-
 .../tables/CustomLabelEntryComponent.scala    |  14 +-
 .../slick/tables/DriverComponent.scala        |  50 ++-
 .../slick/tables/JobStoreEntryComponent.scala |   4 +-
 .../JobStoreSimpletonEntryComponent.scala     |   4 +-
 .../slick/tables/MetadataEntryComponent.scala |   5 +-
 ...orkflowMetadataSummaryEntryComponent.scala | 162 ++++++---
 .../tables/WorkflowStoreEntryComponent.scala  |  14 +-
 .../cromwell/database/sql/SqlConverters.scala |  26 +-
 .../sql/tables/CallCachingDetritusEntry.scala |   4 +-
 .../tables/CallCachingSimpletonEntry.scala    |   4 +-
 .../database/sql/tables/JobStoreEntry.scala   |   4 +-
 .../sql/tables/JobStoreSimpletonEntry.scala   |   4 +-
 .../database/sql/tables/MetadataEntry.scala   |   6 +-
 .../sql/tables/WorkflowStoreEntry.scala       |  14 +-
 .../database/sql/tables/package.scala         |   5 +-
 docs/Configuring.md                           |  27 ++
 .../CallCachingSlickDatabaseSpec.scala        |   2 +
 .../workflowstore/SqlWorkflowStoreSpec.scala  |   2 +
 project/Dependencies.scala                    |   9 +-
 .../cromwell/services/ServicesStoreSpec.scala |  14 +-
 .../keyvalue/impl/KeyValueDatabaseSpec.scala  |   6 +
 .../impl/MetadataDatabaseAccessSpec.scala     |   2 +
 src/ci/bin/test.inc.sh                        |   2 +-
 106 files changed, 946 insertions(+), 250 deletions(-)
 create mode 100644 database/migration/src/main/resources/changesets/postgresql_engine_schema.xml
 create mode 100644 database/migration/src/main/resources/metadata_changesets/postgresql_metadata_schema.xml

diff --git a/.travis.yml b/.travis.yml
index 724c2eba532..9971664b845 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -54,6 +54,9 @@ env:
     - >-
       BUILD_TYPE=centaurLocal
       BUILD_MYSQL=5.7
+    - >-
+      BUILD_TYPE=centaurLocal
+      BUILD_POSTGRESQL=11.3
     - >-
       BUILD_TYPE=centaurPapiV1
       BUILD_MYSQL=5.7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 09dd1bd6e54..ddf68b64e27 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -55,6 +55,13 @@ YAML](https://cromwell.readthedocs.io/en/stable/Configuring/#yaml) for more info
 * It is now possible to use `includeKey` and `excludeKey` at the same time. If so, the metadata key must match the `includeKey` **and not** match the `excludeKey` to be included.
 * It is now possible to use "`calls`" as one of your `excludeKey`s, to request that only workflow metadata gets returned.
 
+### PostgreSQL support
+
+Cromwell now supports PostgreSQL (version 9.6 or higher, with the Large Object
+extension installed) as a database backend.
+See [here](https://cromwell.readthedocs.io/en/stable/Configuring/#database) for
+instructions for configuring the database connection.
+
 ## 42 Release Notes
 
 ### Womtool endpoint
diff --git a/core/src/test/resources/application.conf b/core/src/test/resources/application.conf
index 6d479c92e23..22ce88ed919 100644
--- a/core/src/test/resources/application.conf
+++ b/core/src/test/resources/application.conf
@@ -63,13 +63,13 @@ database-test-mariadb {
   }
 }
 
-database-test-postgres {
+database-test-postgresql {
   # Run the following to (optionally) drop and (re-)create the database:
   # psql postgres <<< 'drop database if exists cromwell_test; create database cromwell_test;'
   profile = "slick.jdbc.PostgresProfile$"
   db {
     driver = "org.postgresql.Driver"
-    url = "jdbc:postgresql://localhost:5432/cromwell_test"
+    url = "jdbc:postgresql://localhost:5432/cromwell_test?reWriteBatchedInserts=true"
     url = ${?CROMWELL_BUILD_POSTGRES_JDBC_URL}
     user = "cromwell"
     user = ${?CROMWELL_BUILD_POSTGRES_USERNAME}
diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf
index 4318bce04bd..243b32d4210 100644
--- a/cromwell.example.backends/cromwell.examples.conf
+++ b/cromwell.example.backends/cromwell.examples.conf
@@ -630,4 +630,17 @@ database {
   #    connectionTimeout = 3000
   #  }
   #}
+
+  # Postgresql example
+  #database {
+  #  profile = "slick.jdbc.PostgresProfile$"
+  #  db {
+  #    driver = "org.postgresql.Driver"
+  #    url = "jdbc:postgresql://localhost:5432/cromwell"
+  #    user = ""
+  #    password = ""
+  #    port = 5432
+  #    connectionTimeout = 5000
+  #  }
+  #}
 }
diff --git a/database/migration/src/main/resources/changelog.xml b/database/migration/src/main/resources/changelog.xml
index 5b755a12cba..2accb5c6f94 100644
--- a/database/migration/src/main/resources/changelog.xml
+++ b/database/migration/src/main/resources/changelog.xml
@@ -77,6 +77,7 @@
     <include file="changesets/change_max_size_workflow_url.xml" relativeToChangelogFile="true" />
     <include file="changesets/docker_hash_store_add_size_column.xml" relativeToChangelogFile="true" />
     <include file="changesets/enlarge_call_caching_hash_entry_id.xml" relativeToChangelogFile="true" />
+    <include file="changesets/postgresql_engine_schema.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 
diff --git a/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml b/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml
index ff109b7218e..217a63ec703 100644
--- a/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml
+++ b/database/migration/src/main/resources/changesets/add_attempt_in_call_caching_entry.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet id="add_attempt_in_call_caching_entry" author="tjeandet">
+    <changeSet id="add_attempt_in_call_caching_entry" author="tjeandet" dbms="mysql,hsqldb">
         <addColumn tableName="CALL_CACHING_ENTRY">
             <column name="JOB_ATTEMPT" type="INT" defaultValue="NULL">
                 <constraints nullable="true" />
diff --git a/database/migration/src/main/resources/changesets/add_attempt_in_execution.xml b/database/migration/src/main/resources/changesets/add_attempt_in_execution.xml
index 08e05a436d1..74eb8c2bd5b 100644
--- a/database/migration/src/main/resources/changesets/add_attempt_in_execution.xml
+++ b/database/migration/src/main/resources/changesets/add_attempt_in_execution.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="add-attempt-in-execution">
+    <changeSet author="tjeandet" id="add-attempt-in-execution" dbms="mysql,hsqldb">
         <addColumn
                 tableName="EXECUTION">
             <column name="ATTEMPT" type="INT" />
diff --git a/database/migration/src/main/resources/changesets/add_index_in_execution.xml b/database/migration/src/main/resources/changesets/add_index_in_execution.xml
index 44a71d3d308..65b7e7a264e 100644
--- a/database/migration/src/main/resources/changesets/add_index_in_execution.xml
+++ b/database/migration/src/main/resources/changesets/add_index_in_execution.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="add-index-in-execution">
+    <changeSet author="tjeandet" id="add-index-in-execution" dbms="mysql,hsqldb">
         <addColumn
             tableName="EXECUTION">
             <column name="INDEX" type="INT" />
diff --git a/database/migration/src/main/resources/changesets/add_start_end_time_in_execution.xml b/database/migration/src/main/resources/changesets/add_start_end_time_in_execution.xml
index 4fc46de77c3..91351255927 100644
--- a/database/migration/src/main/resources/changesets/add_start_end_time_in_execution.xml
+++ b/database/migration/src/main/resources/changesets/add_start_end_time_in_execution.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="jgentry" id="add-start-end-time-in-execution">
+    <changeSet author="jgentry" id="add-start-end-time-in-execution" dbms="mysql,hsqldb">
         <addColumn tableName="EXECUTION">
            <column name="START_DT" type="datetime"/>
         </addColumn>
diff --git a/database/migration/src/main/resources/changesets/add_unique_constraints.xml b/database/migration/src/main/resources/changesets/add_unique_constraints.xml
index 7192cb78d82..4373f07dfaa 100644
--- a/database/migration/src/main/resources/changesets/add_unique_constraints.xml
+++ b/database/migration/src/main/resources/changesets/add_unique_constraints.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="add_unique_constraints">
+    <changeSet author="kshakir" id="add_unique_constraints" dbms="mysql,hsqldb">
         <comment>
             Adds unique constraints to other tables besides the existing
             key WORKFLOW_EXECUTION_AUX.UK_WE_AUX_WORKFLOW_EXECUTION_ID.
diff --git a/database/migration/src/main/resources/changesets/add_workflow_name.xml b/database/migration/src/main/resources/changesets/add_workflow_name.xml
index fa379c5f1a3..b1d0fb11890 100644
--- a/database/migration/src/main/resources/changesets/add_workflow_name.xml
+++ b/database/migration/src/main/resources/changesets/add_workflow_name.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="mcovarr" id="workflow-name">
+    <changeSet author="mcovarr" id="workflow-name" dbms="mysql,hsqldb">
         <addColumn tableName="WORKFLOW_EXECUTION">
             <column name="WORKFLOW_NAME" type="VARCHAR(500)">
                 <constraints nullable="false" />
diff --git a/database/migration/src/main/resources/changesets/add_workflow_url_in_workflow_store_entry.xml b/database/migration/src/main/resources/changesets/add_workflow_url_in_workflow_store_entry.xml
index 63c3f02dcd5..bb6b2c7066a 100644
--- a/database/migration/src/main/resources/changesets/add_workflow_url_in_workflow_store_entry.xml
+++ b/database/migration/src/main/resources/changesets/add_workflow_url_in_workflow_store_entry.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="sshah" id="add_workflow_url_in_workflow_store_entry">
+    <changeSet author="sshah" id="add_workflow_url_in_workflow_store_entry" dbms="mysql,hsqldb">
         <addColumn tableName="WORKFLOW_STORE_ENTRY">
             <column name="WORKFLOW_URL" type="VARCHAR(255)" />
         </addColumn>
diff --git a/database/migration/src/main/resources/changesets/backend_KV_Store.xml b/database/migration/src/main/resources/changesets/backend_KV_Store.xml
index dc88ddf8fb9..d21ceac04a8 100644
--- a/database/migration/src/main/resources/changesets/backend_KV_Store.xml
+++ b/database/migration/src/main/resources/changesets/backend_KV_Store.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="rmunshi" id="BACKEND_KV_STORE">
+    <changeSet author="rmunshi" id="BACKEND_KV_STORE" dbms="mysql,hsqldb">
         <comment>
             Storage for all Backend K/V Pairs.
         </comment>
@@ -31,7 +31,7 @@
         </createTable>
     </changeSet>
 
-    <changeSet author="rmunshi" id="backend_KV_store_job_key_constraint">
+    <changeSet author="rmunshi" id="backend_KV_store_job_key_constraint" dbms="mysql,hsqldb">
         <addUniqueConstraint
                 tableName="BACKEND_KV_STORE"
                 columnNames="WORKFLOW_EXECUTION_UUID, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT, STORE_KEY"
diff --git a/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml b/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml
index 87a1fb96992..d8cda49dc6f 100644
--- a/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml
+++ b/database/migration/src/main/resources/changesets/call_caching_aggregated_hashes.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="call_caching_aggregation_entry">
+    <changeSet author="tjeandet" id="call_caching_aggregation_entry" dbms="mysql,hsqldb">
         <comment>
             Holds an aggregated version of the hashes for each cache entry.
             Split into "BaseAggregation" and "InputFilesAggregation"
@@ -28,7 +28,7 @@
         </createTable>
     </changeSet>
 
-    <changeSet author="tjeandet" id="call_caching_aggregation_entry_keys_and_indexes">
+    <changeSet author="tjeandet" id="call_caching_aggregation_entry_keys_and_indexes" dbms="mysql,hsqldb">
         <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID"
                                  baseTableName="CALL_CACHING_AGGREGATION_ENTRY"
                                  constraintName="FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID"
diff --git a/database/migration/src/main/resources/changesets/call_caching_allow_result_reuse_fix.xml b/database/migration/src/main/resources/changesets/call_caching_allow_result_reuse_fix.xml
index 390e0c8f38a..514a8c3d8a6 100644
--- a/database/migration/src/main/resources/changesets/call_caching_allow_result_reuse_fix.xml
+++ b/database/migration/src/main/resources/changesets/call_caching_allow_result_reuse_fix.xml
@@ -3,16 +3,16 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <property name="boolean.type" value="BOOLEAN" dbms="!mysql"/>
+    <property name="boolean.type" value="BOOLEAN" dbms="hsqldb"/>
     <property name="boolean.type" value="TINYINT" dbms="mysql"/>
 
-    <changeSet id="call_caching_fix_boolean" author="mcovarr">
+    <changeSet id="call_caching_fix_boolean" author="mcovarr" dbms="mysql,hsqldb">
         <modifyDataType columnName="ALLOW_RESULT_REUSE"
                         newDataType="${boolean.type}"
                         tableName="CALL_CACHING_RESULT_METAINFO"/>
     </changeSet>
 
-    <changeSet id="call_caching_reuse_default" author="mcovarr">
+    <changeSet id="call_caching_reuse_default" author="mcovarr" dbms="mysql,hsqldb">
         <addDefaultValue columnName="ALLOW_RESULT_REUSE"
                          defaultValueBoolean="true"
                          tableName="CALL_CACHING_RESULT_METAINFO"/>
diff --git a/database/migration/src/main/resources/changesets/call_caching_job_detritus.xml b/database/migration/src/main/resources/changesets/call_caching_job_detritus.xml
index 901630bad49..c9cf3f2bb26 100644
--- a/database/migration/src/main/resources/changesets/call_caching_job_detritus.xml
+++ b/database/migration/src/main/resources/changesets/call_caching_job_detritus.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="rmunshi" id="call_caching_job_detritus">
+    <changeSet author="rmunshi" id="call_caching_job_detritus" dbms="mysql,hsqldb">
         <comment>
             One row per job detritus file (i.e, one row for the stdout, stderr, etc)
             The job detritus key contains the filename, such as hello-stdout, and the job detritus
@@ -28,13 +28,13 @@
     </changeSet>
 
     <!-- Don't want to store the same standard output key twice for each job/RESULT_METAINFO_ID) -->
-    <changeSet author="rmunshi" id="call_caching_job_detritus_key_uniqueness">
+    <changeSet author="rmunshi" id="call_caching_job_detritus_key_uniqueness" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_CALL_CACHING_JOB_DETRITUS_KEY"
                              tableName="CALL_CACHING_JOB_DETRITUS" columnNames="JOB_DETRITUS_KEY, RESULT_METAINFO_ID" />
     </changeSet>
 
     <!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
-    <changeSet author="rmunshi" id="call_caching_job_detritus_foreign_key">
+    <changeSet author="rmunshi" id="call_caching_job_detritus_foreign_key" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="CCJD_RESULT_METAINFO_ID_FK"
                                  baseTableName="CALL_CACHING_JOB_DETRITUS"
                                  baseColumnNames="RESULT_METAINFO_ID"
diff --git a/database/migration/src/main/resources/changesets/call_result_caching.xml b/database/migration/src/main/resources/changesets/call_result_caching.xml
index c979e3e604e..caf7a84569f 100644
--- a/database/migration/src/main/resources/changesets/call_result_caching.xml
+++ b/database/migration/src/main/resources/changesets/call_result_caching.xml
@@ -2,9 +2,9 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <property name="boolean.type" value="BOOLEAN" dbms="!mysql"/>
+    <property name="boolean.type" value="BOOLEAN" dbms="hsqldb"/>
     <property name="boolean.type" value="TINYINT" dbms="mysql"/>
-    <changeSet author="chrisl" id="call_result_caching">
+    <changeSet author="chrisl" id="call_result_caching" dbms="mysql,hsqldb">
         <comment>Adding some tracking columns for determining eligibility for Call Result Caching.</comment>
         <addColumn tableName="EXECUTION">
             <column name="ALLOWS_RESULT_REUSE" type="${boolean.type}" defaultValueBoolean="true" >
@@ -38,4 +38,4 @@
                                  constraintName="FK_RESULTS_CLONED_FROM"
                                  onDelete="SET NULL" />
     </changeSet>
-</databaseChangeLog>
\ No newline at end of file
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/callcaching.xml b/database/migration/src/main/resources/changesets/callcaching.xml
index 69f87ef6554..391c7a6d26e 100644
--- a/database/migration/src/main/resources/changesets/callcaching.xml
+++ b/database/migration/src/main/resources/changesets/callcaching.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="chrisl" id="call_caching_result_metainfo">
+    <changeSet author="chrisl" id="call_caching_result_metainfo" dbms="mysql,hsqldb">
         <comment>
             One row per cached job result.
             Stores meta info about which job the result came from.
@@ -41,12 +41,12 @@
     </changeSet>
 
     <!-- We don't want to store results for the exact same job twice -->
-    <changeSet author="chrisl" id="call_caching_result_metainfo_uniqueness">
+    <changeSet author="chrisl" id="call_caching_result_metainfo_uniqueness" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_CALL_CACHING_RESULT_METAINFO"
                              tableName="CALL_CACHING_RESULT_METAINFO" columnNames="WORKFLOW_UUID, CALL_FQN, JOB_SCATTER_INDEX" />
     </changeSet>
 
-    <changeSet author="chrisl" id="call_caching_hashes">
+    <changeSet author="chrisl" id="call_caching_hashes" dbms="mysql,hsqldb">
         <comment>
             One row per hashkey per call cache meta info.
             Allows us to link hash keys and values to any matching call cache results.
@@ -71,13 +71,13 @@
     </changeSet>
 
     <!-- We don't want to store the same hash key twice for the same RESULT_METAINFO_ID -->
-    <changeSet author="chrisl" id="call_caching_result_hashes_uniqueness">
+    <changeSet author="chrisl" id="call_caching_result_hashes_uniqueness" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_CALL_CACHING_HASH"
                              tableName="CALL_CACHING_HASH" columnNames="HASH_KEY, RESULT_METAINFO_ID" />
     </changeSet>
 
     <!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
-    <changeSet author="chrisl" id="call_caching_result_hashes_uniqueness_foreign_key">
+    <changeSet author="chrisl" id="call_caching_result_hashes_uniqueness_foreign_key" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="CCH_RESULT_METAINFO_ID_FK"
                                  baseTableName="CALL_CACHING_HASH"
                                  baseColumnNames="RESULT_METAINFO_ID"
@@ -86,7 +86,7 @@
         />
     </changeSet>
 
-    <changeSet author="chrisl" id="call_caching_result_simpletons">
+    <changeSet author="chrisl" id="call_caching_result_simpletons" dbms="mysql,hsqldb">
         <comment>
             One row per result simpleton in the job result.
             Simpleton: a single non-complex WDL value.
@@ -115,13 +115,13 @@
     </changeSet>
 
     <!-- Unlike the metadata journal, we never want to store the same simpleton key twice (for the same RESULT_METAINFO_ID) -->
-    <changeSet author="chrisl" id="call_caching_result_simpletons_uniqueness">
+    <changeSet author="chrisl" id="call_caching_result_simpletons_uniqueness" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_CALL_CACHING_RESULT_SIMPLETON"
                              tableName="CALL_CACHING_RESULT_SIMPLETON" columnNames="SIMPLETON_KEY, RESULT_METAINFO_ID" />
     </changeSet>
 
     <!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
-    <changeSet author="chrisl" id="call_caching_result_simpletons_foreign_key">
+    <changeSet author="chrisl" id="call_caching_result_simpletons_foreign_key" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="CCRS_RESULT_METAINFO_ID_FK"
                                  baseTableName="CALL_CACHING_RESULT_SIMPLETON"
                                  baseColumnNames="RESULT_METAINFO_ID"
diff --git a/database/migration/src/main/resources/changesets/change_execution_unique_constraint.xml b/database/migration/src/main/resources/changesets/change_execution_unique_constraint.xml
index 5b0449bd98c..700987cda08 100644
--- a/database/migration/src/main/resources/changesets/change_execution_unique_constraint.xml
+++ b/database/migration/src/main/resources/changesets/change_execution_unique_constraint.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="change_execution_unique_constraint">
+    <changeSet author="tjeandet" id="change_execution_unique_constraint" dbms="mysql,hsqldb">
         <comment>
             Change unique constraint for Execution Table to include IDX column.  For MySQL this requires
             first dropping the foreign key constraint, which we then restore after adding back the enhanced
diff --git a/database/migration/src/main/resources/changesets/change_max_size_label_entry.xml b/database/migration/src/main/resources/changesets/change_max_size_label_entry.xml
index dd82e754bd3..22bf0f6f760 100644
--- a/database/migration/src/main/resources/changesets/change_max_size_label_entry.xml
+++ b/database/migration/src/main/resources/changesets/change_max_size_label_entry.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="sshah" id="change_max_size_for_key_and_value_in_label">
+    <changeSet author="sshah" id="change_max_size_for_key_and_value_in_label" dbms="mysql,hsqldb">
         <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_KEY" newDataType="VARCHAR(255)"/>
         <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_VALUE" newDataType="VARCHAR(255)"/>
     </changeSet>
diff --git a/database/migration/src/main/resources/changesets/change_max_size_workflow_url.xml b/database/migration/src/main/resources/changesets/change_max_size_workflow_url.xml
index b549371b65d..d9512d4626a 100644
--- a/database/migration/src/main/resources/changesets/change_max_size_workflow_url.xml
+++ b/database/migration/src/main/resources/changesets/change_max_size_workflow_url.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="sshah" id="change_max_size_for_workflow_url">
+    <changeSet author="sshah" id="change_max_size_for_workflow_url" dbms="mysql,hsqldb">
         <modifyDataType tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_URL" newDataType="VARCHAR(2000)"/>
     </changeSet>
 
diff --git a/database/migration/src/main/resources/changesets/custom_label_entry.xml b/database/migration/src/main/resources/changesets/custom_label_entry.xml
index 49f09da8534..a207982f0bc 100644
--- a/database/migration/src/main/resources/changesets/custom_label_entry.xml
+++ b/database/migration/src/main/resources/changesets/custom_label_entry.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet id="custom_LABEL_entry" author="rmunshi">
+    <changeSet id="custom_LABEL_entry" author="rmunshi" dbms="mysql,hsqldb">
         <createTable tableName="CUSTOM_LABEL_ENTRY">
             <column autoIncrement="true" name="CUSTOM_LABEL_ENTRY_ID" type="BIGINT">
                 <constraints primaryKey="true" primaryKeyName="PK_CUSTOM_LABEL_ENTRY"/>
@@ -20,7 +20,7 @@
         </createTable>
     </changeSet>
 
-    <changeSet author="rmunshi" id="WMSE_workflow_execution_uuid_foreign_key">
+    <changeSet author="rmunshi" id="WMSE_workflow_execution_uuid_foreign_key" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID"
                                  baseTableName="CUSTOM_LABEL_ENTRY"
                                  baseColumnNames="WORKFLOW_EXECUTION_UUID"
@@ -30,13 +30,13 @@
         />
     </changeSet>
 
-    <changeSet author="rmunshi" id="custom_label_entry_index">
+    <changeSet author="rmunshi" id="custom_label_entry_index" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UC_CUSTOM_LABEL_ENTRY_CLK_CLV_WEU"
                              tableName="CUSTOM_LABEL_ENTRY" columnNames="CUSTOM_LABEL_KEY, CUSTOM_LABEL_VALUE, WORKFLOW_EXECUTION_UUID" />
 
     </changeSet>
 
-    <changeSet author="rmunshi" id="modify label key/value data types">
+    <changeSet author="rmunshi" id="modify label key/value data types" dbms="mysql,hsqldb">
         <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_KEY" newDataType="VARCHAR(63)"/>
         <modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_VALUE" newDataType="VARCHAR(63)"/>
     </changeSet>
diff --git a/database/migration/src/main/resources/changesets/db_schema.xml b/database/migration/src/main/resources/changesets/db_schema.xml
index 6fa0c397eee..1aad77dd3d7 100644
--- a/database/migration/src/main/resources/changesets/db_schema.xml
+++ b/database/migration/src/main/resources/changesets/db_schema.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="scottfrazer" id="db_schema_other_table_alldb">
+    <changeSet author="scottfrazer" id="db_schema_other_table_alldb" dbms="mysql,hsqldb">
         <createTable tableName="WORKFLOW_EXECUTION">
             <column autoIncrement="true" name="WORKFLOW_EXECUTION_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_WORKFLOW_EXECUTION"/>
@@ -104,7 +104,7 @@
         </modifySql>
     </changeSet>
 
-    <changeSet id="db_schema_symbol_table_otherdb" author="scottfrazer" dbms="!mysql">
+    <changeSet id="db_schema_symbol_table_otherdb" author="scottfrazer" dbms="hsqldb">
         <createTable tableName="SYMBOL">
             <column autoIncrement="true" name="SYMBOL_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_SYMBOL"/>
@@ -131,7 +131,7 @@
         </createTable>
     </changeSet>
 
-    <changeSet author="scottfrazer" id="db_schema_constraints">
+    <changeSet author="scottfrazer" id="db_schema_constraints" dbms="mysql,hsqldb">
         <createIndex indexName="STATUS_IDX"
                      tableName="WORKFLOW_EXECUTION" unique="false">
             <column name="STATUS"/>
diff --git a/database/migration/src/main/resources/changesets/docker_hash_store.xml b/database/migration/src/main/resources/changesets/docker_hash_store.xml
index 7ec9fbbc4ff..37b3971daf1 100644
--- a/database/migration/src/main/resources/changesets/docker_hash_store.xml
+++ b/database/migration/src/main/resources/changesets/docker_hash_store.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="DOCKER_HASH_STORE_ENTRY">
+    <changeSet author="tjeandet" id="DOCKER_HASH_STORE_ENTRY" dbms="mysql,hsqldb">
         <comment>
             Temporary storage area for docker hashes from workflows that are still in progress.
         </comment>
@@ -26,7 +26,7 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="tjeandet" id="docker_hash_store_uuid_unique_constraint">
+    <changeSet author="tjeandet" id="docker_hash_store_uuid_unique_constraint" dbms="mysql,hsqldb">
         <addUniqueConstraint
                 constraintName="UC_DOCKER_HASH_STORE_ENTRY_WEU_DT"
                 tableName="DOCKER_HASH_STORE_ENTRY"
diff --git a/database/migration/src/main/resources/changesets/docker_hash_store_add_size_column.xml b/database/migration/src/main/resources/changesets/docker_hash_store_add_size_column.xml
index aaf8724faf6..3a574b6c1c4 100644
--- a/database/migration/src/main/resources/changesets/docker_hash_store_add_size_column.xml
+++ b/database/migration/src/main/resources/changesets/docker_hash_store_add_size_column.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="docker_hash_store_entry_add_size_column">
+    <changeSet author="tjeandet" id="docker_hash_store_entry_add_size_column" dbms="mysql,hsqldb">
         <comment>
             Add a size column corresponding to the sum of all the layers size from the manifest
         </comment>
diff --git a/database/migration/src/main/resources/changesets/drop_workflow_uri_and_local_command.xml b/database/migration/src/main/resources/changesets/drop_workflow_uri_and_local_command.xml
index 645a496c41b..863c31733ed 100644
--- a/database/migration/src/main/resources/changesets/drop_workflow_uri_and_local_command.xml
+++ b/database/migration/src/main/resources/changesets/drop_workflow_uri_and_local_command.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="drop_workflow_uri_and_local_command">
+    <changeSet author="kshakir" id="drop_workflow_uri_and_local_command" dbms="mysql,hsqldb">
         <comment>
             Workflow URI is not needed in the DB.
             Local jobs don't need to store the command either.
diff --git a/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml b/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml
index 096f17a4c63..d1c9c397b57 100644
--- a/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml
+++ b/database/migration/src/main/resources/changesets/embiggen_detritus_value.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="rmunshi" id="embiggen_call_caching_detritus_entry">
+    <changeSet author="rmunshi" id="embiggen_call_caching_detritus_entry" dbms="mysql,hsqldb">
         <modifyDataType tableName="CALL_CACHING_DETRITUS_ENTRY"
                         columnName="DETRITUS_VALUE"
                         newDataType="${clob.type}"/>
diff --git a/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml b/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml
index e0b1ed38d85..d965a9b9ee1 100644
--- a/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml
+++ b/database/migration/src/main/resources/changesets/embiggen_metadata_value.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="entry_or_journal_existence_xor">
+    <changeSet author="mcovarr" id="entry_or_journal_existence_xor" dbms="mysql,hsqldb">
         <preConditions>
             <or>
                 <and>
@@ -25,7 +25,7 @@
         </comment>
     </changeSet>
 
-    <changeSet author="mcovarr" id="embiggen_metadata_entry">
+    <changeSet author="mcovarr" id="embiggen_metadata_entry" dbms="mysql,hsqldb">
         <preConditions onFail="MARK_RAN">
             <tableExists tableName="METADATA_ENTRY"/>
         </preConditions>
@@ -36,7 +36,7 @@
         <modifyDataType tableName="METADATA_ENTRY" columnName="METADATA_VALUE" newDataType="LONGTEXT"/>
     </changeSet>
 
-    <changeSet author="mcovarr" id="embiggen_metadata_journal">
+    <changeSet author="mcovarr" id="embiggen_metadata_journal" dbms="mysql,hsqldb">
         <preConditions onFail="MARK_RAN">
             <tableExists tableName="METADATA_JOURNAL"/>
         </preConditions>
diff --git a/database/migration/src/main/resources/changesets/encrypt_and_clear_workflow_options.xml b/database/migration/src/main/resources/changesets/encrypt_and_clear_workflow_options.xml
index 2fe2d766100..e44422d1fda 100644
--- a/database/migration/src/main/resources/changesets/encrypt_and_clear_workflow_options.xml
+++ b/database/migration/src/main/resources/changesets/encrypt_and_clear_workflow_options.xml
@@ -3,11 +3,11 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="encrypt_workflow_store_entry_workflow_options">
+    <changeSet author="kshakir" id="encrypt_workflow_store_entry_workflow_options" dbms="mysql,hsqldb">
         <customChange class="cromwell.database.migration.workflowoptions.EncryptWorkflowStoreEntryWorkflowOptions"/>
     </changeSet>
 
-    <changeSet author="kshakir" id="clear_metadata_entry_workflow_options">
+    <changeSet author="kshakir" id="clear_metadata_entry_workflow_options" dbms="mysql,hsqldb">
         <customChange class="cromwell.database.migration.workflowoptions.ClearMetadataEntryWorkflowOptions"/>
     </changeSet>
 
diff --git a/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
index 9e69c154730..e875f45dc0d 100644
--- a/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
+++ b/database/migration/src/main/resources/changesets/enlarge_call_caching_hash_entry_id.xml
@@ -8,7 +8,7 @@
          could both alter the datatype of and add autoincrement to the PK in one shot. The changeset cannot be renamed
          as the name is part of the Liquibase key that will prevent it from running again in environments that already
          suffered through the old two-changeset migration. -->
-    <changeSet author="mcovarr" id="restore_auto_increment_call_caching_hash_entry_id">
+    <changeSet author="mcovarr" id="restore_auto_increment_call_caching_hash_entry_id" dbms="mysql,hsqldb">
         <!-- This is the original comment from the two-changeset version of the code and it doesn't make a lot of sense in
              the new one-changeset reality. Removing the comment doesn't alter the Liquibase hash but it does make the
              resulting row in the DATABASECHANGELOG different. Keeping it for now out of an abundance of caution. -->
diff --git a/database/migration/src/main/resources/changesets/events_table.xml b/database/migration/src/main/resources/changesets/events_table.xml
index 47014b6f8fb..681ba71c452 100644
--- a/database/migration/src/main/resources/changesets/events_table.xml
+++ b/database/migration/src/main/resources/changesets/events_table.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="chrisl" id="execution_event_table">
+    <changeSet author="chrisl" id="execution_event_table" dbms="mysql,hsqldb">
         <createTable tableName="EXECUTION_EVENT">
             <column autoIncrement="true" name="EVENT_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_EVENT"/>
diff --git a/database/migration/src/main/resources/changesets/execution_backend_info.xml b/database/migration/src/main/resources/changesets/execution_backend_info.xml
index 1368b9d1b4e..551112a6321 100644
--- a/database/migration/src/main/resources/changesets/execution_backend_info.xml
+++ b/database/migration/src/main/resources/changesets/execution_backend_info.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="CREATE_EXECUTION_INFO">
+    <changeSet author="mcovarr" id="CREATE_EXECUTION_INFO" dbms="mysql,hsqldb">
         <createTable tableName="EXECUTION_INFO">
             <column autoIncrement="true" name="EXECUTION_INFO_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_EXECUTION_INFO"/>
@@ -20,14 +20,14 @@
         </createTable>
     </changeSet>
 
-    <changeSet id="add_foreign_key_constraints" author="mcovarr">
+    <changeSet id="add_foreign_key_constraints" author="mcovarr" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="FK_EXECUTION_INFO_EXECUTION"
                                  baseTableName="EXECUTION_INFO" baseColumnNames="EXECUTION_ID"
                                  referencedTableName="EXECUTION"
                                  referencedColumnNames="EXECUTION_ID"/>
     </changeSet>
 
-    <changeSet id="add_backend_type_column" author="mcovarr">
+    <changeSet id="add_backend_type_column" author="mcovarr" dbms="mysql,hsqldb">
         <addColumn tableName="EXECUTION">
             <column name="BACKEND_TYPE" type="VARCHAR(255)">
                 <constraints nullable="true"/>
@@ -35,7 +35,7 @@
         </addColumn>
     </changeSet>
 
-    <changeSet id="insert_data_from_backend_tables" author="mcovarr">
+    <changeSet id="insert_data_from_backend_tables" author="mcovarr" dbms="mysql,hsqldb">
         <sql>
             insert into EXECUTION_INFO(EXECUTION_ID, INFO_KEY, INFO_VALUE)
             select EXECUTION_ID, "JES_RUN_ID", JES_RUN_ID from JES_JOB;
@@ -51,7 +51,7 @@
         </sql>
     </changeSet>
 
-    <changeSet id="set_backend_type" author="mcovarr">
+    <changeSet id="set_backend_type" author="mcovarr" dbms="mysql,hsqldb">
         <sql>
             update EXECUTION e set BACKEND_TYPE = 'JES' where exists
             (select 1 from JES_JOB jj where jj.EXECUTION_ID = e.EXECUTION_ID);
@@ -64,19 +64,19 @@
         </sql>
     </changeSet>
 
-    <changeSet id="drop_job_tables" author="mcovarr">
+    <changeSet id="drop_job_tables" author="mcovarr" dbms="mysql,hsqldb">
         <dropTable tableName="JES_JOB" />
         <dropTable tableName="SGE_JOB" />
         <dropTable tableName="LOCAL_JOB" />
     </changeSet>
 
-    <changeSet id="execution_info_uniqueness_constraint" author="mcovarr">
+    <changeSet id="execution_info_uniqueness_constraint" author="mcovarr" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_EXECUTION_INFO"
                              tableName="EXECUTION_INFO"
                              columnNames="EXECUTION_ID, INFO_KEY" />
     </changeSet>
 
-    <changeSet id="execution_backend_type_not_null" author="mcovarr">
+    <changeSet id="execution_backend_type_not_null" author="mcovarr" dbms="mysql,hsqldb">
         <addNotNullConstraint tableName="EXECUTION"
                               columnName="BACKEND_TYPE"
                               columnDataType="VARCHAR(255)"/>
diff --git a/database/migration/src/main/resources/changesets/failure_table.xml b/database/migration/src/main/resources/changesets/failure_table.xml
index 295e8d366ba..c00a73edc5b 100644
--- a/database/migration/src/main/resources/changesets/failure_table.xml
+++ b/database/migration/src/main/resources/changesets/failure_table.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="chrisl" id="failure_table">
+    <changeSet author="chrisl" id="failure_table" dbms="mysql,hsqldb">
         <createTable tableName="FAILURE_EVENT">
             <column autoIncrement="true" name="FAILURE_EVENT_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_FAILURE_EVENT"/>
diff --git a/database/migration/src/main/resources/changesets/jes_id_update.xml b/database/migration/src/main/resources/changesets/jes_id_update.xml
index cbf10fd7f50..b28ec05ac0f 100644
--- a/database/migration/src/main/resources/changesets/jes_id_update.xml
+++ b/database/migration/src/main/resources/changesets/jes_id_update.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="chrisl" id="jes_id_update">
+    <changeSet author="chrisl" id="jes_id_update" dbms="mysql,hsqldb">
         <modifyDataType
                 tableName="JES_JOB"
                 columnName="JES_ID"
diff --git a/database/migration/src/main/resources/changesets/job_store.xml b/database/migration/src/main/resources/changesets/job_store.xml
index df1e89524cd..5c424f3bebd 100644
--- a/database/migration/src/main/resources/changesets/job_store.xml
+++ b/database/migration/src/main/resources/changesets/job_store.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="chrisl" id="JOB_STORE">
+    <changeSet author="chrisl" id="JOB_STORE" dbms="mysql,hsqldb">
         <comment>
             Temporary storage area for completed jobs which belong to workflows that are still in progress.
         </comment>
@@ -49,14 +49,14 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="chrisl" id="job_store_uuid_index">
+    <changeSet author="chrisl" id="job_store_uuid_index" dbms="mysql,hsqldb">
         <createIndex indexName="JOB_STORE_UUID_IDX"
                      tableName="JOB_STORE">
             <column name="WORKFLOW_UUID"/>
         </createIndex>
     </changeSet>
 
-    <changeSet author="chrisl" id="job_store_jobkey_index">
+    <changeSet author="chrisl" id="job_store_jobkey_index" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="JOB_STORE_JOBKEY_IDX"
                      tableName="JOB_STORE" columnNames="WORKFLOW_UUID, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT" />
 
diff --git a/database/migration/src/main/resources/changesets/job_store_simpletons.xml b/database/migration/src/main/resources/changesets/job_store_simpletons.xml
index a251021bfb0..d6e60308018 100644
--- a/database/migration/src/main/resources/changesets/job_store_simpletons.xml
+++ b/database/migration/src/main/resources/changesets/job_store_simpletons.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="mcovarr" id="job_store_result_simpletons">
+    <changeSet author="mcovarr" id="job_store_result_simpletons" dbms="mysql,hsqldb">
         <comment>
             One row per result simpleton in the job result.
             Simpleton: a single non-complex WDL value.
@@ -35,13 +35,13 @@
     </changeSet>
 
     <!-- Unlike the metadata journal, we never want to store the same simpleton key twice (for the same RESULT_METAINFO_ID) -->
-    <changeSet author="mcovarr" id="job_store_result_simpletons_uniqueness">
+    <changeSet author="mcovarr" id="job_store_result_simpletons_uniqueness" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UK_JOB_STORE_RESULT_SIMPLETON"
                              tableName="JOB_STORE_RESULT_SIMPLETON" columnNames="SIMPLETON_KEY, JOB_STORE_ID" />
     </changeSet>
 
     <!-- Make JOB_STORE_ID a FK into JOB_STORE -->
-    <changeSet author="mcovarr" id="job_store_result_simpletons_foreign_key">
+    <changeSet author="mcovarr" id="job_store_result_simpletons_foreign_key" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="JSRS_JOB_STORE_FK"
                                  baseTableName="JOB_STORE_RESULT_SIMPLETON"
                                  baseColumnNames="JOB_STORE_ID"
@@ -51,7 +51,7 @@
         />
     </changeSet>
 
-    <changeSet id="job_store_remove_job_output" author="mcovarr">
+    <changeSet id="job_store_remove_job_output" author="mcovarr" dbms="mysql,hsqldb">
         <comment>
             There is no attempt at migrating the contents of JOB_STORE.JOB_OUTPUTS to simpletons, this just removes the column.
         </comment>
diff --git a/database/migration/src/main/resources/changesets/job_store_tinyints.xml b/database/migration/src/main/resources/changesets/job_store_tinyints.xml
index 1e91a696f09..cfe263d18bc 100644
--- a/database/migration/src/main/resources/changesets/job_store_tinyints.xml
+++ b/database/migration/src/main/resources/changesets/job_store_tinyints.xml
@@ -3,23 +3,23 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <property name="boolean.type" value="BOOLEAN" dbms="!mysql"/>
+    <property name="boolean.type" value="BOOLEAN" dbms="hsqldb"/>
     <property name="boolean.type" value="TINYINT" dbms="mysql"/>
 
-    <changeSet id="job_store_fix_job_successful" author="mcovarr">
+    <changeSet id="job_store_fix_job_successful" author="mcovarr" dbms="mysql,hsqldb">
         <modifyDataType columnName="JOB_SUCCESSFUL"
                         newDataType="${boolean.type}"
                         tableName="JOB_STORE"/>
     </changeSet>
 
-    <changeSet id="job_store_fix_job_retryable_failure" author="mcovarr">
+    <changeSet id="job_store_fix_job_retryable_failure" author="mcovarr" dbms="mysql,hsqldb">
         <modifyDataType columnName="RETRYABLE_FAILURE"
                         newDataType="${boolean.type}"
                         tableName="JOB_STORE"/>
 
     </changeSet>
 
-    <changeSet id="job_store_not_nullable_job_successful" author="mcovarr">
+    <changeSet id="job_store_not_nullable_job_successful" author="mcovarr" dbms="mysql,hsqldb">
         <addNotNullConstraint tableName="JOB_STORE"
                               columnName="JOB_SUCCESSFUL"
                               columnDataType="${boolean.type}" />
diff --git a/database/migration/src/main/resources/changesets/lengthen_wdl_value.xml b/database/migration/src/main/resources/changesets/lengthen_wdl_value.xml
index a138d58e30c..6595fe7f194 100644
--- a/database/migration/src/main/resources/changesets/lengthen_wdl_value.xml
+++ b/database/migration/src/main/resources/changesets/lengthen_wdl_value.xml
@@ -4,7 +4,7 @@
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
-    <changeSet author="sfrazer" id="LENGTHEN_WDL_VALUE">
+    <changeSet author="sfrazer" id="LENGTHEN_WDL_VALUE" dbms="mysql,hsqldb">
         <comment>
             WDL_VALUE should accept large strings
         </comment>
diff --git a/database/migration/src/main/resources/changesets/local_job_allow_null.xml b/database/migration/src/main/resources/changesets/local_job_allow_null.xml
index a4b564f1eeb..81598f88da6 100644
--- a/database/migration/src/main/resources/changesets/local_job_allow_null.xml
+++ b/database/migration/src/main/resources/changesets/local_job_allow_null.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="local_job_allow_null">
+    <changeSet author="kshakir" id="local_job_allow_null" dbms="mysql,hsqldb">
         <comment>
             The local jobs don't have process ID and result codes at the start.
         </comment>
diff --git a/database/migration/src/main/resources/changesets/metadata_journal.xml b/database/migration/src/main/resources/changesets/metadata_journal.xml
index 0eeedc27a5a..99ba1d9382f 100644
--- a/database/migration/src/main/resources/changesets/metadata_journal.xml
+++ b/database/migration/src/main/resources/changesets/metadata_journal.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="metadata_journal">
+    <changeSet author="mcovarr" id="metadata_journal" dbms="mysql,hsqldb">
         <createTable tableName="METADATA_JOURNAL">
             <column autoIncrement="true" name="METADATA_JOURNAL_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_METADATA_JOURNAL"/>
@@ -36,14 +36,14 @@
         </modifySql>
     </changeSet>
 
-    <changeSet id="metadata_workflow_index" author="mcovarr">
+    <changeSet id="metadata_workflow_index" author="mcovarr" dbms="mysql,hsqldb">
         <createIndex indexName="METADATA_WORKFLOW_IDX"
                      tableName="METADATA_JOURNAL" unique="false">
             <column name="WORKFLOW_EXECUTION_UUID"/>
         </createIndex>
     </changeSet>
 
-    <changeSet id="metadata_job_index" author="mcovarr">
+    <changeSet id="metadata_job_index" author="mcovarr" dbms="mysql,hsqldb">
         <createIndex indexName="METADATA_JOB_IDX"
                      tableName="METADATA_JOURNAL" unique="false">
             <column name="WORKFLOW_EXECUTION_UUID"/>
@@ -53,7 +53,7 @@
         </createIndex>
     </changeSet>
 
-    <changeSet id="metadata_job_and_key_index" author="mcovarr">
+    <changeSet id="metadata_job_and_key_index" author="mcovarr" dbms="mysql,hsqldb">
         <createIndex indexName="METADATA_JOB_AND_KEY_IDX"
                      tableName="METADATA_JOURNAL" unique="false">
             <column name="WORKFLOW_EXECUTION_UUID"/>
diff --git a/database/migration/src/main/resources/changesets/metadata_journal_subsecond_timestamp.xml b/database/migration/src/main/resources/changesets/metadata_journal_subsecond_timestamp.xml
index ae463a19225..d1751af3d9a 100644
--- a/database/migration/src/main/resources/changesets/metadata_journal_subsecond_timestamp.xml
+++ b/database/migration/src/main/resources/changesets/metadata_journal_subsecond_timestamp.xml
@@ -3,14 +3,14 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet id="metadata_journal_subsecond_timestamp" author="mcovarr">
+    <changeSet id="metadata_journal_subsecond_timestamp" author="mcovarr" dbms="mysql,hsqldb">
         <modifyDataType
                 columnName="METADATA_TIMESTAMP"
                 newDataType="DATETIME(6)"
                 tableName="METADATA_JOURNAL"/>
     </changeSet>
 
-    <changeSet id="metadata_journal_timestamp_not_null" author="mcovarr">
+    <changeSet id="metadata_journal_timestamp_not_null" author="mcovarr" dbms="mysql,hsqldb">
         <addNotNullConstraint
                 tableName="METADATA_JOURNAL"
                 columnName="METADATA_TIMESTAMP"
diff --git a/database/migration/src/main/resources/changesets/metadata_typed_values.xml b/database/migration/src/main/resources/changesets/metadata_typed_values.xml
index 70a9f213668..acfcb2c35b8 100644
--- a/database/migration/src/main/resources/changesets/metadata_typed_values.xml
+++ b/database/migration/src/main/resources/changesets/metadata_typed_values.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="metadata_typed_values">
+    <changeSet author="tjeandet" id="metadata_typed_values" dbms="mysql,hsqldb">
         <addColumn
                 tableName="METADATA_JOURNAL">
             <column name="METADATA_VALUE_TYPE" type="VARCHAR(10)" />
diff --git a/database/migration/src/main/resources/changesets/move_sql_metadata_changelog.xml b/database/migration/src/main/resources/changesets/move_sql_metadata_changelog.xml
index dc6c81b59cc..2edd9137aad 100644
--- a/database/migration/src/main/resources/changesets/move_sql_metadata_changelog.xml
+++ b/database/migration/src/main/resources/changesets/move_sql_metadata_changelog.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="move_metadata_changelog">
+    <changeSet author="kshakir" id="move_metadata_changelog" dbms="mysql,hsqldb">
         <preConditions onFail="MARK_RAN">
             <sqlCheck expectedResult="0">SELECT COUNT(1) FROM METADATA_ENTRY</sqlCheck>
         </preConditions>
diff --git a/database/migration/src/main/resources/changesets/nullable_lobs.xml b/database/migration/src/main/resources/changesets/nullable_lobs.xml
index 13783585c79..9f977a0aed6 100644
--- a/database/migration/src/main/resources/changesets/nullable_lobs.xml
+++ b/database/migration/src/main/resources/changesets/nullable_lobs.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="kshakir" id="nullable_lobs">
+    <changeSet author="kshakir" id="nullable_lobs" dbms="mysql,hsqldb">
         <dropNotNullConstraint columnDataType="LONGTEXT"
                                tableName="CALL_CACHING_DETRITUS_ENTRY" columnName="DETRITUS_VALUE"/>
         <dropNotNullConstraint columnDataType="LONGTEXT"
diff --git a/database/migration/src/main/resources/changesets/optional_sge_ids.xml b/database/migration/src/main/resources/changesets/optional_sge_ids.xml
index 386208adb65..44613a71d7b 100644
--- a/database/migration/src/main/resources/changesets/optional_sge_ids.xml
+++ b/database/migration/src/main/resources/changesets/optional_sge_ids.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="chrisl" id="optional_sge_ids">
+    <changeSet author="chrisl" id="optional_sge_ids" dbms="mysql,hsqldb">
         <modifyDataType
                 tableName="SGE_JOB"
                 columnName="SGE_JOB_NUMBER"
diff --git a/database/migration/src/main/resources/changesets/postgresql_engine_schema.xml b/database/migration/src/main/resources/changesets/postgresql_engine_schema.xml
new file mode 100644
index 00000000000..df1ea8a1a6c
--- /dev/null
+++ b/database/migration/src/main/resources/changesets/postgresql_engine_schema.xml
@@ -0,0 +1,326 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog objectQuotingStrategy="QUOTE_ALL_OBJECTS"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet author="nechols" id="postgresql-engine-init-1" dbms="postgresql">
+        <createTable tableName="CALL_CACHING_AGGREGATION_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_AGGREGATION_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_AGGREGATION_ENTRY"/>
+            </column>
+            <column name="CALL_CACHING_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="BASE_AGGREGATION" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="INPUT_FILES_AGGREGATION" type="VARCHAR(255)"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-2" dbms="postgresql">
+        <createTable tableName="CALL_CACHING_DETRITUS_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_DETRITUS_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_DETRITUS_ENTRY"/>
+            </column>
+            <column name="DETRITUS_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DETRITUS_VALUE" type="CLOB"/>
+            <column name="CALL_CACHING_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-3" dbms="postgresql">
+        <createTable tableName="CALL_CACHING_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_FULLY_QUALIFIED_NAME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_INDEX" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="RETURN_CODE" type="INT"/>
+            <column defaultValueBoolean="true" name="ALLOW_RESULT_REUSE" type="BOOLEAN">
+                <constraints nullable="false"/>
+            </column>
+            <column defaultValueComputed="NULL" name="JOB_ATTEMPT" type="INT"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-4" dbms="postgresql">
+        <createTable tableName="CALL_CACHING_HASH_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_HASH_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_HASH_ENTRY"/>
+            </column>
+            <column name="HASH_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="HASH_VALUE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_CACHING_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-5" dbms="postgresql">
+        <createTable tableName="CALL_CACHING_SIMPLETON_ENTRY">
+            <column autoIncrement="true" name="CALL_CACHING_SIMPLETON_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CALL_CACHING_SIMPLETON_ENTRY"/>
+            </column>
+            <column name="SIMPLETON_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SIMPLETON_VALUE" type="CLOB"/>
+            <column name="WDL_TYPE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_CACHING_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-6" dbms="postgresql">
+        <createTable tableName="DOCKER_HASH_STORE_ENTRY">
+            <column autoIncrement="true" name="DOCKER_HASH_STORE_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_DOCKER_HASH_STORE_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DOCKER_TAG" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="DOCKER_HASH" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column defaultValueComputed="NULL" name="DOCKER_SIZE" type="BIGINT"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-7" dbms="postgresql">
+        <createTable tableName="JOB_KEY_VALUE_ENTRY">
+            <column autoIncrement="true" name="JOB_KEY_VALUE_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_JOB_KEY_VALUE_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_FULLY_QUALIFIED_NAME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_INDEX" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_ATTEMPT" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="STORE_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="STORE_VALUE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-8" dbms="postgresql">
+        <createTable tableName="JOB_STORE_ENTRY">
+            <column autoIncrement="true" name="JOB_STORE_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_JOB_STORE_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_FULLY_QUALIFIED_NAME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_INDEX" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_ATTEMPT" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_SUCCESSFUL" type="BOOLEAN">
+                <constraints nullable="false"/>
+            </column>
+            <column name="RETURN_CODE" type="INT"/>
+            <column name="EXCEPTION_MESSAGE" type="CLOB"/>
+            <column name="RETRYABLE_FAILURE" type="BOOLEAN"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-9" dbms="postgresql">
+        <createTable tableName="JOB_STORE_SIMPLETON_ENTRY">
+            <column autoIncrement="true" name="JOB_STORE_SIMPLETON_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_JOB_STORE_SIMPLETON_ENTRY"/>
+            </column>
+            <column name="SIMPLETON_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SIMPLETON_VALUE" type="CLOB"/>
+            <column name="WDL_TYPE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="JOB_STORE_ENTRY_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-10" dbms="postgresql">
+        <createTable tableName="SUB_WORKFLOW_STORE_ENTRY">
+            <column autoIncrement="true" name="SUB_WORKFLOW_STORE_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_SUB_WORKFLOW_STORE_ENTRY"/>
+            </column>
+            <column name="ROOT_WORKFLOW_ID" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="PARENT_WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_FULLY_QUALIFIED_NAME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_INDEX" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_ATTEMPT" type="INT">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SUB_WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-11" dbms="postgresql">
+        <createTable tableName="WORKFLOW_STORE_ENTRY">
+            <column autoIncrement="true" name="WORKFLOW_STORE_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_WORKFLOW_STORE_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="WORKFLOW_DEFINITION" type="CLOB"/>
+            <column name="WORKFLOW_INPUTS" type="CLOB"/>
+            <column name="WORKFLOW_OPTIONS" type="CLOB"/>
+            <column name="WORKFLOW_STATE" type="VARCHAR(20)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SUBMISSION_TIME" type="TIMESTAMP">
+                <constraints nullable="false"/>
+            </column>
+            <column name="IMPORTS_ZIP" type="BLOB"/>
+            <column name="CUSTOM_LABELS" type="CLOB">
+                <constraints nullable="false"/>
+            </column>
+            <column name="WORKFLOW_TYPE" type="VARCHAR(30)"/>
+            <column name="WORKFLOW_TYPE_VERSION" type="VARCHAR(255)"/>
+            <column name="WORKFLOW_ROOT" type="VARCHAR(100)"/>
+            <column name="CROMWELL_ID" type="VARCHAR(100)"/>
+            <column name="HEARTBEAT_TIMESTAMP" type="TIMESTAMP"/>
+            <column name="WORKFLOW_URL" type="VARCHAR(2000)"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-12" dbms="postgresql">
+        <addUniqueConstraint columnNames="CALL_CACHING_ENTRY_ID, DETRITUS_KEY" constraintName="UC_CALL_CACHING_DETRITUS_ENTRY_CCEI_DK" tableName="CALL_CACHING_DETRITUS_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-13" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, JOB_INDEX" constraintName="UC_CALL_CACHING_ENTRY_WEU_CFQN_JI" tableName="CALL_CACHING_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-14" dbms="postgresql">
+        <addUniqueConstraint columnNames="CALL_CACHING_ENTRY_ID, HASH_KEY" constraintName="UC_CALL_CACHING_HASH_ENTRY_CCEI_HK" tableName="CALL_CACHING_HASH_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-15" dbms="postgresql">
+        <addUniqueConstraint columnNames="CALL_CACHING_ENTRY_ID, SIMPLETON_KEY" constraintName="UC_CALL_CACHING_SIMPLETON_ENTRY_CCEI_SK" tableName="CALL_CACHING_SIMPLETON_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-16" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID, DOCKER_TAG" constraintName="UC_DOCKER_HASH_STORE_ENTRY_WEU_DT" tableName="DOCKER_HASH_STORE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-17" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, JOB_INDEX, JOB_ATTEMPT, STORE_KEY" constraintName="UC_JOB_KEY_VALUE_ENTRY_WEU_CFQN_JI_JA_SK" tableName="JOB_KEY_VALUE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-18" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, JOB_INDEX, JOB_ATTEMPT" constraintName="UC_JOB_STORE_ENTRY_WEU_CFQN_JI_JA" tableName="JOB_STORE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-19" dbms="postgresql">
+        <addUniqueConstraint columnNames="JOB_STORE_ENTRY_ID, SIMPLETON_KEY" constraintName="UC_JOB_STORE_SIMPLETON_ENTRY_JSEI_SK" tableName="JOB_STORE_SIMPLETON_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-20" dbms="postgresql">
+        <addUniqueConstraint columnNames="PARENT_WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, CALL_INDEX, CALL_ATTEMPT" constraintName="UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA" tableName="SUB_WORKFLOW_STORE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-21" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID" constraintName="UC_WORKFLOW_STORE_ENTRY_WEU" tableName="WORKFLOW_STORE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-22" dbms="postgresql">
+        <createIndex indexName="IX_CALL_CACHING_AGGREGATION_ENTRY_BA_IFA" tableName="CALL_CACHING_AGGREGATION_ENTRY">
+            <column name="BASE_AGGREGATION"/>
+            <column name="INPUT_FILES_AGGREGATION"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-23" dbms="postgresql">
+        <createIndex indexName="IX_JOB_STORE_ENTRY_WEU" tableName="JOB_STORE_ENTRY">
+            <column name="WORKFLOW_EXECUTION_UUID"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-24" dbms="postgresql">
+        <createIndex indexName="IX_SUB_WORKFLOW_STORE_ENTRY_PWEU" tableName="SUB_WORKFLOW_STORE_ENTRY">
+            <column name="PARENT_WORKFLOW_EXECUTION_UUID"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-25" dbms="postgresql">
+        <createIndex indexName="IX_WORKFLOW_STORE_ENTRY_WS" tableName="WORKFLOW_STORE_ENTRY">
+            <column name="WORKFLOW_STATE"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-26" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID" baseTableName="CALL_CACHING_AGGREGATION_ENTRY" constraintName="FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID" referencedColumnNames="CALL_CACHING_ENTRY_ID" referencedTableName="CALL_CACHING_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-27" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID" baseTableName="CALL_CACHING_DETRITUS_ENTRY" constraintName="FK_CALL_CACHING_DETRITUS_ENTRY_CALL_CACHING_ENTRY_ID" referencedColumnNames="CALL_CACHING_ENTRY_ID" referencedTableName="CALL_CACHING_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-28" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID" baseTableName="CALL_CACHING_HASH_ENTRY" constraintName="FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID" referencedColumnNames="CALL_CACHING_ENTRY_ID" referencedTableName="CALL_CACHING_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-29" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID" baseTableName="CALL_CACHING_SIMPLETON_ENTRY" constraintName="FK_CALL_CACHING_SIMPLETON_ENTRY_CALL_CACHING_ENTRY_ID" referencedColumnNames="CALL_CACHING_ENTRY_ID" referencedTableName="CALL_CACHING_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-30" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="JOB_STORE_ENTRY_ID" baseTableName="JOB_STORE_SIMPLETON_ENTRY" constraintName="FK_JOB_STORE_SIMPLETON_ENTRY_JOB_STORE_ENTRY_ID" onDelete="CASCADE" referencedColumnNames="JOB_STORE_ENTRY_ID" referencedTableName="JOB_STORE_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-engine-init-31" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="ROOT_WORKFLOW_ID" baseTableName="SUB_WORKFLOW_STORE_ENTRY" constraintName="FK_SUB_WORKFLOW_STORE_ENTRY_ROOT_WORKFLOW_ID" onDelete="CASCADE" referencedColumnNames="WORKFLOW_STORE_ENTRY_ID" referencedTableName="WORKFLOW_STORE_ENTRY"/>
+    </changeSet>
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/rc.xml b/database/migration/src/main/resources/changesets/rc.xml
index 5f03d69ea35..b0425f5e42d 100644
--- a/database/migration/src/main/resources/changesets/rc.xml
+++ b/database/migration/src/main/resources/changesets/rc.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="rc">
+    <changeSet author="mcovarr" id="rc" dbms="mysql,hsqldb">
         <comment>Refactor the RC column off LOCAL_JOB up into EXECUTION since it should be usable by all backends.</comment>
         <dropColumn tableName="LOCAL_JOB" columnName="RC" />
         <addColumn tableName="EXECUTION">
@@ -11,4 +11,4 @@
         </addColumn>
     </changeSet>
 
-</databaseChangeLog>
\ No newline at end of file
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml b/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml
index e2892b91b93..613eb7bb2ab 100644
--- a/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml
+++ b/database/migration/src/main/resources/changesets/remove_pre_pbe_tables.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="drop-pre-pbe-tables">
+    <changeSet author="mcovarr" id="drop-pre-pbe-tables" dbms="mysql,hsqldb">
         <comment>Remove the old pre-pluggable backend tables.</comment>
         <dropTable tableName="EXECUTION_INFO" />
         <dropTable tableName="EXECUTION_EVENT" />
diff --git a/database/migration/src/main/resources/changesets/rename_iteration_to_index.xml b/database/migration/src/main/resources/changesets/rename_iteration_to_index.xml
index a85483f7908..afd71b2ae83 100644
--- a/database/migration/src/main/resources/changesets/rename_iteration_to_index.xml
+++ b/database/migration/src/main/resources/changesets/rename_iteration_to_index.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="rename-iteration-to-index">
+    <changeSet author="tjeandet" id="rename-iteration-to-index" dbms="mysql,hsqldb">
         <renameColumn columnDataType="INT"
                       newColumnName="INDEX"
                       oldColumnName="ITERATION"
diff --git a/database/migration/src/main/resources/changesets/rename_jes_id.xml b/database/migration/src/main/resources/changesets/rename_jes_id.xml
index cfaf6ce8e54..30b70d8b575 100644
--- a/database/migration/src/main/resources/changesets/rename_jes_id.xml
+++ b/database/migration/src/main/resources/changesets/rename_jes_id.xml
@@ -3,11 +3,11 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="sfrazer" id="rename-jes-id-to-jes-run-id">
+    <changeSet author="sfrazer" id="rename-jes-id-to-jes-run-id" dbms="mysql,hsqldb">
         <renameColumn columnDataType="VARCHAR(100)"
                       newColumnName="JES_RUN_ID"
                       oldColumnName="JES_ID"
                       tableName="JES_JOB"/>
     </changeSet>
 
-</databaseChangeLog>
\ No newline at end of file
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml b/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml
index b189e535a8c..7f1cff48ca6 100644
--- a/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml
+++ b/database/migration/src/main/resources/changesets/rename_workflow_options_in_metadata.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="rename_workflow_options_in_metadata">
+    <changeSet author="tjeandet" id="rename_workflow_options_in_metadata" dbms="mysql,hsqldb">
         <customChange class="cromwell.database.migration.workflowoptions.RenameWorkflowOptionsInMetadata"/>
     </changeSet>
 
diff --git a/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml b/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml
index 508497a8912..da7b346d3e0 100644
--- a/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml
+++ b/database/migration/src/main/resources/changesets/replace_empty_custom_labels.xml
@@ -7,7 +7,7 @@
         <sqlFile relativeToChangelogFile="true" path="migration/workflow_store/replace_empty_custom_labels.sql" />
     </changeSet>
 
-    <changeSet id="custom_labels_not_null" author="rmunshi">
+    <changeSet id="custom_labels_not_null" author="rmunshi" dbms="mysql,hsqldb">
         <addNotNullConstraint tableName="WORKFLOW_STORE_ENTRY"
                               columnName="CUSTOM_LABELS"
                               columnDataType="LONGTEXT"
diff --git a/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml b/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
index 6cab406445b..7a2ce27db38 100644
--- a/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
+++ b/database/migration/src/main/resources/changesets/restart_and_recover_migration.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet id="WORKFLOW_STORE_MIGRATION" author="mcovarr" >
+    <changeSet id="WORKFLOW_STORE_MIGRATION" author="mcovarr"  dbms="mysql,hsqldb">
         <comment>
             Restart/recover migration from 0.19 to 0.21.
         </comment>
@@ -31,7 +31,7 @@
         </sql>
     </changeSet>
 
-    <changeSet id="JOB_STORE_MIGRATION" author="mcovarr" >
+    <changeSet id="JOB_STORE_MIGRATION" author="mcovarr"  dbms="mysql,hsqldb">
         <comment>
             Restart/recover migration from 0.19 to 0.21.
         </comment>
@@ -76,7 +76,7 @@
         </sql>
     </changeSet>
 
-    <changeSet id="JOB_STORE_RESULT_SIMPLETON_VALUE_NULLABLE" author="mcovarr">
+    <changeSet id="JOB_STORE_RESULT_SIMPLETON_VALUE_NULLABLE" author="mcovarr" dbms="mysql,hsqldb">
         <preConditions onFail="MARK_RAN">
             <tableExists tableName="JOB_STORE_RESULT_SIMPLETON"/>
         </preConditions>
@@ -93,14 +93,14 @@
                                columnDataType="LONGTEXT"/>
     </changeSet>
 
-    <changeSet id="JOB_STORE_RESULT_SIMPLETON_MIGRATION" author="mcovarr" >
+    <changeSet id="JOB_STORE_RESULT_SIMPLETON_MIGRATION" author="mcovarr"  dbms="mysql,hsqldb">
         <comment>
             Restart/recover migration from 0.19 to 0.21.
         </comment>
         <customChange class="cromwell.database.migration.restart.table.JobStoreSimpletonMigration" />
     </changeSet>
 
-    <changeSet id="BACKEND_KV_STORE_MIGRATION" author="mcovarr">
+    <changeSet id="BACKEND_KV_STORE_MIGRATION" author="mcovarr" dbms="mysql,hsqldb">
         <comment>
             Restart/recover migration from 0.19 to 0.21.
         </comment>
@@ -130,7 +130,7 @@
         </sql>
     </changeSet>
 
-    <changeSet id="WORKFLOW_OPTIONS_RENAME_MIGRATION" author="mcovarr">
+    <changeSet id="WORKFLOW_OPTIONS_RENAME_MIGRATION" author="mcovarr" dbms="mysql,hsqldb">
         <comment>
             Restart/recover migration from 0.19 to 0.21.
         </comment>
diff --git a/database/migration/src/main/resources/changesets/runtime_attributes_table.xml b/database/migration/src/main/resources/changesets/runtime_attributes_table.xml
index 4b4cfd368cc..a5ea9bb4611 100644
--- a/database/migration/src/main/resources/changesets/runtime_attributes_table.xml
+++ b/database/migration/src/main/resources/changesets/runtime_attributes_table.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="runtime-attributes-table">
+    <changeSet author="tjeandet" id="runtime-attributes-table" dbms="mysql,hsqldb">
         <createTable tableName="RUNTIME_ATTRIBUTES">
             <column autoIncrement="true" name="RUNTIME_ATTRIBUTE_ID" type="INT">
                 <constraints primaryKey="true" primaryKeyName="PK_RUNTIME_ATTRIBUTE"/>
diff --git a/database/migration/src/main/resources/changesets/sge.xml b/database/migration/src/main/resources/changesets/sge.xml
index a3bca146662..aab8faa4b92 100644
--- a/database/migration/src/main/resources/changesets/sge.xml
+++ b/database/migration/src/main/resources/changesets/sge.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="scottfrazer" id="sge">
+    <changeSet author="scottfrazer" id="sge" dbms="mysql,hsqldb">
 
         <createTable tableName="SGE_JOB">
             <column autoIncrement="true" name="SGE_JOB_ID" type="INT">
@@ -21,7 +21,7 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="scottfrazer" id="db_schema_constraints">
+    <changeSet author="scottfrazer" id="db_schema_constraints" dbms="mysql,hsqldb">
         <addForeignKeyConstraint constraintName="FK_SGE_JOB_EXECUTION_ID"
                                  baseTableName="SGE_JOB" baseColumnNames="EXECUTION_ID"
                                  referencedTableName="EXECUTION" referencedColumnNames="EXECUTION_ID"/>
diff --git a/database/migration/src/main/resources/changesets/sge_job_execution_unique_key.xml b/database/migration/src/main/resources/changesets/sge_job_execution_unique_key.xml
index 71aa97435db..7a362016bc2 100644
--- a/database/migration/src/main/resources/changesets/sge_job_execution_unique_key.xml
+++ b/database/migration/src/main/resources/changesets/sge_job_execution_unique_key.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="sge_job_execution_unique_key">
+    <changeSet author="kshakir" id="sge_job_execution_unique_key" dbms="mysql,hsqldb">
         <comment>
             Adds unique constraints UK_SGE_JOB_EXECUTION_UUID.
         </comment>
diff --git a/database/migration/src/main/resources/changesets/standardize_column_names.xml b/database/migration/src/main/resources/changesets/standardize_column_names.xml
index a309c8ea28b..50d26c429b4 100644
--- a/database/migration/src/main/resources/changesets/standardize_column_names.xml
+++ b/database/migration/src/main/resources/changesets/standardize_column_names.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="rmunshi" id="workflow_execution_uuid">
+    <changeSet author="rmunshi" id="workflow_execution_uuid" dbms="mysql,hsqldb">
         <comment>
             Change all Workflow UUID column names to Workflow Execution UUID.
         </comment>
@@ -21,7 +21,7 @@
                       tableName="JOB_STORE"/>
     </changeSet>
 
-    <changeSet author="rmunshi" id="call_job_identifiers">
+    <changeSet author="rmunshi" id="call_job_identifiers" dbms="mysql,hsqldb">
         <comment>
             Choose and implement common call/job identifiers.
         </comment>
@@ -39,4 +39,4 @@
                       tableName="METADATA_JOURNAL"/>
     </changeSet>
 
-</databaseChangeLog>
\ No newline at end of file
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/standardize_column_names_again.xml b/database/migration/src/main/resources/changesets/standardize_column_names_again.xml
index 111bcd74fed..a98e35f3448 100644
--- a/database/migration/src/main/resources/changesets/standardize_column_names_again.xml
+++ b/database/migration/src/main/resources/changesets/standardize_column_names_again.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="standardize_column_names_again">
+    <changeSet author="kshakir" id="standardize_column_names_again" dbms="mysql,hsqldb">
 
         <!--
         NOTE: Column related changes to METADATA_JOURNAL (aka METADATA_ENTRY) are not included at this time.
diff --git a/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml b/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml
index 85eef3be762..e24f0bee1c0 100644
--- a/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml
+++ b/database/migration/src/main/resources/changesets/standardize_column_names_patches.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="standardize_column_names_patches">
+    <changeSet author="kshakir" id="standardize_column_names_patches" dbms="mysql,hsqldb">
         <comment>
             Columns now all match standardized naming.
             FK_CALL_CACHING_HASH_ENTRY_CALL_CACHING_ENTRY_ID must be dropped and re-added so that
diff --git a/database/migration/src/main/resources/changesets/sub_workflow_store.xml b/database/migration/src/main/resources/changesets/sub_workflow_store.xml
index ebde540f1d9..f96f0c12ed0 100644
--- a/database/migration/src/main/resources/changesets/sub_workflow_store.xml
+++ b/database/migration/src/main/resources/changesets/sub_workflow_store.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="tjeandet" id="SUB_WORKFLOW_STORE_ENTRY">
+    <changeSet author="tjeandet" id="SUB_WORKFLOW_STORE_ENTRY" dbms="mysql,hsqldb">
         <comment>
             Temporary storage area for sub workflows which belong to workflows that are still in progress.
         </comment>
@@ -40,20 +40,20 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="tjeandet" id="sub_workflow_store_uuid_index">
+    <changeSet author="tjeandet" id="sub_workflow_store_uuid_index" dbms="mysql,hsqldb">
         <createIndex indexName="IX_SUB_WORKFLOW_STORE_ENTRY_PWEU"
                      tableName="SUB_WORKFLOW_STORE_ENTRY">
             <column name="PARENT_WORKFLOW_EXECUTION_UUID"/>
         </createIndex>
     </changeSet>
 
-    <changeSet author="tjeandet" id="sub_workflow_store_jobkey_index">
+    <changeSet author="tjeandet" id="sub_workflow_store_jobkey_index" dbms="mysql,hsqldb">
         <addUniqueConstraint constraintName="UC_SUB_WORKFLOW_STORE_ENTRY_PWEU_CFQN_CI_CA"
                      tableName="SUB_WORKFLOW_STORE_ENTRY" columnNames="PARENT_WORKFLOW_EXECUTION_UUID, CALL_FULLY_QUALIFIED_NAME, CALL_INDEX, CALL_ATTEMPT" />
 
     </changeSet>
 
-    <changeSet author="tjeandet" id="sub_workflow_store_root_workflow_fk">
+    <changeSet author="tjeandet" id="sub_workflow_store_root_workflow_fk" dbms="mysql,hsqldb">
     <addForeignKeyConstraint constraintName="FK_SUB_WORKFLOW_STORE_ROOT_WORKFLOW_ID_WORKFLOW_STORE_ENTRY_ID" onDelete="CASCADE"
                              referencedTableName="WORKFLOW_STORE_ENTRY" referencedColumnNames="WORKFLOW_STORE_ENTRY_ID"
                              baseTableName="SUB_WORKFLOW_STORE_ENTRY" baseColumnNames="ROOT_WORKFLOW_ID"/>
diff --git a/database/migration/src/main/resources/changesets/summary_status_table.xml b/database/migration/src/main/resources/changesets/summary_status_table.xml
index 27e72402e60..cb534f0e5ec 100644
--- a/database/migration/src/main/resources/changesets/summary_status_table.xml
+++ b/database/migration/src/main/resources/changesets/summary_status_table.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="kshakir" id="summary_status_table">
+    <changeSet author="kshakir" id="summary_status_table" dbms="mysql,hsqldb">
         <createTable tableName="SUMMARY_STATUS">
             <column name="SUMMARY_STATUS_ID" type="INT" autoIncrement="true">
                 <constraints primaryKey="true" primaryKeyName="PK_SUMMARY_STATUS"/>
@@ -24,7 +24,7 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="kshakir" id="summary_status_summarized_table_name_index">
+    <changeSet author="kshakir" id="summary_status_summarized_table_name_index" dbms="mysql,hsqldb">
         <addUniqueConstraint tableName="SUMMARY_STATUS"
                              columnNames="SUMMARY_TABLE_NAME, SUMMARIZED_TABLE_NAME"
                              constraintName="SUMMARY_STATUS_SUMMARY_TABLE_NAME_SUMMARIZED_TABLE_NAME_INDEX"/>
diff --git a/database/migration/src/main/resources/changesets/symbol_iteration_not_null.xml b/database/migration/src/main/resources/changesets/symbol_iteration_not_null.xml
index be6d97aedcd..8388008b0ad 100644
--- a/database/migration/src/main/resources/changesets/symbol_iteration_not_null.xml
+++ b/database/migration/src/main/resources/changesets/symbol_iteration_not_null.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="kshakir" id="symbol_iteration_not_null">
+    <changeSet author="kshakir" id="symbol_iteration_not_null" dbms="mysql,hsqldb">
         <comment>
             Thanks to "Not a bug" https://bugs.mysql.com/bug.php?id=8173,
             Need to store flag values in the db to make the unique constraint work
diff --git a/database/migration/src/main/resources/changesets/symbol_iteration_null.xml b/database/migration/src/main/resources/changesets/symbol_iteration_null.xml
index bd4800c0504..3aec131b00e 100644
--- a/database/migration/src/main/resources/changesets/symbol_iteration_null.xml
+++ b/database/migration/src/main/resources/changesets/symbol_iteration_null.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="kshakir" id="symbol_iteration_null">
+    <changeSet author="kshakir" id="symbol_iteration_null" dbms="mysql,hsqldb">
         <dropNotNullConstraint tableName="SYMBOL" columnName="ITERATION" columnDataType="int"/>
     </changeSet>
 </databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/sync_not_null_constraints.xml b/database/migration/src/main/resources/changesets/sync_not_null_constraints.xml
index b2421fd0408..fbd84aaf905 100644
--- a/database/migration/src/main/resources/changesets/sync_not_null_constraints.xml
+++ b/database/migration/src/main/resources/changesets/sync_not_null_constraints.xml
@@ -2,7 +2,7 @@
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
-    <changeSet author="kshakir" id="job_not_null_columns">
+    <changeSet author="kshakir" id="job_not_null_columns" dbms="mysql,hsqldb">
         <comment>
             Adds missing not null constraints to various tables.
         </comment>
@@ -17,7 +17,7 @@
                 columnDataType="INT"/>
     </changeSet>
 
-    <changeSet author="kshakir" id="events_end_dt_not_null" dbms="!mysql">
+    <changeSet author="kshakir" id="events_end_dt_not_null" dbms="hsqldb">
         <comment>
             Changes the execution event end date to not null.
         </comment>
diff --git a/database/migration/src/main/resources/changesets/top_level_output.xml b/database/migration/src/main/resources/changesets/top_level_output.xml
index eaf3ef246cc..a8ced8d267d 100644
--- a/database/migration/src/main/resources/changesets/top_level_output.xml
+++ b/database/migration/src/main/resources/changesets/top_level_output.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="chrisl" id="top_level_output">
+    <changeSet author="chrisl" id="top_level_output" dbms="mysql,hsqldb">
         <comment>Adds a REPORTABLE_RESULT field to SYMBOL so that the DB can track which symbols represent top-level results.</comment>
         <addColumn tableName="SYMBOL">
             <column name="REPORTABLE_RESULT" type="TINYINT" >
@@ -12,4 +12,4 @@
         </addColumn>
     </changeSet>
 
-</databaseChangeLog>
\ No newline at end of file
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/changesets/wdl_and_inputs.xml b/database/migration/src/main/resources/changesets/wdl_and_inputs.xml
index 76e590dd191..001b7794622 100644
--- a/database/migration/src/main/resources/changesets/wdl_and_inputs.xml
+++ b/database/migration/src/main/resources/changesets/wdl_and_inputs.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="mcovarr" id="WORKFLOW_EXECUTION_AUX">
+    <changeSet author="mcovarr" id="WORKFLOW_EXECUTION_AUX" dbms="mysql,hsqldb">
         <comment>
             Auxiliary data related to a workflow execution not stored with the workflow execution itself
             for size reasons.
@@ -24,7 +24,7 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="mcovarr" id="WE_AUX_CONSTRAINTS">
+    <changeSet author="mcovarr" id="WE_AUX_CONSTRAINTS" dbms="mysql,hsqldb">
         <comment>
             This adds:
             A foreign key constraint from WORKFLOW_EXECUTION_ID to WORKFLOW_EXECUTION.WORKFLOW_EXECUTION_ID.
diff --git a/database/migration/src/main/resources/changesets/workflow_execution_aux_not_null.xml b/database/migration/src/main/resources/changesets/workflow_execution_aux_not_null.xml
index 91bfeffbb97..e6bf9c85535 100644
--- a/database/migration/src/main/resources/changesets/workflow_execution_aux_not_null.xml
+++ b/database/migration/src/main/resources/changesets/workflow_execution_aux_not_null.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="tjeandet" id="workflow_execution_aux_not_null">
+    <changeSet author="tjeandet" id="workflow_execution_aux_not_null" dbms="mysql,hsqldb">
         <addNotNullConstraint
                 tableName="WORKFLOW_EXECUTION_AUX" columnName="WORKFLOW_EXECUTION_ID"
                 columnDataType="INT"/>
diff --git a/database/migration/src/main/resources/changesets/workflow_metadata_summary.xml b/database/migration/src/main/resources/changesets/workflow_metadata_summary.xml
index 4c7e61d3686..07d25e96c55 100644
--- a/database/migration/src/main/resources/changesets/workflow_metadata_summary.xml
+++ b/database/migration/src/main/resources/changesets/workflow_metadata_summary.xml
@@ -3,12 +3,12 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet id="metadata_journal_id_int_to_big_int" author="kshakir">
+    <changeSet id="metadata_journal_id_int_to_big_int" author="kshakir" dbms="mysql,hsqldb">
         <modifyDataType tableName="METADATA_JOURNAL" columnName="METADATA_JOURNAL_ID" newDataType="BIGINT" />
         <addAutoIncrement tableName="METADATA_JOURNAL" columnName="METADATA_JOURNAL_ID" columnDataType="BIGINT" />
     </changeSet>
 
-    <changeSet id="workflow_metadata_summary" author="mcovarr">
+    <changeSet id="workflow_metadata_summary" author="mcovarr" dbms="mysql,hsqldb">
         <createTable tableName="WORKFLOW_METADATA_SUMMARY">
             <column autoIncrement="true" name="WORKFLOW_METADATA_SUMMARY_ID" type="BIGINT">
                 <constraints primaryKey="true" primaryKeyName="PK_WORKFLOW_METADATA_SUMMARY"/>
@@ -23,20 +23,20 @@
         </createTable>
     </changeSet>
 
-    <changeSet id="workflow_metadata_uuid_idx" author="mcovarr">
+    <changeSet id="workflow_metadata_uuid_idx" author="mcovarr" dbms="mysql,hsqldb">
         <addUniqueConstraint tableName="WORKFLOW_METADATA_SUMMARY"
                              constraintName="WORKFLOW_METADATA_UUID_IDX"
                              columnNames="WORKFLOW_EXECUTION_UUID" />
     </changeSet>
 
-    <changeSet id="workflow_metadata_name_idx" author="mcovarr">
+    <changeSet id="workflow_metadata_name_idx" author="mcovarr" dbms="mysql,hsqldb">
         <createIndex indexName="WORKFLOW_METADATA_NAME_IDX"
                      tableName="WORKFLOW_METADATA_SUMMARY" unique="false">
             <column name="WORKFLOW_NAME"/>
         </createIndex>
     </changeSet>
 
-    <changeSet id="workflow_metadata_status_idx" author="mcovarr">
+    <changeSet id="workflow_metadata_status_idx" author="mcovarr" dbms="mysql,hsqldb">
         <createIndex indexName="WORKFLOW_METADATA_STATUS_IDX"
                      tableName="WORKFLOW_METADATA_SUMMARY" unique="false">
             <column name="WORKFLOW_STATUS"/>
diff --git a/database/migration/src/main/resources/changesets/workflow_options.xml b/database/migration/src/main/resources/changesets/workflow_options.xml
index 7d3852eb090..e8b208781ef 100644
--- a/database/migration/src/main/resources/changesets/workflow_options.xml
+++ b/database/migration/src/main/resources/changesets/workflow_options.xml
@@ -4,7 +4,7 @@
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
-    <changeSet author="sfrazer" id="workflow-options">
+    <changeSet author="sfrazer" id="workflow-options" dbms="mysql,hsqldb">
         <addColumn tableName="WORKFLOW_EXECUTION_AUX">
             <column name="WORKFLOW_OPTIONS" type="${clob.type}" />
         </addColumn>
diff --git a/database/migration/src/main/resources/changesets/workflow_store.xml b/database/migration/src/main/resources/changesets/workflow_store.xml
index d80840ff4fa..42c1ed4fca6 100644
--- a/database/migration/src/main/resources/changesets/workflow_store.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="chrisl" id="WORKFLOW_STORE">
+    <changeSet author="chrisl" id="WORKFLOW_STORE" dbms="mysql,hsqldb">
         <comment>
             Temporary storage area for workflows that are still in progress.
         </comment>
@@ -38,14 +38,14 @@
         </modifySql>
     </changeSet>
 
-    <changeSet author="kshakir" id="workflow_store_uuid_index">
+    <changeSet author="kshakir" id="workflow_store_uuid_index" dbms="mysql,hsqldb">
         <addUniqueConstraint
                 constraintName="WORKFLOW_STORE_UUID_IDX"
                 tableName="WORKFLOW_STORE"
                 columnNames="WORKFLOW_UUID" />
     </changeSet>
 
-    <changeSet author="chrisl" id="workflow_store_state_index">
+    <changeSet author="chrisl" id="workflow_store_state_index" dbms="mysql,hsqldb">
         <createIndex indexName="WORKFLOW_STORE_STATE_IDX"
                      tableName="WORKFLOW_STORE" unique="false">
             <column name="WORKFLOW_STATE"/>
diff --git a/database/migration/src/main/resources/changesets/workflow_store_horizontal_db.xml b/database/migration/src/main/resources/changesets/workflow_store_horizontal_db.xml
index e920b24b596..5093bce3382 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_horizontal_db.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_horizontal_db.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="workflow-store-horizontal-db">
+    <changeSet author="mcovarr" id="workflow-store-horizontal-db" dbms="mysql,hsqldb">
         <dropColumn tableName="WORKFLOW_STORE_ENTRY" columnName="RESTARTED" />
         <addColumn tableName="WORKFLOW_STORE_ENTRY">
             <column name="CROMWELL_ID" type="VARCHAR(100)">
diff --git a/database/migration/src/main/resources/changesets/workflow_store_imports_file.xml b/database/migration/src/main/resources/changesets/workflow_store_imports_file.xml
index bd6d3f05106..554879d9ab1 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_imports_file.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_imports_file.xml
@@ -6,7 +6,7 @@
     <property name="blob.type" value="LONGBLOB" dbms="mysql"/>
     <property name="blob.type" value="BLOB" dbms="hsqldb"/>
 
-    <changeSet author="cjllanwarne" id="workflow-store-imports-file">
+    <changeSet author="cjllanwarne" id="workflow-store-imports-file" dbms="mysql,hsqldb">
         <addColumn
                 tableName="WORKFLOW_STORE_ENTRY">
             <column name="IMPORTS_ZIP" type="${blob.type}" />
diff --git a/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml b/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
index 225c13b78cb..e2891a406a9 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_labels_file.xml
@@ -6,7 +6,7 @@
     <property name="clob.type" value="LONGTEXT" dbms="mysql"/>
     <property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>
 
-    <changeSet author="cjllanwarne" id="workflow-store-labels-file">
+    <changeSet author="cjllanwarne" id="workflow-store-labels-file" dbms="mysql,hsqldb">
         <addColumn
                 tableName="WORKFLOW_STORE_ENTRY">
             <column name="CUSTOM_LABELS" type="${clob.type}">
diff --git a/database/migration/src/main/resources/changesets/workflow_store_restarted_column.xml b/database/migration/src/main/resources/changesets/workflow_store_restarted_column.xml
index 97e2035b003..7e066f62068 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_restarted_column.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_restarted_column.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="workflow-store-restarted-column">
+    <changeSet author="tjeandet" id="workflow-store-restarted-column" dbms="mysql,hsqldb">
         <addColumn
                 tableName="WORKFLOW_STORE_ENTRY">
             <column name="RESTARTED" type="BOOLEAN" valueBoolean = "false">
@@ -11,7 +11,7 @@
             </column>
         </addColumn>
     </changeSet>
-    <changeSet author="tjeandet" id="update-restartable">
+    <changeSet author="tjeandet" id="update-restartable" dbms="mysql,hsqldb">
         <!--RestartableRunning to Running-->
         <update
                 tableName="WORKFLOW_STORE_ENTRY">
diff --git a/database/migration/src/main/resources/changesets/workflow_store_state_widening.xml b/database/migration/src/main/resources/changesets/workflow_store_state_widening.xml
index 157fd43bd3b..5d3b37bce87 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_state_widening.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_state_widening.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="workflow-store-state-widening">
+    <changeSet author="tjeandet" id="workflow-store-state-widening" dbms="mysql,hsqldb">
         <modifyDataType columnName="WORKFLOW_STATE"
                         newDataType="varchar(20)"
                         tableName="WORKFLOW_STORE_ENTRY"/>
diff --git a/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml b/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml
index bdb38ad09ed..59876e1dffa 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_type_and_version.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="mcovarr" id="workflow-store-type-and-version">
+    <changeSet author="mcovarr" id="workflow-store-type-and-version" dbms="mysql,hsqldb">
         <addColumn tableName="WORKFLOW_STORE_ENTRY">
             <column name="WORKFLOW_TYPE" type="VARCHAR(30)">
                 <constraints nullable="true"/>
diff --git a/database/migration/src/main/resources/changesets/workflow_store_workflow_root_column.xml b/database/migration/src/main/resources/changesets/workflow_store_workflow_root_column.xml
index dab1df1b71e..7358ffd4347 100644
--- a/database/migration/src/main/resources/changesets/workflow_store_workflow_root_column.xml
+++ b/database/migration/src/main/resources/changesets/workflow_store_workflow_root_column.xml
@@ -3,7 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
 
-    <changeSet author="tjeandet" id="workflow-store-workflow-root-column">
+    <changeSet author="tjeandet" id="workflow-store-workflow-root-column" dbms="mysql,hsqldb">
         <addColumn
                 tableName="WORKFLOW_STORE_ENTRY">
             <column name="WORKFLOW_ROOT" type="VARCHAR(100)" />
diff --git a/database/migration/src/main/resources/metadata_changesets/postgresql_metadata_schema.xml b/database/migration/src/main/resources/metadata_changesets/postgresql_metadata_schema.xml
new file mode 100644
index 00000000000..d1c5835954a
--- /dev/null
+++ b/database/migration/src/main/resources/metadata_changesets/postgresql_metadata_schema.xml
@@ -0,0 +1,130 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog objectQuotingStrategy="QUOTE_ALL_OBJECTS"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet author="nechols" id="postgresql-metadata-init-1" dbms="postgresql">
+        <createTable tableName="CUSTOM_LABEL_ENTRY">
+            <column autoIncrement="true" name="CUSTOM_LABEL_ENTRY_ID" type="BIGSERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_CUSTOM_LABEL_ENTRY"/>
+            </column>
+            <column name="CUSTOM_LABEL_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CUSTOM_LABEL_VALUE" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(100)">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-2" dbms="postgresql">
+        <createTable tableName="METADATA_ENTRY">
+            <column autoIncrement="true" name="METADATA_JOURNAL_ID" type="BIGSERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_METADATA_JOURNAL"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="METADATA_KEY" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="CALL_FQN" type="VARCHAR(255)"/>
+            <column name="JOB_SCATTER_INDEX" type="INT"/>
+            <column name="JOB_RETRY_ATTEMPT" type="INT"/>
+            <column name="METADATA_VALUE" type="CLOB"/>
+            <column name="METADATA_TIMESTAMP" type="TIMESTAMP">
+                <constraints nullable="false"/>
+            </column>
+            <column name="METADATA_VALUE_TYPE" type="VARCHAR(10)"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-3" dbms="postgresql">
+        <createTable tableName="SUMMARY_STATUS_ENTRY">
+            <column autoIncrement="true" name="SUMMARY_STATUS_ENTRY_ID" type="SERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_SUMMARY_STATUS_ENTRY"/>
+            </column>
+            <column name="SUMMARY_NAME" type="VARCHAR(255)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="SUMMARY_POSITION" type="BIGINT">
+                <constraints nullable="false"/>
+            </column>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-4" dbms="postgresql">
+        <createTable tableName="WORKFLOW_METADATA_SUMMARY_ENTRY">
+            <column autoIncrement="true" name="WORKFLOW_METADATA_SUMMARY_ENTRY_ID" type="BIGSERIAL">
+                <constraints primaryKey="true" primaryKeyName="PK_WORKFLOW_METADATA_SUMMARY_ENTRY"/>
+            </column>
+            <column name="WORKFLOW_EXECUTION_UUID" type="VARCHAR(100)">
+                <constraints nullable="false"/>
+            </column>
+            <column name="WORKFLOW_NAME" type="VARCHAR(100)"/>
+            <column name="WORKFLOW_STATUS" type="VARCHAR(50)"/>
+            <column name="START_TIMESTAMP" type="TIMESTAMP"/>
+            <column name="END_TIMESTAMP" type="TIMESTAMP"/>
+            <column name="SUBMISSION_TIMESTAMP" type="TIMESTAMP"/>
+            <column name="PARENT_WORKFLOW_EXECUTION_UUID" type="VARCHAR(100)"/>
+            <column name="ROOT_WORKFLOW_EXECUTION_UUID" type="VARCHAR(100)"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-5" dbms="postgresql">
+        <addUniqueConstraint columnNames="CUSTOM_LABEL_KEY, WORKFLOW_EXECUTION_UUID" constraintName="UC_CUSTOM_LABEL_ENTRY_CLK_WEU" tableName="CUSTOM_LABEL_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-6" dbms="postgresql">
+        <addUniqueConstraint columnNames="SUMMARY_NAME" constraintName="UC_SUMMARY_STATUS_ENTRY_SN" tableName="SUMMARY_STATUS_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-7" dbms="postgresql">
+        <addUniqueConstraint columnNames="WORKFLOW_EXECUTION_UUID" constraintName="UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU" tableName="WORKFLOW_METADATA_SUMMARY_ENTRY"/>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-8" dbms="postgresql">
+        <createIndex indexName="IX_CUSTOM_LABEL_ENTRY_CLK_CLV" tableName="CUSTOM_LABEL_ENTRY">
+            <column name="CUSTOM_LABEL_KEY"/>
+            <column name="CUSTOM_LABEL_VALUE"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-9" dbms="postgresql">
+        <createIndex indexName="IX_WORKFLOW_METADATA_SUMMARY_ENTRY_PWEU" tableName="WORKFLOW_METADATA_SUMMARY_ENTRY">
+            <column name="PARENT_WORKFLOW_EXECUTION_UUID"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-10" dbms="postgresql">
+        <createIndex indexName="IX_WORKFLOW_METADATA_SUMMARY_ENTRY_RWEU" tableName="WORKFLOW_METADATA_SUMMARY_ENTRY">
+            <column name="ROOT_WORKFLOW_EXECUTION_UUID"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-11" dbms="postgresql">
+        <createIndex indexName="IX_WORKFLOW_METADATA_SUMMARY_ENTRY_WN" tableName="WORKFLOW_METADATA_SUMMARY_ENTRY">
+            <column name="WORKFLOW_NAME"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-12" dbms="postgresql">
+        <createIndex indexName="IX_WORKFLOW_METADATA_SUMMARY_ENTRY_WS" tableName="WORKFLOW_METADATA_SUMMARY_ENTRY">
+            <column name="WORKFLOW_STATUS"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-13" dbms="postgresql">
+        <createIndex indexName="METADATA_WORKFLOW_IDX" tableName="METADATA_ENTRY">
+            <column name="WORKFLOW_EXECUTION_UUID"/>
+        </createIndex>
+    </changeSet>
+
+    <changeSet author="nechols" id="postgresql-metadata-init-14" dbms="postgresql">
+        <addForeignKeyConstraint baseColumnNames="WORKFLOW_EXECUTION_UUID" baseTableName="CUSTOM_LABEL_ENTRY" constraintName="FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID" onDelete="CASCADE" referencedColumnNames="WORKFLOW_EXECUTION_UUID" referencedTableName="WORKFLOW_METADATA_SUMMARY_ENTRY"/>
+    </changeSet>
+</databaseChangeLog>
diff --git a/database/migration/src/main/resources/sql_metadata_changelog.xml b/database/migration/src/main/resources/sql_metadata_changelog.xml
index 05500b36bc5..809942372cc 100644
--- a/database/migration/src/main/resources/sql_metadata_changelog.xml
+++ b/database/migration/src/main/resources/sql_metadata_changelog.xml
@@ -11,5 +11,6 @@
     <include file="metadata_changesets/add_parent_and_root_workflow_execution_uuid.xml" relativeToChangelogFile="true"/>
     <include file="metadata_changesets/add_parent_and_root_workflow_typo_fix.xml" relativeToChangelogFile="true"/>
     <include file="metadata_changesets/metadata_index_removals.xml" relativeToChangelogFile="true" />
+    <include file="metadata_changesets/postgresql_metadata_schema.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!-- See Dos and Don'ts in changelog.xml -->
diff --git a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
index 622e62aaf33..d28801e7fb5 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/MetadataSlickDatabase.scala
@@ -34,7 +34,7 @@ class MetadataSlickDatabase(originalDatabaseConfig: Config)
   override def addMetadataEntries(metadataEntries: Iterable[MetadataEntry])
                                  (implicit ec: ExecutionContext): Future[Unit] = {
     val action = DBIO.seq(metadataEntries.grouped(insertBatchSize).map(dataAccess.metadataEntries ++= _).toSeq:_*)
-    runAction(action)
+    runLobAction(action)
   }
 
   override def metadataEntryExists(workflowExecutionUuid: String)(implicit ec: ExecutionContext): Future[Boolean] = {
diff --git a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
index 725ee999451..b1d68f5c590 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/SlickDatabase.scala
@@ -8,6 +8,7 @@ import com.typesafe.config.{Config, ConfigFactory}
 import cromwell.database.slick.tables.DataAccessComponent
 import cromwell.database.sql.SqlDatabase
 import net.ceedubs.ficus.Ficus._
+import org.postgresql.util.{PSQLException, ServerErrorMessage}
 import org.slf4j.LoggerFactory
 import slick.basic.DatabaseConfig
 import slick.jdbc.{JdbcCapabilities, JdbcProfile, TransactionIsolation}
@@ -58,6 +59,7 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend
 
   override val urlKey = SlickDatabase.urlKey(originalDatabaseConfig)
   protected val slickConfig = DatabaseConfig.forConfig[JdbcProfile]("", databaseConfig)
+  lazy val isPostgresql = databaseConfig.getOrElse("db.driver", "unknown") == "org.postgresql.Driver"
 
   /*
   Not a def because we need to have a "stable identifier" for the imports below.
@@ -167,10 +169,22 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend
     runActionInternal(action.transactionally.withTransactionIsolation(isolationLevel))
   }
 
+  /* Note that this is only appropriate for actions that do not involve Blob
+   * or Clob fields in Postgres, since large object support requires running
+   * transactionally.  Use runLobAction instead, which will still run in
+   * auto-commit mode when using other database engines.
+   */
   protected[this] def runAction[R](action: DBIO[R]): Future[R] = {
     runActionInternal(action.withPinnedSession)
   }
 
+  /* Wrapper for queries where Clob/Blob types are used
+   * https://stackoverflow.com/questions/3164072/large-objects-may-not-be-used-in-auto-commit-mode#answer-3164352
+   */
+  protected[this] def runLobAction[R](action: DBIO[R]): Future[R] = {
+    if (isPostgresql) runTransaction(action) else runAction(action)
+  }
+
   private def runActionInternal[R](action: DBIO[R]): Future[R] = {
     //database.run(action) <-- See comment above private val actionThreadPool
     Future {
@@ -186,6 +200,33 @@ abstract class SlickDatabase(override val originalDatabaseConfig: Config) extend
             case _ => /* keep going */
           }
           throw rollbackException
+        case pSQLException: PSQLException =>
+          val detailOption = for {
+            message <- Option(pSQLException.getServerErrorMessage)
+            detail <- Option(message.getDetail)
+          } yield detail
+
+          detailOption match {
+            case None => throw pSQLException
+            case Some(_) =>
+              /*
+              The exception may contain possibly sensitive row contents within the DETAIL section. Remove it.
+
+              Tried adjusting this using configuration:
+              - log_error_verbosity=TERSE
+              - log_min_messages=PANIC
+              - client_min_messages=ERROR
+
+              Instead resorting to reflection.
+               */
+              val message = pSQLException.getServerErrorMessage
+              val field = classOf[ServerErrorMessage].getDeclaredField("m_mesgParts")
+              field.setAccessible(true)
+              val parts = field.get(message).asInstanceOf[java.util.Map[Character, String]]
+              parts.remove('D')
+              // The original exception has already stored the DETAIL into a string. So we must create a new Exception.
+              throw new PSQLException(message)
+          }
       }
     }(actionExecutionContext)
   }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
index c36b8047d11..db7c1f3d826 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingDetritusEntryComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 import cromwell.database.sql.tables.CallCachingDetritusEntry
 
@@ -16,7 +16,7 @@ trait CallCachingDetritusEntryComponent {
 
     def detritusKey = column[String]("DETRITUS_KEY", O.Length(255))
 
-    def detritusValue = column[Option[Clob]]("DETRITUS_VALUE")
+    def detritusValue = column[Option[SerialClob]]("DETRITUS_VALUE")
 
     def callCachingEntryId = column[Int]("CALL_CACHING_ENTRY_ID")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
index 7170ae9025c..38a095a9682 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CallCachingSimpletonEntryComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 import cromwell.database.sql.tables.CallCachingSimpletonEntry
 
@@ -16,7 +16,7 @@ trait CallCachingSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY", O.Length(255))
 
-    def simpletonValue = column[Option[Clob]]("SIMPLETON_VALUE")
+    def simpletonValue = column[Option[SerialClob]]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE", O.Length(255))
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala
index 8121b3705ae..153b6815810 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/CustomLabelEntryComponent.scala
@@ -1,12 +1,14 @@
 package cromwell.database.slick.tables
 
 import cromwell.database.sql.tables.CustomLabelEntry
+import shapeless.syntax.std.tuple._
 import slick.model.ForeignKeyAction.Cascade
 
 trait CustomLabelEntryComponent {
 
   this: DriverComponent with WorkflowMetadataSummaryEntryComponent =>
 
+  import driver.api.TupleMethods._
   import driver.api._
 
   class CustomLabelEntries(tag: Tag)
@@ -19,8 +21,14 @@ trait CustomLabelEntryComponent {
 
     def workflowExecutionUuid = column[String]("WORKFLOW_EXECUTION_UUID", O.Length(100))
 
-    override def * = (customLabelKey, customLabelValue, workflowExecutionUuid,
-    customLabelEntryId.?) <> (CustomLabelEntry.tupled, CustomLabelEntry.unapply)
+    def baseProjection = (customLabelKey, customLabelValue, workflowExecutionUuid)
+
+    override def * = baseProjection ~ customLabelEntryId.? <> (CustomLabelEntry.tupled, CustomLabelEntry.unapply)
+
+    def forUpdate = baseProjection.shaped <> (
+      tuple => CustomLabelEntry.tupled(tuple :+ None),
+      CustomLabelEntry.unapply(_: CustomLabelEntry).map(_.reverse.tail.reverse)
+    )
 
     def fkCustomLabelEntryWorkflowExecutionUuid = foreignKey("FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID",
       workflowExecutionUuid, workflowMetadataSummaryEntries)(_.workflowExecutionUuid, onDelete = Cascade)
@@ -41,7 +49,7 @@ trait CustomLabelEntryComponent {
       customLabelEntry <- customLabelEntries
       if customLabelEntry.workflowExecutionUuid === workflowExecutionUuid &&
         customLabelEntry.customLabelKey === labelKey
-    } yield customLabelEntry)
+    } yield customLabelEntry.forUpdate)
 
   def existsWorkflowIdLabelKeyAndValue(workflowId: Rep[String],
                                        labelKey: Rep[String],
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
index d5f78601862..f9343883f4d 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/DriverComponent.scala
@@ -1,7 +1,55 @@
 package cromwell.database.slick.tables
 
-import slick.jdbc.JdbcProfile
+import java.sql.{Blob, Clob}
+
+import javax.sql.rowset.serial.{SerialBlob, SerialClob}
+import org.apache.commons.io.IOUtils
+import slick.jdbc.{JdbcProfile, PostgresProfile}
 
 trait DriverComponent {
   val driver: JdbcProfile
+
+  import driver.api._
+
+  /** Ensure clobs are retrieved inside the transaction, not after */
+  implicit val serialClobColumnType = MappedColumnType.base[SerialClob, Clob](
+    identity,
+    {
+      case serialClob: SerialClob => serialClob
+      case clob =>
+        /*
+        PostgreSQL's JDBC driver has issues with non-ascii characters.
+        https://stackoverflow.com/questions/5043992/postgres-utf-8-clobs-with-jdbc
+
+        It returns bad values for length() and getAsciiStream(), and causes an extra null bytes to be added at the end
+        of the resultant SerialClob.
+
+        Example via copy_workflow_outputs/unscattered.wdl:
+
+          "... Enfin un peu de francais pour contrer ce raz-de-marée anglais ! ..."
+
+        The 'é' in results in an extra null byte at the end of getAsciiStream().
+         */
+        val string = IOUtils.toString(clob.getCharacterStream)
+        new SerialClob(string.toCharArray)
+    }
+  )
+
+  /** Ensure clobs are retrieved inside the transaction, not after */
+  implicit val serialBlobColumnType = MappedColumnType.base[SerialBlob, Blob](
+    identity,
+    {
+      case serialBlob: SerialBlob => serialBlob
+      case blob => new SerialBlob(blob)
+    }
+  )
+
+  private val shouldQuote = this.driver match {
+    // https://stackoverflow.com/questions/43111996/why-postgresql-does-not-like-uppercase-table-names#answer-43112096
+    case PostgresProfile => true
+    case _ => false
+  }
+
+  /** Adds quotes around the string if required by the DBMS. */
+  def quoted(string: String) = if (shouldQuote) s""""$string"""" else string
 }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
index 6422e535ef7..de5bfe57698 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreEntryComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 import cromwell.database.sql.tables.JobStoreEntry
 
@@ -26,7 +26,7 @@ trait JobStoreEntryComponent {
     def returnCode = column[Option[Int]]("RETURN_CODE")
 
     // Only set for failure:
-    def exceptionMessage = column[Option[Clob]]("EXCEPTION_MESSAGE")
+    def exceptionMessage = column[Option[SerialClob]]("EXCEPTION_MESSAGE")
 
     def retryableFailure = column[Option[Boolean]]("RETRYABLE_FAILURE")
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
index 40d3f094ea3..e2e9c83dac9 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/JobStoreSimpletonEntryComponent.scala
@@ -1,6 +1,6 @@
 package cromwell.database.slick.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 import cromwell.database.sql.tables.JobStoreSimpletonEntry
 import slick.model.ForeignKeyAction.Cascade
@@ -16,7 +16,7 @@ trait JobStoreSimpletonEntryComponent {
 
     def simpletonKey = column[String]("SIMPLETON_KEY", O.Length(255))
 
-    def simpletonValue = column[Option[Clob]]("SIMPLETON_VALUE")
+    def simpletonValue = column[Option[SerialClob]]("SIMPLETON_VALUE")
 
     def wdlType = column[String]("WDL_TYPE", O.Length(255))
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
index 9e8106aebd4..6e440fdeebf 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/MetadataEntryComponent.scala
@@ -1,6 +1,7 @@
 package cromwell.database.slick.tables
 
-import java.sql.{Clob, Timestamp}
+import java.sql.Timestamp
+import javax.sql.rowset.serial.SerialClob
 
 import cromwell.database.sql.tables.MetadataEntry
 
@@ -35,7 +36,7 @@ trait MetadataEntryComponent {
 
     def metadataKey = column[String]("METADATA_KEY", O.Length(255))
 
-    def metadataValue = column[Option[Clob]]("METADATA_VALUE")
+    def metadataValue = column[Option[SerialClob]]("METADATA_VALUE")
 
     def metadataValueType = column[Option[String]]("METADATA_VALUE_TYPE", O.Length(10))
 
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
index 14721104971..d5c5273d245 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowMetadataSummaryEntryComponent.scala
@@ -1,11 +1,11 @@
 package cromwell.database.slick.tables
 
 import java.sql.Timestamp
-import java.util.concurrent.atomic.AtomicInteger
 
 import cats.data
 import cats.data.NonEmptyList
 import cromwell.database.sql.tables.WorkflowMetadataSummaryEntry
+import shapeless.syntax.std.tuple._
 import slick.jdbc.{GetResult, PositionedParameters, SQLActionBuilder}
 
 //noinspection SqlDialectInspection
@@ -13,6 +13,7 @@ trait WorkflowMetadataSummaryEntryComponent {
 
   this: DriverComponent with CustomLabelEntryComponent with MetadataEntryComponent =>
 
+  import driver.api.TupleMethods._
   import driver.api._
 
   class WorkflowMetadataSummaryEntries(tag: Tag)
@@ -35,9 +36,15 @@ trait WorkflowMetadataSummaryEntryComponent {
 
     def rootWorkflowExecutionUuid = column[Option[String]]("ROOT_WORKFLOW_EXECUTION_UUID", O.Length(100))
 
-    override def * = (workflowExecutionUuid, workflowName, workflowStatus, startTimestamp, endTimestamp,
-      submissionTimestamp, parentWorkflowExecutionUuid, rootWorkflowExecutionUuid,
-      workflowMetadataSummaryEntryId.?) <> (WorkflowMetadataSummaryEntry.tupled, WorkflowMetadataSummaryEntry.unapply)
+    def baseProjection = (workflowExecutionUuid, workflowName, workflowStatus, startTimestamp, endTimestamp,
+      submissionTimestamp, parentWorkflowExecutionUuid, rootWorkflowExecutionUuid)
+
+    override def * = baseProjection ~ workflowMetadataSummaryEntryId.? <> (WorkflowMetadataSummaryEntry.tupled, WorkflowMetadataSummaryEntry.unapply)
+
+    def forUpdate = baseProjection.shaped <> (
+      tuple => WorkflowMetadataSummaryEntry.tupled(tuple :+ None),
+      WorkflowMetadataSummaryEntry.unapply(_: WorkflowMetadataSummaryEntry).map(_.reverse.tail.reverse)
+    )
 
     def ucWorkflowMetadataSummaryEntryWeu =
       index("UC_WORKFLOW_METADATA_SUMMARY_ENTRY_WEU", workflowExecutionUuid, unique = true)
@@ -63,7 +70,7 @@ trait WorkflowMetadataSummaryEntryComponent {
     (workflowExecutionUuid: Rep[String]) => for {
       workflowMetadataSummaryEntry <- workflowMetadataSummaryEntries
       if workflowMetadataSummaryEntry.workflowExecutionUuid === workflowExecutionUuid
-    } yield workflowMetadataSummaryEntry)
+    } yield workflowMetadataSummaryEntry.forUpdate)
 
   val workflowMetadataSummaryEntryExistsForWorkflowExecutionUuid = Compiled(
     (workflowExecutionUuid: Rep[String]) => (for {
@@ -81,8 +88,8 @@ trait WorkflowMetadataSummaryEntryComponent {
 
   def concat(a: SQLActionBuilder, b: SQLActionBuilder): SQLActionBuilder = {
     SQLActionBuilder(a.queryParts ++ b.queryParts, (p: Unit, pp: PositionedParameters) => {
-        a.unitPConv.apply(p, pp)
-        b.unitPConv.apply(p, pp)
+      a.unitPConv.apply(p, pp)
+      b.unitPConv.apply(p, pp)
     })
   }
 
@@ -118,79 +125,130 @@ trait WorkflowMetadataSummaryEntryComponent {
                        endTimestampOption: Option[Timestamp],
                        includeSubworkflows: Boolean): SQLActionBuilder = {
 
-    val summaryTableAlias = "summaryTable"
-    val labelsOrTableAlias = "labelsOrMixin"
-    val labelsAndTableAliases = labelAndKeyLabelValues.zipWithIndex.map { case (labelPair, i) => s"labelAndTable$i" -> labelPair }.toMap
+    val customLabelEntryTable = quoted("CUSTOM_LABEL_ENTRY")
+    val workflowMetadataSummaryEntryTable = quoted("WORKFLOW_METADATA_SUMMARY_ENTRY")
+
+    val workflowExecutionUuidColumn = quoted("WORKFLOW_EXECUTION_UUID")
+    val customLabelKeyColumn = quoted("CUSTOM_LABEL_KEY")
+    val customLabelValueColumn = quoted("CUSTOM_LABEL_VALUE")
+    val parentWorkflowExecutionUuidColumn = quoted("PARENT_WORKFLOW_EXECUTION_UUID")
+
+    val summaryTableAlias = quoted("summaryTable")
+    val labelsOrTableAlias = quoted("labelsOrMixin")
+    val labelsAndTableAliases = labelAndKeyLabelValues.zipWithIndex.map {
+      case (labelPair, i) => quoted(s"labelAndTable$i") -> labelPair
+    }.toMap
+
+    val selectColumns = List(
+      "WORKFLOW_EXECUTION_UUID",
+      "WORKFLOW_NAME",
+      "WORKFLOW_STATUS",
+      "START_TIMESTAMP",
+      "END_TIMESTAMP",
+      "SUBMISSION_TIMESTAMP",
+      "PARENT_WORKFLOW_EXECUTION_UUID",
+      "ROOT_WORKFLOW_EXECUTION_UUID",
+      "WORKFLOW_METADATA_SUMMARY_ENTRY_ID",
+    )
+      .map(quoted)
+      .mkString(s"$summaryTableAlias.", ", ", "")
 
     val select = selectOrCount match {
       case Select =>
-        sql"""|SELECT #$summaryTableAlias.WORKFLOW_EXECUTION_UUID,
-              |  #$summaryTableAlias.WORKFLOW_NAME,
-              |  #$summaryTableAlias.WORKFLOW_STATUS,
-              |  #$summaryTableAlias.START_TIMESTAMP,
-              |  #$summaryTableAlias.END_TIMESTAMP,
-              |  #$summaryTableAlias.SUBMISSION_TIMESTAMP,
-              |  #$summaryTableAlias.PARENT_WORKFLOW_EXECUTION_UUID,
-              |  #$summaryTableAlias.ROOT_WORKFLOW_EXECUTION_UUID,
-              |  #$summaryTableAlias.WORKFLOW_METADATA_SUMMARY_ENTRY_ID
-              | """.stripMargin
+        sql"""|SELECT #$selectColumns
+              |""".stripMargin
       case Count =>
-        sql"""SELECT COUNT(1)
-             | """.stripMargin
+        sql"""|SELECT COUNT(1)
+              |""".stripMargin
     }
 
     val labelOrJoin = if (labelOrKeyLabelValues.nonEmpty) {
       Option(
-        sql""" JOIN CUSTOM_LABEL_ENTRY #$labelsOrTableAlias on #$summaryTableAlias.WORKFLOW_EXECUTION_UUID = #$labelsOrTableAlias.WORKFLOW_EXECUTION_UUID
-              | """.stripMargin)
+        sql"""|  JOIN #$customLabelEntryTable #$labelsOrTableAlias
+              |    ON #$summaryTableAlias.#$workflowExecutionUuidColumn
+              |      = #$labelsOrTableAlias.#$workflowExecutionUuidColumn
+              |""".stripMargin)
     } else None
 
     val labelAndJoins = labelsAndTableAliases.toList.map { case (labelAndTableAlias, _) =>
-        sql""" JOIN CUSTOM_LABEL_ENTRY #$labelAndTableAlias on #$summaryTableAlias.WORKFLOW_EXECUTION_UUID = #$labelAndTableAlias.WORKFLOW_EXECUTION_UUID
-             | """.stripMargin
+      sql"""|  JOIN #$customLabelEntryTable #$labelAndTableAlias
+            |    ON #$summaryTableAlias.#$workflowExecutionUuidColumn
+            |      = #$labelAndTableAlias.#$workflowExecutionUuidColumn
+            |""".stripMargin
     }
 
     val from = concatNel(NonEmptyList.of(
-      sql"""FROM WORKFLOW_METADATA_SUMMARY_ENTRY #$summaryTableAlias
-           | """.stripMargin) ++ labelOrJoin.toList ++ labelAndJoins )
+      sql"""|FROM #$workflowMetadataSummaryEntryTable #$summaryTableAlias
+            |""".stripMargin) ++ labelOrJoin.toList ++ labelAndJoins)
+
+    def makeSetConstraint(column: String, elements: Set[String]) = {
+      val list = elements.toList.map(element => sql"""#$summaryTableAlias.#${quoted(column)} = $element""")
+      NonEmptyList.fromList(list).map(or).toList
+    }
 
-    val statusConstraint = NonEmptyList.fromList(workflowStatuses.toList.map(status => sql"""#$summaryTableAlias.WORKFLOW_STATUS=$status""")).map(or).toList
-    val nameConstraint = NonEmptyList.fromList(workflowNames.toList.map(name => sql"""#$summaryTableAlias.WORKFLOW_NAME=$name""")).map(or).toList
-    val idConstraint = NonEmptyList.fromList(workflowExecutionUuids.toList.map(uuid => sql"""#$summaryTableAlias.WORKFLOW_EXECUTION_UUID=$uuid""")).map(or).toList
-    val submissionTimeConstraint = submissionTimestampOption.map(ts => sql"""#$summaryTableAlias.SUBMISSION_TIMESTAMP>=$ts""").toList
-    val startTimeConstraint = startTimestampOption.map(ts => sql"""#$summaryTableAlias.START_TIMESTAMP>=$ts""").toList
-    val endTimeConstraint = endTimestampOption.map(ts => sql"""#$summaryTableAlias.END_TIMESTAMP<=$ts""").toList
+    def makeTimeConstraint(column: String, comparison: String, elementOption: Option[Timestamp]) = {
+      elementOption.map(element => sql"""#$summaryTableAlias.#${quoted(column)} #$comparison $element""").toList
+    }
+
+    val statusConstraint = makeSetConstraint("WORKFLOW_STATUS", workflowStatuses)
+    val nameConstraint = makeSetConstraint("WORKFLOW_NAME", workflowNames)
+    val idConstraint = makeSetConstraint("WORKFLOW_EXECUTION_UUID", workflowExecutionUuids)
+
+    val submissionTimeConstraint = makeTimeConstraint("SUBMISSION_TIMESTAMP", ">=", submissionTimestampOption)
+    val startTimeConstraint = makeTimeConstraint("START_TIMESTAMP", ">=", startTimestampOption)
+    val endTimeConstraint = makeTimeConstraint("END_TIMESTAMP", "<=", endTimestampOption)
 
     // *ALL* of the labelAnd list of KV pairs must exist:
-    val labelsAndConstraint = NonEmptyList.fromList(labelsAndTableAliases.toList.map { case (labelsAndTableAlias, (labelKey, labelValue)) =>
-      and(NonEmptyList.of(sql"#$labelsAndTableAlias.custom_label_key=$labelKey") :+ sql"#$labelsAndTableAlias.custom_label_value=$labelValue")
+    val labelsAndConstraint = NonEmptyList.fromList(labelsAndTableAliases.toList.map {
+      case (labelsAndTableAlias, (labelKey, labelValue)) =>
+        and(NonEmptyList.of(
+          sql"""#$labelsAndTableAlias.#$customLabelKeyColumn = $labelKey""",
+          sql"""#$labelsAndTableAlias.#$customLabelValueColumn = $labelValue""",
+        ))
     }).map(and).toList
 
     // At least one of the labelOr list of KV pairs must exist:
-    val labelOrConstraint = NonEmptyList.fromList(labelOrKeyLabelValues.toList.map { case (k, v) =>
-      and(NonEmptyList.of(sql"#$labelsOrTableAlias.custom_label_key=$k") :+ sql"#$labelsOrTableAlias.custom_label_value=$v")
+    val labelOrConstraint = NonEmptyList.fromList(labelOrKeyLabelValues.toList.map {
+      case (labelKey, labelValue) =>
+        and(NonEmptyList.of(
+          sql"""#$labelsOrTableAlias.#$customLabelKeyColumn = $labelKey""",
+          sql"""#$labelsOrTableAlias.#$customLabelValueColumn = $labelValue""",
+        ))
     }).map(or).toList
 
 
-    val mixinTableCounter = new AtomicInteger(0)
+    var mixinTableCounter = 0
 
     def labelExists(labelKey: String, labelValue: String) = {
-      val tableName = s"labelsMixin" + mixinTableCounter.getAndIncrement()
-      sql"""EXISTS(SELECT 1 from CUSTOM_LABEL_ENTRY #$tableName WHERE ((#$tableName.WORKFLOW_EXECUTION_UUID = #$summaryTableAlias.WORKFLOW_EXECUTION_UUID) AND (#$tableName.CUSTOM_LABEL_KEY = $labelKey) AND (#$tableName.CUSTOM_LABEL_VALUE = $labelValue)))"""
+      val tableName = quoted(s"labelsMixin" + mixinTableCounter)
+      mixinTableCounter += 1
+      sql"""|EXISTS (
+            |  SELECT 1 FROM #$customLabelEntryTable #$tableName
+            |    WHERE (
+            |      (#$tableName.#$workflowExecutionUuidColumn = #$summaryTableAlias.#$workflowExecutionUuidColumn)
+            |        AND (#$tableName.#$customLabelKeyColumn = $labelKey)
+            |        AND (#$tableName.#$customLabelValueColumn = $labelValue)
+            |    )
+            |)
+            |""".stripMargin
     }
 
     // *ALL* of the excludeLabelOr list of KV pairs must *NOT* exist:
-    val excludeLabelsOrConstraint = NonEmptyList.fromList(excludeLabelOrValues.toList.map { case (labelKey, labelValue) => not(labelExists(labelKey, labelValue)) } ).map(and).toList
+    val excludeLabelsOrConstraint = NonEmptyList.fromList(excludeLabelOrValues.toList map {
+      case (labelKey, labelValue) => not(labelExists(labelKey, labelValue))
+    }).map(and).toList
 
     // At least one of the excludeLabelAnd list of KV pairs must *NOT* exist:
-    val excludeLabelsAndConstraint = NonEmptyList.fromList(excludeLabelAndValues.toList.map { case (labelKey, labelValue) => not(labelExists(labelKey, labelValue)) } ).map(or).toList
+    val excludeLabelsAndConstraint = NonEmptyList.fromList(excludeLabelAndValues.toList.map {
+      case (labelKey, labelValue) => not(labelExists(labelKey, labelValue))
+    }).map(or).toList
 
     val includeSubworkflowsConstraint = if (includeSubworkflows) List.empty else {
-      List(sql"""#$summaryTableAlias.PARENT_WORKFLOW_EXECUTION_UUID IS NULL""".stripMargin)
+      List(sql"""#$summaryTableAlias.#$parentWorkflowExecutionUuidColumn IS NULL""".stripMargin)
     }
 
     val constraintList =
-        statusConstraint ++
+      statusConstraint ++
         nameConstraint ++
         idConstraint ++
         submissionTimeConstraint ++
@@ -274,18 +332,20 @@ trait WorkflowMetadataSummaryEntryComponent {
     )
 
     val paginationAddendum: List[SQLActionBuilder] = (page, pageSize) match {
-      case (Some(p), Some(ps)) => List(sql""" LIMIT #${Integer.max(p-1, 0) * ps},#$ps """)
-      case (None, Some(ps)) => List(sql""" LIMIT 0,#$ps """)
+      case (Some(p), Some(ps)) => List(sql""" LIMIT #$ps OFFSET #${ps * ((p - 1) max 0)}""")
+      case (None, Some(ps)) => List(sql""" LIMIT #$ps OFFSET 0""")
       case _ => List.empty
     }
 
-    val orderByAddendum = sql""" ORDER BY WORKFLOW_METADATA_SUMMARY_ENTRY_ID DESC
-                               | """.stripMargin
+    val orderByAddendum =
+      sql"""|  ORDER BY #${quoted("WORKFLOW_METADATA_SUMMARY_ENTRY_ID")} DESC
+            |""".stripMargin
 
     // NB you can preview the prepared statement created here by using, for example: println(result.statements.head)
 
-    concatNel((NonEmptyList.of(mainQuery) :+ orderByAddendum) ++ paginationAddendum)
-      .as[WorkflowMetadataSummaryEntry](rconv = GetResult { r =>
+    val fullQuery = concatNel(NonEmptyList(mainQuery, orderByAddendum :: paginationAddendum))
+
+    fullQuery.as[WorkflowMetadataSummaryEntry](rconv = GetResult { r =>
       WorkflowMetadataSummaryEntry(r.<<, r.<<, r.<<, r.<<, r.<<, r.<<, r.<<, r.<<, r.<<)
     })
   }
diff --git a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
index 3d99dafcd0c..d3888c8aa49 100644
--- a/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
+++ b/database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala
@@ -1,7 +1,7 @@
 package cromwell.database.slick.tables
 
-import java.sql.{Blob, Clob, Timestamp}
-
+import java.sql.Timestamp
+import javax.sql.rowset.serial.{SerialBlob, SerialClob}
 import cromwell.database.sql.tables.WorkflowStoreEntry
 
 trait WorkflowStoreEntryComponent {
@@ -21,21 +21,21 @@ trait WorkflowStoreEntryComponent {
 
     def workflowTypeVersion = column[Option[String]]("WORKFLOW_TYPE_VERSION", O.Length(255))
 
-    def workflowDefinition = column[Option[Clob]]("WORKFLOW_DEFINITION")
+    def workflowDefinition = column[Option[SerialClob]]("WORKFLOW_DEFINITION")
 
     def workflowUrl = column[Option[String]]("WORKFLOW_URL", O.Length(2000))
 
-    def workflowInputs = column[Option[Clob]]("WORKFLOW_INPUTS")
+    def workflowInputs = column[Option[SerialClob]]("WORKFLOW_INPUTS")
 
-    def workflowOptions = column[Option[Clob]]("WORKFLOW_OPTIONS")
+    def workflowOptions = column[Option[SerialClob]]("WORKFLOW_OPTIONS")
 
-    def customLabels = column[Clob]("CUSTOM_LABELS")
+    def customLabels = column[SerialClob]("CUSTOM_LABELS")
 
     def workflowState = column[String]("WORKFLOW_STATE", O.Length(20))
 
     def submissionTime = column[Timestamp]("SUBMISSION_TIME")
 
-    def importsZip = column[Option[Blob]]("IMPORTS_ZIP")
+    def importsZip = column[Option[SerialBlob]]("IMPORTS_ZIP")
 
     def cromwellId = column[Option[String]]("CROMWELL_ID", O.Length(100))
 
diff --git a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
index 44a7a5e8853..fe1bf5f7dba 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/SqlConverters.scala
@@ -32,12 +32,16 @@ object SqlConverters {
   }
 
   implicit class ClobToRawString(val clob: Clob) extends AnyVal {
-    // yes, it starts at 1
-    def toRawString: String = clob.getSubString(1, clob.length.toInt)
+    def toRawString: String = {
+      // See notes on empty clob issues in StringToClobOption
+      val length = clob.length.toInt
+      // yes, it starts at 1
+      if (length == 0) "" else clob.getSubString(1, length)
+    }
   }
 
   implicit class StringOptionToClobOption(val strOption: Option[String]) extends AnyVal {
-    def toClobOption: Option[Clob] = strOption.flatMap(_.toClobOption)
+    def toClobOption: Option[SerialClob] = strOption.flatMap(_.toClobOption)
   }
 
   implicit class StringToClobOption(val str: String) extends AnyVal {
@@ -52,17 +56,21 @@ object SqlConverters {
     import eu.timepit.refined.api.Refined
     import eu.timepit.refined.collection.NonEmpty
 
-    def toClobOption: Option[Clob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
+    def toClobOption: Option[SerialClob] = if (str.isEmpty) None else Option(new SerialClob(str.toCharArray))
 
-    def toClob(default: String Refined NonEmpty): Clob = {
+    def toClob(default: String Refined NonEmpty): SerialClob = {
       val nonEmpty = if (str.isEmpty) default.value else str
       new SerialClob(nonEmpty.toCharArray)
     }
   }
 
   implicit class BlobToBytes(val blob: Blob) extends AnyVal {
-    // yes, it starts at 1
-    def toBytes: Array[Byte] = blob.getBytes(1, blob.length.toInt)
+    def toBytes: Array[Byte] = {
+      // See notes on empty blob issues in BytesOptionToBlob
+      val length = blob.length.toInt
+      // yes, it starts at 1
+      if (length == 0) Array.empty else blob.getBytes(1, length)
+    }
   }
 
   implicit class BlobOptionToBytes(val blobOption: Option[Blob]) extends AnyVal {
@@ -79,11 +87,11 @@ object SqlConverters {
     https://github.com/apache/derby/blob/10.13/java/engine/org/apache/derby/iapi/types/HarmonySerialBlob.java#L111
     OK! -> https://github.com/arteam/hsqldb/blob/2.3.4/src/org/hsqldb/jdbc/JDBCBlob.java#L184
      */
-    def toBlobOption: Option[Blob] = bytesOption.flatMap(_.toBlobOption)
+    def toBlobOption: Option[SerialBlob] = bytesOption.flatMap(_.toBlobOption)
   }
 
   implicit class BytesToBlobOption(val bytes: Array[Byte]) extends AnyVal {
-    def toBlobOption: Option[Blob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes))
+    def toBlobOption: Option[SerialBlob] = if (bytes.isEmpty) None else Option(new SerialBlob(bytes))
   }
 
   implicit class EnhancedFiniteDuration(val duration: FiniteDuration) extends AnyVal {
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
index 0fd6d87cb15..31e6af183ae 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingDetritusEntry.scala
@@ -1,11 +1,11 @@
 package cromwell.database.sql.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 case class CallCachingDetritusEntry
 (
   detritusKey: String,
-  detritusValue: Option[Clob],
+  detritusValue: Option[SerialClob],
   callCachingEntryId: Option[Int] = None,
   callCachingDetritusEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
index fb2627ba3be..626246c7bf2 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/CallCachingSimpletonEntry.scala
@@ -1,11 +1,11 @@
 package cromwell.database.sql.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 case class CallCachingSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: Option[Clob],
+  simpletonValue: Option[SerialClob],
   wdlType: String,
   callCachingEntryId: Option[Int] = None,
   callCachingSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
index bf819e63283..c1a904af3be 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreEntry.scala
@@ -1,6 +1,6 @@
 package cromwell.database.sql.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 case class JobStoreEntry
 (
@@ -10,7 +10,7 @@ case class JobStoreEntry
   jobAttempt: Int,
   jobSuccessful: Boolean,
   returnCode: Option[Int],
-  exceptionMessage: Option[Clob],
+  exceptionMessage: Option[SerialClob],
   retryableFailure: Option[Boolean],
   jobStoreEntryId: Option[Int] = None
 )
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
index 909dd17fea2..e0c66921973 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/JobStoreSimpletonEntry.scala
@@ -1,11 +1,11 @@
 package cromwell.database.sql.tables
 
-import java.sql.Clob
+import javax.sql.rowset.serial.SerialClob
 
 case class JobStoreSimpletonEntry
 (
   simpletonKey: String,
-  simpletonValue: Option[Clob],
+  simpletonValue: Option[SerialClob],
   wdlType: String,
   jobStoreEntryId: Option[Int] = None,
   jobStoreSimpletonEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
index fcc4a40006e..c273c3e47e3 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/MetadataEntry.scala
@@ -1,6 +1,8 @@
 package cromwell.database.sql.tables
 
-import java.sql.{Clob, Timestamp}
+import java.sql.Timestamp
+
+import javax.sql.rowset.serial.SerialClob
 
 case class MetadataEntry
 (
@@ -9,7 +11,7 @@ case class MetadataEntry
   jobIndex: Option[Int],
   jobAttempt: Option[Int],
   metadataKey: String,
-  metadataValue: Option[Clob],
+  metadataValue: Option[SerialClob],
   metadataValueType: Option[String],
   metadataTimestamp: Timestamp,
   metadataEntryId: Option[Long] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
index 66f22ae12b7..b969939d16c 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/WorkflowStoreEntry.scala
@@ -1,21 +1,23 @@
 package cromwell.database.sql.tables
 
-import java.sql.{Blob, Clob, Timestamp}
+import java.sql.Timestamp
+
+import javax.sql.rowset.serial.{SerialBlob, SerialClob}
 
 case class WorkflowStoreEntry
 (
   workflowExecutionUuid: String,
-  workflowDefinition: Option[Clob],
+  workflowDefinition: Option[SerialClob],
   workflowUrl: Option[String],
   workflowRoot: Option[String],
   workflowType: Option[String],
   workflowTypeVersion: Option[String],
-  workflowInputs: Option[Clob],
-  workflowOptions: Option[Clob],
+  workflowInputs: Option[SerialClob],
+  workflowOptions: Option[SerialClob],
   workflowState: String,
   submissionTime: Timestamp,
-  importsZip: Option[Blob],
-  customLabels: Clob,
+  importsZip: Option[SerialBlob],
+  customLabels: SerialClob,
   cromwellId: Option[String],
   heartbeatTimestamp: Option[Timestamp],
   workflowStoreEntryId: Option[Int] = None
diff --git a/database/sql/src/main/scala/cromwell/database/sql/tables/package.scala b/database/sql/src/main/scala/cromwell/database/sql/tables/package.scala
index b9421d7b615..9ed1e801abf 100644
--- a/database/sql/src/main/scala/cromwell/database/sql/tables/package.scala
+++ b/database/sql/src/main/scala/cromwell/database/sql/tables/package.scala
@@ -20,10 +20,11 @@ package cromwell.database.sql
   * - `Double`
   * - `Int`
   * - `Long`
-  * - `java.sql.Clob`
+  * - `javax.sql.rowset.serial.SerialClob`
   * - `java.sql.Timestamp`
   *
-  * Nullable columns should be wrapped in an `Option`.
+  * Nullable columns should be wrapped in an `Option`.  Note that SerialClob is
+  * required instead of java.sql.Clob, for Postgres support.
   *
   * Primary and foreign key columns are the only columns that should be defaulted, as they are to be filled in by the
   * database, and cannot and should not be set within the business logic. On the other hand, columns to be filled in by
diff --git a/docs/Configuring.md b/docs/Configuring.md
index 08764bf3382..60248d0bbfd 100644
--- a/docs/Configuring.md
+++ b/docs/Configuring.md
@@ -297,6 +297,33 @@ url = "jdbc:mysql://host/cromwell?rewriteBatchedStatements=true&serverTimezone=U
 
 Using this option does not alter your database's underlying timezone; rather, it causes Cromwell to "speak UTC" when communicating with the DB, and the DB server performs the conversion for you. 
 
+**Using Cromwell with Postgresql**
+
+To use Postgresql as the database, you will need to install and enable the
+Large Object extension.  If the extension is present, setting up the database
+requires just these commands:
+
+```
+$ createdb cromwell
+$ psql -d cromwell -c "create extension lo;"
+```
+
+Postgresql configuration in Cromwell is very similar to MySQL.  An example:
+
+```hocon
+database {
+  profile = "slick.jdbc.PostgresProfile$"
+  db {
+    driver = "org.postgresql.Driver"
+    url = "jdbc:postgresql//localhost:5432/cromwell"
+    user = "user"
+    password = "pass"
+    port = 5432
+    connectionTimeout = 5000
+  }
+}
+```
+
 ### Abort
 
 **Control-C (SIGINT) abort handler**
diff --git a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
index b28c250490b..b2d73e21fcb 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/lifecycle/execution/callcaching/CallCachingSlickDatabaseSpec.scala
@@ -27,6 +27,8 @@ class CallCachingSlickDatabaseSpec extends FlatSpec with Matchers with ScalaFutu
 
   "SlickDatabase (mariadb)" should behave like testWith("database-test-mariadb")
 
+  "SlickDatabase (postgresql)" should behave like testWith("database-test-postgresql")
+
   def testWith(configPath: String): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
     lazy val dataAccess = new EngineSlickDatabase(databaseConfig)
diff --git a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
index aab401c9934..62c8b3879b2 100644
--- a/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
+++ b/engine/src/test/scala/cromwell/engine/workflow/workflowstore/SqlWorkflowStoreSpec.scala
@@ -31,6 +31,8 @@ class SqlWorkflowStoreSpec extends FlatSpec with Matchers with ScalaFutures with
 
   "SqlWorkflowStore (mariadb)" should behave like testWith("database-test-mariadb")
 
+  "SqlWorkflowStore (postgresql)" should behave like testWith("database-test-postgresql")
+
   def testWith(configPath: String): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
 
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
index e7c23b5a7de..fcebf3a8f76 100644
--- a/project/Dependencies.scala
+++ b/project/Dependencies.scala
@@ -61,6 +61,7 @@ object Dependencies {
   private val owlApiV = "5.1.9"
   private val paradiseV = "2.1.1"
   private val pegdownV = "1.6.0"
+  private val postgresV = "42.2.5"
   private val rdf4jV = "2.4.2"
   private val refinedV = "0.9.8"
   private val rhinoV = "1.7.10"
@@ -321,7 +322,8 @@ object Dependencies {
 
   private val dbmsDependencies = List(
     "org.hsqldb" % "hsqldb" % hsqldbV,
-    "mysql" % "mysql-connector-java" % mysqlV
+    "mysql" % "mysql-connector-java" % mysqlV,
+    "org.postgresql" % "postgresql" % postgresV
   )
 
   private val refinedTypeDependenciesList = List(
@@ -338,8 +340,9 @@ object Dependencies {
 
   val cloudSupportDependencies = googleApiClientDependencies ++ googleCloudDependencies ++ betterFilesDependencies ++ awsCloudDependencies
 
-  val databaseSqlDependencies = configDependencies ++ catsDependencies ++ slickDependencies ++ dbmsDependencies ++
-    refinedTypeDependenciesList
+  val databaseSqlDependencies = List(
+    "commons-io" % "commons-io" % commonsIoV,
+  ) ++ configDependencies ++ catsDependencies ++ slickDependencies ++ dbmsDependencies ++ refinedTypeDependenciesList
 
   val statsDDependencies = List(
     "nl.grons" %% "metrics-scala" % metrics3ScalaV,
diff --git a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
index 275398a6609..4106a5e8e62 100644
--- a/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
+++ b/services/src/test/scala/cromwell/services/ServicesStoreSpec.scala
@@ -253,6 +253,8 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
 
   "SlickDatabase (mariadb)" should behave like testWith("database-test-mariadb")
 
+  "SlickDatabase (postgresql)" should behave like testWith("database-test-postgresql")
+
   def testWith(configPath: String): Unit = {
     import ServicesStore.EnhancedSqlDatabase
 
@@ -300,14 +302,14 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       val future = for {
         product <- dataAccess.database.run(getProduct)
         _ <- product match {
-          case "HSQL Database Engine" =>
-            // HSQLDB doesn't crash because it calls getCharacterStream instead of getSubString.
-            dataAccess.addJobStores(jobStoreJoins, 1)
           case "MySQL" =>
+            // MySQL crashes because it calls SerialClob's getSubString instead of getCharacterStream
             dataAccess.addJobStores(jobStoreJoins, 1).failed map { exception =>
               exception should be(a[SerialException])
               exception.getMessage should be("Invalid position in SerialClob object set")
             }
+          case "HSQL Database Engine" | "PostgreSQL" =>
+            dataAccess.addJobStores(jobStoreJoins, 1)
         }
       } yield ()
 
@@ -343,15 +345,15 @@ class ServicesStoreSpec extends FlatSpec with Matchers with ScalaFutures with St
       val future = for {
         product <- dataAccess.database.run(getProduct)
         _ <- product match {
-          case "HSQL Database Engine" =>
-            // HSQLDB doesn't crash because it calls getBinaryStream instead of getBytes.
-            dataAccess.addWorkflowStoreEntries(workflowStoreEntries)
           case "MySQL" =>
+            // MySQL crashes because it calls SerialBlob's getBytes instead of getBinaryStream
             dataAccess.addWorkflowStoreEntries(workflowStoreEntries).failed map { exception =>
               exception should be(a[SerialException])
               exception.getMessage should
                 be("Invalid arguments: position cannot be less than 1 or greater than the length of the SerialBlob")
             }
+          case _ =>
+            dataAccess.addWorkflowStoreEntries(workflowStoreEntries)
         }
       } yield ()
 
diff --git a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
index 95749caddd7..8641d9e39b5 100644
--- a/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
+++ b/services/src/test/scala/cromwell/services/keyvalue/impl/KeyValueDatabaseSpec.scala
@@ -9,6 +9,7 @@ import cromwell.database.slick.EngineSlickDatabase
 import cromwell.database.sql.tables.JobKeyValueEntry
 import cromwell.services.EngineServicesStore
 import cromwell.services.ServicesStore.EnhancedSqlDatabase
+import org.postgresql.util.PSQLException
 import org.scalatest.concurrent.ScalaFutures
 import org.scalatest.time.{Millis, Seconds, Span}
 import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers, RecoverMethods}
@@ -37,6 +38,11 @@ class KeyValueDatabaseSpec extends FlatSpec with Matchers with ScalaFutures with
     "Column 'STORE_VALUE' cannot be null"
   )
 
+  "SlickDatabase (postgresql)" should behave like testWith[PSQLException](
+    "database-test-postgresql",
+    """ERROR: null value in column "STORE_VALUE" violates not-null constraint"""
+  )
+
   def testWith[E <: Throwable](configPath: String, failureMessage: String)(implicit classTag: ClassTag[E]): Unit = {
     lazy val databaseConfig = ConfigFactory.load.getConfig(configPath)
     lazy val dataAccess = new EngineSlickDatabase(databaseConfig)
diff --git a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
index 27f112d3e1e..2af41be01fb 100644
--- a/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
+++ b/services/src/test/scala/cromwell/services/metadata/impl/MetadataDatabaseAccessSpec.scala
@@ -42,6 +42,8 @@ class MetadataDatabaseAccessSpec extends FlatSpec with Matchers with ScalaFuture
 
   "MetadataDatabaseAccess (mariadb)" should behave like testWith("database-test-mariadb")
 
+  "MetadataDatabaseAccess (postgresql)" should behave like testWith("database-test-postgresql")
+
   implicit val ec = ExecutionContext.global
 
   implicit val defaultPatience = PatienceConfig(scaled(Span(30, Seconds)), scaled(Span(100, Millis)))
diff --git a/src/ci/bin/test.inc.sh b/src/ci/bin/test.inc.sh
index 9749082c628..bbbad2d60bc 100644
--- a/src/ci/bin/test.inc.sh
+++ b/src/ci/bin/test.inc.sh
@@ -326,7 +326,7 @@ cromwell::private::create_database_variables() {
 
     CROMWELL_BUILD_MARIADB_JDBC_URL="jdbc:mysql://${CROMWELL_BUILD_MARIADB_HOSTNAME}:${CROMWELL_BUILD_MARIADB_PORT}/${CROMWELL_BUILD_MARIADB_SCHEMA}?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
     CROMWELL_BUILD_MYSQL_JDBC_URL="jdbc:mysql://${CROMWELL_BUILD_MYSQL_HOSTNAME}:${CROMWELL_BUILD_MYSQL_PORT}/${CROMWELL_BUILD_MYSQL_SCHEMA}?useSSL=false&rewriteBatchedStatements=true&serverTimezone=UTC"
-    CROMWELL_BUILD_POSTGRESQL_JDBC_URL="jdbc:postgresql://${CROMWELL_BUILD_POSTGRESQL_HOSTNAME}:${CROMWELL_BUILD_POSTGRESQL_PORT}/${CROMWELL_BUILD_POSTGRESQL_SCHEMA}"
+    CROMWELL_BUILD_POSTGRESQL_JDBC_URL="jdbc:postgresql://${CROMWELL_BUILD_POSTGRESQL_HOSTNAME}:${CROMWELL_BUILD_POSTGRESQL_PORT}/${CROMWELL_BUILD_POSTGRESQL_SCHEMA}?reWriteBatchedInserts=true"
 
     export CROMWELL_BUILD_MARIADB_DOCKER_TAG
     export CROMWELL_BUILD_MARIADB_HOSTNAME

From a439100220d3c13fc16cc02282f4fb0c9400a3c3 Mon Sep 17 00:00:00 2001
From: Saloni Shah <salonipshah11@gmail.com>
Date: Wed, 26 Jun 2019 12:08:41 -0400
Subject: [PATCH 28/28] Fixes size() returning 0 for dos inputs [BA-5731]
 (#5039)

---
 .../drs_tests/wf_level_file_size.wdl           | 11 +++++++++++
 .../drs_wf_level_file_size.test                | 17 +++++++++++++++++
 .../nio/impl/drs/DrsCloudNioFileProvider.scala | 16 ++++++++++++----
 .../filesystems/drs/DrsPathBuilder.scala       | 18 +++++++++++++++++-
 .../filesystems/drs/DrsPathBuilderSpec.scala   |  8 ++++----
 .../resources/papi_application.inc.conf.ctmpl  | 12 ++++++++++++
 6 files changed, 73 insertions(+), 9 deletions(-)
 create mode 100644 centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl
 create mode 100644 centaur/src/main/resources/standardTestCases/drs_wf_level_file_size.test

diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl b/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl
new file mode 100644
index 00000000000..56d89094921
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl
@@ -0,0 +1,11 @@
+version 1.0
+
+workflow wf_level_file_size {
+    File input1 = "dos://wb-mock-drs-dev.storage.googleapis.com/4a3908ad-1f0b-4e2a-8a92-611f2123e8b0"
+    File input2 = "dos://wb-mock-drs-dev.storage.googleapis.com/0c8e7bc6-fd76-459d-947b-808b0605beb3"
+
+    output {
+        Float fileSize1 = size(input1)
+        Float fileSize2 = size(input2)
+    }
+}
diff --git a/centaur/src/main/resources/standardTestCases/drs_wf_level_file_size.test b/centaur/src/main/resources/standardTestCases/drs_wf_level_file_size.test
new file mode 100644
index 00000000000..67210745838
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/drs_wf_level_file_size.test
@@ -0,0 +1,17 @@
+name: drs_wf_level_read_size
+testFormat: workflowsuccess
+backends: [Papiv2NoDockerHubConfig]
+
+files {
+  workflow: drs_tests/wf_level_file_size.wdl
+}
+
+metadata {
+  workflowName: wf_level_file_size
+  status: Succeeded
+
+  "outputs.wf_level_file_size.fileSize1": 43.0
+  "outputs.wf_level_file_size.fileSize2": 45.0
+}
+
+
diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala
index dbe4e7c0b94..38d443e2f87 100644
--- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala
+++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala
@@ -39,10 +39,18 @@ class DrsCloudNioFileProvider(scheme: String,
 
 
   private def checkIfPathExistsThroughMartha(drsPath: String): IO[Boolean] = {
-    drsPathResolver.rawMarthaResponse(drsPath).use { marthaResponse =>
-      val errorMsg = s"Status line was null for martha response $marthaResponse."
-      toIO(Option(marthaResponse.getStatusLine), errorMsg)
-    }.map(_.getStatusCode == HttpStatus.SC_OK)
+    /*
+     * Unlike other cloud providers where directories are identified with a trailing slash at the end like `gs://bucket/dir/`,
+     * DRS has a concept of bundles for directories (not supported yet). Hence for method `checkDirectoryExists` which appends a trailing '/'
+     * to see if the current path is a directory, return false
+     */
+    if (drsPath.endsWith("/")) IO(false)
+    else {
+      drsPathResolver.rawMarthaResponse(drsPath).use { marthaResponse =>
+        val errorMsg = s"Status line was null for martha response $marthaResponse."
+        toIO(Option(marthaResponse.getStatusLine), errorMsg)
+      }.map(_.getStatusCode == HttpStatus.SC_OK)
+    }
   }
 
 
diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala
index d9cd2566be6..778bf01b95c 100644
--- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala
+++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilder.scala
@@ -15,9 +15,25 @@ case class DrsPathBuilder(fileSystemProvider: DrsCloudNioFileSystemProvider) ext
 
   override def name: String = "DRS"
 
+  /**
+    * Unlike other cloud providers where directories are identified with a trailing slash at the end like `gs://bucket/dir/`,
+    * DRS has a concept of bundles for directories (not supported yet). This sanitizes the path by removing the trailing '/'(s)
+    * so that the CloudNioFileSystemProvider & CloudNioPath does not treat a DRS path ending with '/' as a directory, otherwise
+    * its size is returned as 0
+    */
+  private def removeTrailingSlashes(path: String): String = {
+    if (path.length > (drsScheme.length + 3)) { //3: length of '://'
+      val pathArray = path.split(s"://")
+      val transformedPath = pathArray(1).replaceAll("[/]+$", "")
+      s"$drsScheme://$transformedPath"
+    }
+    else path
+  }
+
   override def build(pathAsString: String): Try[Path] = {
     if (pathAsString.startsWith(s"$drsScheme://")) {
-      Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(pathAsString))) flatMap { uri =>
+      val pathWithoutTrailingSlashes = removeTrailingSlashes(pathAsString)
+      Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(pathWithoutTrailingSlashes))) flatMap { uri =>
         if (!Option(uri.getScheme).exists(_.equalsIgnoreCase(fileSystemProvider.getScheme))) {
           Failure(new IllegalArgumentException(s"$pathAsString does not have a $drsScheme scheme."))
         } else {
diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala
index 713f7e19ac6..9d5f227125c 100644
--- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala
+++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala
@@ -99,8 +99,8 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi
       description = "a path ending in /",
       path = s"dos://$bucket/hello/world/",
       normalize = false,
-      pathAsString = s"dos://$bucket/hello/world/",
-      pathWithoutScheme = s"$bucket/hello/world/",
+      pathAsString = s"dos://$bucket/hello/world",
+      pathWithoutScheme = s"$bucket/hello/world",
       parent = s"dos://$bucket/hello/",
       getParent = s"dos://$bucket/hello/",
       root = s"dos://$bucket/",
@@ -278,8 +278,8 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi
       root = s"dos://blah/",
       name = "",
       getFileName = null,
-      getNameCount = 0,
-      isAbsolute = true),
+      getNameCount = 1,
+      isAbsolute = false),
 
     GoodPath(
       description = "an absolute path without a host",
diff --git a/src/ci/resources/papi_application.inc.conf.ctmpl b/src/ci/resources/papi_application.inc.conf.ctmpl
index 45389907230..c5051409bed 100644
--- a/src/ci/resources/papi_application.inc.conf.ctmpl
+++ b/src/ci/resources/papi_application.inc.conf.ctmpl
@@ -10,6 +10,9 @@ engine {
       auth = "service_account"
       project = "broad-dsde-cromwell-dev"
     }
+    drs {
+      auth = "service_account"
+    }
   }
 }
 
@@ -63,3 +66,12 @@ services {
 	}
   }
 }
+
+filesystems.drs.global.config.martha.url = "https://us-central1-broad-dsde-dev.cloudfunctions.net/martha_v2"
+
+drs {
+  localization {
+    docker-image = "broadinstitute/cromwell-dos:34-d8acfe3"
+    command-template = "mkdir -p $(dirname ${containerPath}) && /scripts/dosUrlLocalizer.sc ${drsPath} ${containerPath}"
+  }
+}