diff --git a/CHANGELOG.md b/CHANGELOG.md index e6954ea74bd..11e9a5dbd80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Cromwell Change Log +## 71 Release Notes + +### Bug Fixes + +* Fixed an issue handling data in Google Cloud Storage buckets with requester pays enabled that could sometimes cause I/O to fail. + ## 70 Release Notes ### CWL security fix [#6510](https://github.com/broadinstitute/cromwell/pull/6510) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala index 9d84493732b..95e898d6711 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala @@ -12,7 +12,7 @@ import wom.graph.CommandCallNode import wom.values.WomValue import scala.concurrent.Future -import scala.util.Try +import scala.util.{Success, Try} trait StandardInitializationActorParams { def workflowDescriptor: BackendWorkflowDescriptor @@ -82,20 +82,29 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo RuntimeAttributesDefault.workflowOptionsDefault(options, runtimeAttributesBuilder.coercionMap) } - override def validate(): Future[Unit] = { - Future.fromTry(Try { - calls foreach { call => - val runtimeAttributeKeys = call.callable.runtimeAttributes.attributes.keys.toList - val notSupportedAttributes = runtimeAttributesBuilder.unsupportedKeys(runtimeAttributeKeys).toList - - if (notSupportedAttributes.nonEmpty) { - val notSupportedAttrString = notSupportedAttributes mkString ", " - workflowLogger.warn( - s"Key/s [$notSupportedAttrString] is/are not supported by backend. " + - s"Unsupported attributes will not be part of job executions.") - } + def validateWorkflowOptions(): Try[Unit] = Success(()) + + def checkForUnsupportedRuntimeAttributes(): Try[Unit] = Try { + calls foreach { call => + val runtimeAttributeKeys = call.callable.runtimeAttributes.attributes.keys.toList + val notSupportedAttributes = runtimeAttributesBuilder.unsupportedKeys(runtimeAttributeKeys).toList + + if (notSupportedAttributes.nonEmpty) { + val notSupportedAttrString = notSupportedAttributes mkString ", " + workflowLogger.warn( + s"Key/s [$notSupportedAttrString] is/are not supported by backend. " + + s"Unsupported attributes will not be part of job executions.") } - }) + } + } + + override def validate(): Future[Unit] = { + Future.fromTry( + for { + _ <- validateWorkflowOptions() + _ <- checkForUnsupportedRuntimeAttributes() + } yield () + ) } override protected lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_hca.test b/centaur/src/main/resources/standardTestCases/drs_usa_hca.test index c97a75674b2..f8efd921d21 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_hca.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_hca.test @@ -16,7 +16,7 @@ metadata { status: Succeeded "outputs.drs_usa_hca.path" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_4641bafb-5190-425b-aea9-9c7b125515c8_e37266ba-790d-4641-aa76-854d94be2fbe/E18_20161004_Neurons_Sample_49_S048_L004_R2_005.fastq.gz" + "/cromwell_root/drs_localization_paths/hca_dev_20201217_test4/5acd55ef/E18_20161004_Neurons_Sample_49_S048_L004_R2_005.fastq.gz" "outputs.drs_usa_hca.hash" = "badf266412ff0e307232421e56d647ed" "outputs.drs_usa_hca.size" = 438932948 "outputs.drs_usa_hca.cloud" = diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test index c6d523f3937..4025860ed8d 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test @@ -16,16 +16,16 @@ metadata { status: Succeeded "outputs.drs_usa_jdr.path1" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_f90f5d7f-c507-4e56-abfc-b965a66023fb_585f3f19-985f-43b0-ab6a-79fa4c8310fc/hello_jade.json" + "/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" "outputs.drs_usa_jdr.path2" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_001afc86-da4c-4739-85be-26ca98d2693f_ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" + "/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs2/hello_jade_2.json" "outputs.drs_usa_jdr.hash1" = "faf12e94c25bef7df62e4a5eb62573f5" "outputs.drs_usa_jdr.hash2" = "19e1b021628130fda04c79ee9a056b67" "outputs.drs_usa_jdr.size1" = 18.0 "outputs.drs_usa_jdr.size2" = 38.0 # This JDR file has a gsUri that doesn't end in /fileName so it must be downloaded with the DRS localizer "outputs.drs_usa_jdr.cloud1" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_f90f5d7f-c507-4e56-abfc-b965a66023fb_585f3f19-985f-43b0-ab6a-79fa4c8310fc/hello_jade.json" + "/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" # This JDR file has a gsUri that can skip localization "outputs.drs_usa_jdr.cloud2" = "gs://broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test index 4d6cfd57936..ab050aea69f 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test @@ -16,7 +16,11 @@ metadata { status: Succeeded "outputs.drs_usa_jdr.path1" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_f90f5d7f-c507-4e56-abfc-b965a66023fb_585f3f19-985f-43b0-ab6a-79fa4c8310fc/hello_jade.json" + "/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" + # This JDR file has a gsUri that can be preresolved to a regular GCS file for improved localization performance. + # However this means that the file's container path is determined by the GCS localization logic and not the + # `localizationPath`-aware DRS localization logic. The GCS localization logic always uses a containerized version + # of the GCS path, which is what this expectation represents. "outputs.drs_usa_jdr.path2" = "/cromwell_root/broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" "outputs.drs_usa_jdr.hash1" = "faf12e94c25bef7df62e4a5eb62573f5" @@ -25,7 +29,7 @@ metadata { "outputs.drs_usa_jdr.size2" = 38.0 # This JDR file has a gsUri that doesn't end in /fileName so it must be downloaded with the DRS localizer "outputs.drs_usa_jdr.cloud1" = - "/cromwell_root/jade.datarepo-dev.broadinstitute.org/v1_f90f5d7f-c507-4e56-abfc-b965a66023fb_585f3f19-985f-43b0-ab6a-79fa4c8310fc/hello_jade.json" + "/cromwell_root/drs_localization_paths/CromwellSimpleWithFilerefs/hello_jade.json" # This JDR file has a gsUri that can skip localization "outputs.drs_usa_jdr.cloud2" = "gs://broad-jade-dev-data-bucket/e1941fb9-6537-4e1a-b70d-34352a3a7817/ad783b60-aeba-4055-8f7b-194880f37259/hello_jade_2.json" diff --git a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs index ffcb0c717c7..8e28a8ef541 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs +++ b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs @@ -1,3 +1,3 @@ { - "wf_reference_disk_test.check_if_localized_as_symlink.reference_file_input": "gs://gcp-public-data--broad-references/hg19/v0/README" + "wf_reference_disk_test.check_if_localized_as_symlink.reference_file_input": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.tile_db_header.vcf" } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index 2da9f0fe1f5..9e0b45e259e 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -1,11 +1,13 @@ package cloud.nio.impl.drs import cats.data.NonEmptyList +import cats.data.Validated.{Invalid, Valid} import cats.effect.{IO, Resource} import cats.implicits._ import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition} import cloud.nio.impl.drs.MarthaResponseSupport._ -import common.exception.toIO +import common.exception.{AggregatedMessageException, toIO} +import common.validation.ErrorOr.ErrorOr import io.circe._ import io.circe.generic.semiauto._ import io.circe.parser.decode @@ -35,14 +37,21 @@ abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = clientBuilder } - def getAccessToken: String + def getAccessToken: ErrorOr[String] - private def makeHttpRequestToMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): HttpPost = { - val postRequest = new HttpPost(drsConfig.marthaUrl) - val requestJson = MarthaRequest(drsPath, fields).asJson.noSpaces - postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) - postRequest.setHeader("Authorization", s"Bearer $getAccessToken") - postRequest + private def makeHttpRequestToMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): Resource[IO, HttpPost] = { + val io = getAccessToken match { + case Valid(token) => IO { + val postRequest = new HttpPost(drsConfig.marthaUrl) + val requestJson = MarthaRequest(drsPath, fields).asJson.noSpaces + postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) + postRequest.setHeader("Authorization", s"Bearer $token") + postRequest + } + case Invalid(errors) => + IO.raiseError(AggregatedMessageException("Error getting access token", errors.toList)) + } + Resource.eval(io) } private def httpResponseToMarthaResponse(drsPathForDebugging: String)(httpResponse: HttpResponse): IO[MarthaResponse] = { @@ -83,8 +92,10 @@ abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = } def rawMarthaResponse(drsPath: String, fields: NonEmptyList[MarthaField.Value]): Resource[IO, HttpResponse] = { - val httpPost = makeHttpRequestToMartha(drsPath, fields) - executeMarthaRequest(httpPost) + for { + httpPost <- makeHttpRequestToMartha(drsPath, fields) + response <- executeMarthaRequest(httpPost) + } yield response } /** * @@ -151,6 +162,7 @@ object MarthaField extends Enumeration { val Hashes: MarthaField.Value = Value("hashes") val FileName: MarthaField.Value = Value("fileName") val AccessUrl: MarthaField.Value = Value("accessUrl") + val LocalizationPath: MarthaField.Value = Value("localizationPath") } final case class MarthaRequest(url: String, fields: NonEmptyList[MarthaField.Value]) @@ -171,6 +183,9 @@ final case class AccessUrl(url: String, headers: Option[Map[String, String]]) * @param fileName A possible different file name for the object at gsUri, ex: "gsutil cp gs://bucket/12/345 my.vcf" * @param hashes Hashes for the contents stored at gsUri * @param accessUrl URL to query for signed URL + * @param localizationPath Optional localization path. TDR is currently the sole DRS provider specifying this value in + * DRS metadata, via the `aliases` field. As this is a distinct field from `fileName` in DRS + * metadata it is also made a distinct field in this response object. */ final case class MarthaResponse(size: Option[Long] = None, timeCreated: Option[String] = None, @@ -180,7 +195,8 @@ final case class MarthaResponse(size: Option[Long] = None, googleServiceAccount: Option[SADataObject] = None, fileName: Option[String] = None, hashes: Option[Map[String, String]] = None, - accessUrl: Option[AccessUrl] = None + accessUrl: Option[AccessUrl] = None, + localizationPath: Option[String] = None ) // Adapted from https://github.com/broadinstitute/martha/blob/f31933a3a11e20d30698ec4b4dc1e0abbb31a8bc/common/helpers.js#L210-L218 diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala index f6b9cdfbdd6..b9cf4856ae0 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala @@ -1,8 +1,10 @@ package cloud.nio.impl.drs import com.google.auth.oauth2.{AccessToken, OAuth2Credentials} +import common.validation.ErrorOr.ErrorOr import scala.concurrent.duration._ +import cats.syntax.validated._ case class EngineDrsPathResolver(drsConfig: DrsConfig, @@ -12,16 +14,20 @@ case class EngineDrsPathResolver(drsConfig: DrsConfig, extends DrsPathResolver(drsConfig, retryInternally = false) { //Based on method from GoogleRegistry - override def getAccessToken: String = { + override def getAccessToken: ErrorOr[String] = { def accessTokenTTLIsAcceptable(accessToken: AccessToken): Boolean = { (accessToken.getExpirationTime.getTime - System.currentTimeMillis()).millis.gteq(accessTokenAcceptableTTL) } Option(authCredentials.getAccessToken) match { - case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => accessToken.getTokenValue + case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => + accessToken.getTokenValue.validNel case _ => authCredentials.refresh() - authCredentials.getAccessToken.getTokenValue + Option(authCredentials.getAccessToken.getTokenValue) match { + case Some(accessToken) => accessToken.validNel + case None => "Could not refresh access token".invalidNel + } } } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala index de61b0d7b8f..b6825dcbbd2 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala @@ -9,6 +9,8 @@ object MockDrsPaths { val mockToken = "mock.token" + val DrsLocalizationPathsContainer = "drs_localization_paths" + private val drsPathPrefix = "drs://drs-host" val drsRelativePath = "drs-host/4d427aa3-5640-4f00-81ae-c33443f84acf/f3b148ac-1802-4acc-a0b9-610ea266fb61" @@ -17,12 +19,20 @@ object MockDrsPaths { val gcsRelativePathWithFileName = "drs-host/d7c75399-bcd3-4762-90e9-434de005679b/file.txt" + val gcsRelativePathWithFileNameFromLocalizationPath = s"$DrsLocalizationPathsContainer/dir/subdir/file.txt" + + val gcsRelativePathWithFileNameFromAllThePaths = s"$DrsLocalizationPathsContainer/dir/subdir/file.txt" + val drsPathResolvingGcsPath = s"$drsPathPrefix/4d427aa3-5640-4f00-81ae-c33443f84acf" val drsPathWithNonPathChars = s"$drsPathPrefix/4d427aa3_5640_4f00_81ae_c33443f84acf" val drsPathResolvingWithFileName = s"$drsPathPrefix/d7c75399-bcd3-4762-90e9-434de005679b" + val drsPathResolvingWithLocalizationPath = s"$drsPathPrefix/1e7ecfa6-2a77-41d7-a251-38a2f4919842" + + val drsPathResolvingWithAllThePaths = s"$drsPathPrefix/0524678a-365e-42f3-a1e7-e4c6ac499b35" + val drsPathResolvingToNoGcsPath = s"$drsPathPrefix/226686cf-22c9-4472-9f79-7a0b0044f253" val drsPathNotExistingInMartha = s"$drsPathPrefix/5e21b8c3-8eda-48d5-9a04-2b32e1571765" diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala index 45410656592..540e7be5f03 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala @@ -2,7 +2,9 @@ package cloud.nio.impl.drs import cats.data.NonEmptyList import cats.effect.IO +import cats.syntax.validated._ import com.google.cloud.NoCredentials +import common.validation.ErrorOr.ErrorOr import org.apache.http.impl.client.HttpClientBuilder import org.specs2.mock.Mockito import org.specs2.mock.Mockito._ @@ -37,6 +39,10 @@ class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfi private val marthaObjWithFileName = marthaObjWithGcsPath.copy(fileName = Option("file.txt")) + private val marthaObjWithLocalizationPath = marthaObjWithGcsPath.copy(localizationPath = Option("/dir/subdir/file.txt")) + + private val marthaObjWithAllThePaths = marthaObjWithLocalizationPath.copy(fileName = marthaObjWithFileName.fileName) + private val marthaObjWithNoGcsPath = marthaObjWithGcsPath.copy(gsUri = None) override def resolveDrsThroughMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): IO[MarthaResponse] = { @@ -44,6 +50,8 @@ class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfi case MockDrsPaths.drsPathResolvingGcsPath => IO(marthaObjWithGcsPath) case MockDrsPaths.drsPathWithNonPathChars => IO(marthaObjWithGcsPath) case MockDrsPaths.drsPathResolvingWithFileName => IO(marthaObjWithFileName) + case MockDrsPaths.drsPathResolvingWithLocalizationPath => IO.pure(marthaObjWithLocalizationPath) + case MockDrsPaths.drsPathResolvingWithAllThePaths => IO.pure(marthaObjWithAllThePaths) case MockDrsPaths.drsPathResolvingToNoGcsPath => IO(marthaObjWithNoGcsPath) case MockDrsPaths.drsPathNotExistingInMartha => IO.raiseError( @@ -55,5 +63,5 @@ class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfi } } - override lazy val getAccessToken: String = MockDrsPaths.mockToken + override lazy val getAccessToken: ErrorOr[String] = MockDrsPaths.mockToken.validNel } diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index a5bed18f465..5ae60b8236a 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -306,7 +306,7 @@ google { } # Filesystems available in this Crowmell instance # They can be enabled individually in the engine.filesystems stanza and in the config.filesystems stanza of backends -# There is a default built-in local filesytem that can also be referenced as "local" as well. +# There is a default built-in local filesystem that can also be referenced as "local" as well. filesystems { drs { class = "cromwell.filesystems.drs.DrsPathBuilderFactory" diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala new file mode 100644 index 00000000000..9954ea98671 --- /dev/null +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala @@ -0,0 +1,77 @@ +package drs.localizer + +import common.util.VersionUtil +import drs.localizer.CommandLineParser.AccessTokenStrategy._ +import drs.localizer.CommandLineParser.Usage + + +class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) { + lazy val localizerVersion: String = VersionUtil.getVersion("cromwell-drs-localizer") + + version("version") + + help("help").text("Cromwell DRS Localizer") + + head("cromwell-drs-localizer", localizerVersion) + + arg[String]("drs-object-id").text("DRS object ID").required(). + action((s, c) => + c.copy(drsObject = Option(s))) + arg[String]("container-path").text("Container path").required(). + action((s, c) => + c.copy(containerPath = Option(s))) + arg[String]("requester-pays-project").text("Requester pays project").optional(). + action((s, c) => + c.copy(googleRequesterPaysProject = Option(s))) + opt[String]('t', "access-token-strategy").text(s"Access token strategy, must be one of '$Azure' or '$Google' (default '$Google')"). + action((s, c) => + c.copy(accessTokenStrategy = Option(s.toLowerCase()))) + opt[String]('v', "vault-name").text("Azure vault name"). + action((s, c) => + c.copy(azureVaultName = Option(s))) + opt[String]('s', "secret-name").text("Azure secret name"). + action((s, c) => + c.copy(azureSecretName = Option(s))) + opt[String]('i', "identity-client-id").text("Azure identity client id"). + action((s, c) => + c.copy(azureIdentityClientId = Option(s))) + checkConfig(c => + c.accessTokenStrategy match { + case Some(Azure) if c.googleRequesterPaysProject.isEmpty => Right(()) + case Some(Google) if List(c.azureSecretName, c.azureVaultName, c.azureIdentityClientId).forall(_.isEmpty) => Right(()) + case Some(Azure) => Left(s"Requester pays project is only valid with access token strategy '$Google'") + case Some(Google) => Left(s"One or more specified options are only valid with access token strategy '$Azure'") + case Some(huh) => Left(s"Unrecognized access token strategy '$huh'") + case None => Left("Unspecified access token strategy") + } + ) +} + +object CommandLineParser { + /** + * These access token strategies are named simplistically as there is currently only one access token strategy being + * used for each of these cloud vendors. But it is certainly possible that multiple strategies could come into use + * for a particular vendor, in which case the names may need to become more specific for disambiguation. + */ + object AccessTokenStrategy { + val Azure = "azure" + val Google = "google" + } + + val Usage = + s""" +Usage: + java -jar /path/to/localizer.jar [options] drs://provider/object /local/path/to/file.txt [requester pays project] + + Note that the optional argument is only valid with access token strategy 'Google'. + """ + +} + +case class CommandLineArguments(accessTokenStrategy: Option[String] = Option(Google), + drsObject: Option[String] = None, + containerPath: Option[String] = None, + googleRequesterPaysProject: Option[String] = None, + azureVaultName: Option[String] = None, + azureSecretName: Option[String] = None, + azureIdentityClientId: Option[String] = None) diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala index 1104c482c56..ac27367b932 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala @@ -1,20 +1,10 @@ package drs.localizer import cloud.nio.impl.drs.{DrsConfig, DrsPathResolver} -import com.google.auth.oauth2.GoogleCredentials +import common.validation.ErrorOr.ErrorOr +import drs.localizer.accesstokens.AccessTokenStrategy -import scala.collection.JavaConverters._ - -class DrsLocalizerDrsPathResolver(drsConfig: DrsConfig) - extends DrsPathResolver(drsConfig) { - - private final val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" - private final val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" - private final val UserInfoScopes = List(UserInfoEmailScope, UserInfoProfileScope) - - override def getAccessToken: String = { - val scopedCredentials = GoogleCredentials.getApplicationDefault().createScoped(UserInfoScopes.asJava) - scopedCredentials.refreshAccessToken().getTokenValue - } +class DrsLocalizerDrsPathResolver(drsConfig: DrsConfig, accessTokenStrategy: AccessTokenStrategy) extends DrsPathResolver(drsConfig) { + override def getAccessToken: ErrorOr[String] = accessTokenStrategy.getAccessToken() } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala index dceb1c96afd..c7ecdd3f3e4 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala @@ -6,6 +6,8 @@ import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDi import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsPathResolver, MarthaField} import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff} import com.typesafe.scalalogging.StrictLogging +import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google} +import drs.localizer.accesstokens.{AccessTokenStrategy, AzureB2CAccessTokenStrategy, GoogleAccessTokenStrategy} import drs.localizer.downloaders.AccessUrlDownloader.Hashes import drs.localizer.downloaders._ @@ -14,30 +16,24 @@ import scala.language.postfixOps object DrsLocalizerMain extends IOApp with StrictLogging { - /* This assumes the args are as follows: - 0: DRS input - 1: download location - 2: Optional parameter- Requester Pays Billing project ID - Martha URL is passed as an environment variable - */ override def run(args: List[String]): IO[ExitCode] = { - val argsLength = args.length - - argsLength match { - case 2 => - new DrsLocalizerMain(args.head, args(1), None). - resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode) - case 3 => - new DrsLocalizerMain(args.head, args(1), Option(args(2))). - resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode) - case _ => - val argsList = if (args.nonEmpty) args.mkString(",") else "None" - logger.error(s"Received $argsLength arguments. DRS input and download location path is required. Requester Pays billing project ID is optional. " + - s"Arguments received: $argsList") - IO(ExitCode.Error) - } + val parser = buildParser() + + val parsedArgs = parser.parse(args, CommandLineArguments()) + + val localize: Option[IO[ExitCode]] = for { + pa <- parsedArgs + run <- pa.accessTokenStrategy.collect { + case Azure => runLocalizer(pa, AzureB2CAccessTokenStrategy(pa)) + case Google => runLocalizer(pa, GoogleAccessTokenStrategy) + } + } yield run + + localize getOrElse printUsage } + def buildParser(): scopt.OptionParser[CommandLineArguments] = new CommandLineParser() + val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff( initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2) @@ -48,16 +44,29 @@ object DrsLocalizerMain extends IOApp with StrictLogging { override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption)) } + + private def printUsage: IO[ExitCode] = { + System.err.println(CommandLineParser.Usage) + IO.pure(ExitCode.Error) + } + + def runLocalizer(commandLineArguments: CommandLineArguments, accessTokenStrategy: AccessTokenStrategy): IO[ExitCode] = { + val drsObject = commandLineArguments.drsObject.get + val containerPath = commandLineArguments.containerPath.get + new DrsLocalizerMain(drsObject, containerPath, accessTokenStrategy, commandLineArguments.googleRequesterPaysProject). + resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode) + } } class DrsLocalizerMain(drsUrl: String, downloadLoc: String, + accessTokenStrategy: AccessTokenStrategy, requesterPaysProjectIdOption: Option[String]) extends StrictLogging { def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { IO { val drsConfig = DrsConfig.fromEnv(sys.env) - new DrsLocalizerDrsPathResolver(drsConfig) + new DrsLocalizerDrsPathResolver(drsConfig, accessTokenStrategy) } } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala new file mode 100644 index 00000000000..756f5371c0f --- /dev/null +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala @@ -0,0 +1,7 @@ +package drs.localizer.accesstokens + +import common.validation.ErrorOr.ErrorOr + +trait AccessTokenStrategy { + def getAccessToken(): ErrorOr[String] +} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala new file mode 100644 index 00000000000..f71c17789d3 --- /dev/null +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala @@ -0,0 +1,10 @@ +package drs.localizer.accesstokens + +import cats.syntax.validated._ +import common.validation.ErrorOr.ErrorOr +import drs.localizer.CommandLineArguments + +case class AzureB2CAccessTokenStrategy(commandLineArguments: CommandLineArguments) extends AccessTokenStrategy { + // Wire in logic to extract B2C token from KeyVault using UAMI + override def getAccessToken(): ErrorOr[String] = "Azure UAMI access token strategy not yet implemented".invalidNel +} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala new file mode 100644 index 00000000000..513340d471d --- /dev/null +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala @@ -0,0 +1,28 @@ +package drs.localizer.accesstokens + +import cats.syntax.validated._ +import com.google.auth.oauth2.GoogleCredentials +import common.validation.ErrorOr.ErrorOr + +import scala.collection.JavaConverters._ +import scala.util.{Failure, Success, Try} + +/** + * Strategy for obtaining an access token from Google Application Default credentials that are assumed to already exist. + */ +case object GoogleAccessTokenStrategy extends AccessTokenStrategy { + private final val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" + private final val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" + private final val UserInfoScopes = List(UserInfoEmailScope, UserInfoProfileScope) + + override def getAccessToken(): ErrorOr[String] = { + Try { + val scopedCredentials = GoogleCredentials.getApplicationDefault().createScoped(UserInfoScopes.asJava) + scopedCredentials.refreshAccessToken().getTokenValue + } match { + case Success(null) => "null token value attempting to refresh access token".invalidNel + case Success(value) => value.validNel + case Failure(e) => s"Failed to refresh access token: ${e.getMessage}".invalidNel + } + } +} diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala new file mode 100644 index 00000000000..b0eb996b85c --- /dev/null +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala @@ -0,0 +1,103 @@ +package drs.localizer + +import common.assertion.CromwellTimeoutSpec +import drs.localizer.CommandLineParser.AccessTokenStrategy +import org.scalatest.BeforeAndAfter +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with BeforeAndAfter { + + var parser: scopt.OptionParser[CommandLineArguments] = _ + + private val drsObject = "drs://provider/object" + private val containerPath = "/cromwell_root/my.bam" + private val requesterPaysProject = "i_heart_egress" + private val azureVaultName = "Kwikset" + private val azureSecretName = "shhh" + private val azureIdentityClientId = "itme@azure.com" + + behavior of "DRS Localizer command line parser" + + before { + parser = DrsLocalizerMain.buildParser() + } + + it should "fail to parse with no arguments" in { + parser.parse(Array.empty[String], CommandLineArguments()) shouldBe None + } + + it should "fail to parse with only one argument" in { + parser.parse(Array("some arg"), CommandLineArguments()) shouldBe None + } + + it should "successfully parse with two arguments" in { + val args = parser.parse(Array(drsObject, containerPath), CommandLineArguments()).get + + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject shouldBe empty + args.azureVaultName shouldBe empty + args.azureSecretName shouldBe empty + args.azureIdentityClientId shouldBe empty + } + + it should "successfully parse with three arguments" in { + val args = parser.parse(Array(drsObject, containerPath, requesterPaysProject), CommandLineArguments()).get + + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject.get shouldBe requesterPaysProject + args.azureVaultName shouldBe empty + args.azureSecretName shouldBe empty + args.azureIdentityClientId shouldBe empty + } + + it should "successfully parse an explicit Google access token strategy invocation" in { + val args = parser.parse(Array("--access-token-strategy", "google", drsObject, containerPath, requesterPaysProject), CommandLineArguments()).get + + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject.get shouldBe requesterPaysProject + args.azureVaultName shouldBe empty + args.azureSecretName shouldBe empty + args.azureIdentityClientId shouldBe empty + } + + it should "successfully parse an Azure invocation" in { + val args = parser.parse(Array("--access-token-strategy", AccessTokenStrategy.Azure, drsObject, containerPath), CommandLineArguments()).get + + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Azure + args.googleRequesterPaysProject shouldBe empty + args.azureVaultName shouldBe empty + args.azureSecretName shouldBe empty + args.azureIdentityClientId shouldBe empty + } + + it should "successfully parse an Azure invocation with all the trimmings" in { + val args = parser.parse(Array( + "--access-token-strategy", AccessTokenStrategy.Azure, + "--vault-name", azureVaultName, + "--secret-name", azureSecretName, + "--identity-client-id", azureIdentityClientId, + drsObject, containerPath), CommandLineArguments()).get + + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Azure + args.googleRequesterPaysProject shouldBe empty + args.azureVaultName.get shouldBe azureVaultName + args.azureSecretName.get shouldBe azureSecretName + args.azureIdentityClientId.get shouldBe azureIdentityClientId + } + + it should "fail to parse with an unrecognized access token strategy" in { + val args = parser.parse(Array("--access-token-strategy", "nebulous", drsObject, containerPath), CommandLineArguments()) + args shouldBe None + } +} diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala index eb60ed99207..84b407cec41 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala @@ -2,10 +2,12 @@ package drs.localizer import cats.data.NonEmptyList import cats.effect.{ExitCode, IO} +import cats.syntax.validated._ import cloud.nio.impl.drs.DrsPathResolver.FatalRetryDisposition import cloud.nio.impl.drs.{AccessUrl, DrsConfig, MarthaField, MarthaResponse} import common.assertion.CromwellTimeoutSpec -import drs.localizer.MockDrsLocalizerDrsPathResolver.FakeHashes +import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes} +import drs.localizer.accesstokens.AccessTokenStrategy import drs.localizer.downloaders.AccessUrlDownloader.Hashes import drs.localizer.downloaders._ import org.scalatest.flatspec.AnyFlatSpec @@ -300,7 +302,7 @@ class MockDrsLocalizerMain(drsUrl: String, downloadLoc: String, requesterPaysProjectIdOption: Option[String], ) - extends DrsLocalizerMain(drsUrl, downloadLoc, requesterPaysProjectIdOption) { + extends DrsLocalizerMain(drsUrl, downloadLoc, FakeAccessTokenStrategy, requesterPaysProjectIdOption) { override def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { IO { @@ -311,7 +313,7 @@ class MockDrsLocalizerMain(drsUrl: String, class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends - DrsLocalizerDrsPathResolver(drsConfig) { + DrsLocalizerDrsPathResolver(drsConfig, FakeAccessTokenStrategy) { override def resolveDrsThroughMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): IO[MarthaResponse] = { val marthaResponse = MarthaResponse( @@ -339,4 +341,5 @@ class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends object MockDrsLocalizerDrsPathResolver { val FakeHashes: Option[Map[String, String]] = Option(Map("md5" -> "abc123", "crc32c" -> "34fd67")) + val FakeAccessTokenStrategy: AccessTokenStrategy = () => "testing code: do not call me".invalidNel } diff --git a/docs/backends/Google.md b/docs/backends/Google.md index faa3dc786b0..907b90c10f6 100644 --- a/docs/backends/Google.md +++ b/docs/backends/Google.md @@ -311,48 +311,51 @@ This filesystem has two required configuration options: ### Virtual Private Network -To run your jobs in a private network add the `virtual-private-cloud` stanza in the `config` stanza of the PAPI v2 backend: +Cromwell can arrange for jobs to run in specific GCP private networks via the `config.virtual-private-cloud` stanza of a PAPI v2 backend. +There are two ways of specifying private networks: -#### Virtual Private Network via Labels +* [Literal network and subnetwork values](#virtual-private-network-via-literals) that will apply to all projects +* [Google project labels](#virtual-private-network-via-labels) whose values in a particular Google project will specify the network and subnetwork + +#### Virtual Private Network via Literals ```hocon backend { ... providers { - ... - PapiV2 { - actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory" - config { - ... - virtual-private-cloud { - network-label-key = "my-private-network" - subnetwork-label-key = "my-private-subnetwork" - auth = "reference-to-auth-scheme" - } - ... - } + ... + PapiV2 { + actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory" + config { + ... + virtual-private-cloud { + network-name = "vpc-network" + subnetwork-name = "vpc-subnetwork" + } + ... } + } } } ``` +The `network-name` and `subnetwork-name` should reference the name of your private network and subnetwork within that +network respectively. The `subnetwork-name` is an optional config. -The `network-label-key` and `subnetwork-label-key` should reference the keys in your project's labels whose value is the name of your private network -and subnetwork within that network respectively. `auth` should reference an auth scheme in the `google` stanza which will be used to get the project metadata from Google Cloud. -The `subnetwork-label-key` is an optional config. +For example, if your `virtual-private-cloud` config looks like the one above, then Cromwell will use the value of the +configuration key, which is `vpc-network` here, as the name of private network and run the jobs on this network. +If the network name is not present in the config Cromwell will fall back to trying to run jobs on the default network. -For example, if your `virtual-private-cloud` config looks like the one above, and one of the labels in your project is +If the `network-name` or `subnetwork-name` values contain the string `${projectId}` then that value will be replaced +by Cromwell with the name of the project running the Pipelines API. -``` -"my-private-network" = "vpc-network" -``` +If the `network-name` does not contain a `/` then it will be prefixed with `projects/${projectId}/global/networks/`. -Cromwell will get labels from the project's metadata and look for a label whose key is `my-private-network`. -Then it will use the value of the label, which is `vpc-network` here, as the name of private network and run the jobs on this network. -If the network key is not present in the project's metadata Cromwell will fall back to trying to run jobs using literal -network labels, and then fall back to running on the default network. +Cromwell will then pass the network and subnetwork values to the Pipelines API. See the documentation for the +[Cloud Life Sciences API](https://cloud.google.com/life-sciences/docs/reference/rest/v2beta/projects.locations.pipelines/run#Network) +for more information on the various formats accepted for `network` and `subnetwork`. -#### Virtual Private Network via Literals +#### Virtual Private Network via Labels ```hocon backend { @@ -362,11 +365,12 @@ backend { PapiV2 { actor-factory = "cromwell.backend.google.pipelines.v2beta.PipelinesApiLifecycleActorFactory" config { - ... - virtual-private-cloud { - network-name = "vpc-network" - subnetwork-name = "vpc-subnetwork" - } + ... + virtual-private-cloud { + network-label-key = "my-private-network" + subnetwork-label-key = "my-private-subnetwork" + auth = "reference-to-auth-scheme" + } ... } } @@ -374,21 +378,21 @@ backend { } ``` -The `network-name` and `subnetwork-name` should reference the name of your private network and subnetwork within that -network respectively. The `subnetwork-name` is an optional config. -For example, if your `virtual-private-cloud` config looks like the one above, then Cromwell will use the value of the -configuration key, which is `vpc-network` here, as the name of private network and run the jobs on this network. -If the network name is not present in the config Cromwell will fall back to trying to run jobs on the default network. +The `network-label-key` and `subnetwork-label-key` should reference the keys in your project's labels whose value is the name of your private network +and subnetwork within that network respectively. `auth` should reference an auth scheme in the `google` stanza which will be used to get the project metadata from Google Cloud. +The `subnetwork-label-key` is an optional config. -If the `network-name` or `subnetwork-name` values contain the string `${projectId}` then that value will be replaced -by Cromwell with the name of the project running the Pipelines API. +For example, if your `virtual-private-cloud` config looks like the one above, and one of the labels in your project is -If the `network-name` does not contain a `/` then it will be prefixed with `projects/${projectId}/global/networks/`. +``` +"my-private-network" = "vpc-network" +``` -Cromwell will then pass the network and subnetwork values to the Pipelines API. See the documentation for the -[Cloud Life Sciences API](https://cloud.google.com/life-sciences/docs/reference/rest/v2beta/projects.locations.pipelines/run#Network) -for more information on the various formats accepted for `network` and `subnetwork`. +Cromwell will get labels from the project's metadata and look for a label whose key is `my-private-network`. +Then it will use the value of the label, which is `vpc-network` here, as the name of private network and run the jobs on this network. +If the network key is not present in the project's metadata Cromwell will fall back to trying to run jobs using literal +network labels, and then fall back to running on the default network. ### Custom Google Cloud SDK container Cromwell can't use Google's container registry if VPC Perimeter is used in project. diff --git a/docs/developers/Centaur.md b/docs/developers/Centaur.md index 795a8e24453..cf168a3026c 100644 --- a/docs/developers/Centaur.md +++ b/docs/developers/Centaur.md @@ -14,7 +14,7 @@ You can now run the tests from another terminal. There are two ways to invoke the integration tests: -* `sbt "centaur/it:test"` - compiles and run via sbt directly, simple but also has the problem of running 2x cores tests in parallel which can overwhelm your Cromwell server if running in a development environment +* `sbt "centaur / IntegrationTest / test"` - compiles and run via sbt directly, simple but also has the problem of running 2x cores tests in parallel which can overwhelm your Cromwell server if running in a development environment * `src/ci/bin/testCentaurLocal.sh` - runs the same tests using the continuous integration pipeline configuration @@ -26,12 +26,12 @@ Tag names are all lower case, so a test named "tagFoo" has a tag "tagfoo". To run only those tests which have been tagged with a specified tag `tagFoo`: ``` -sbt "centaur/it:testOnly * -- -n tagfoo" +sbt "centaur / IntegrationTest / testOnly * -- -n tagfoo" ``` Or to instead exclude all tests which have been tagged with a specified tag `tagFoo`: ``` -sbt "centaur/it:testOnly * -- -l tagfoo" +sbt "centaur / IntegrationTest / testOnly * -- -l tagfoo" ``` ## Adding custom tests diff --git a/engine/src/main/resources/swagger/cromwell.yaml b/engine/src/main/resources/swagger/cromwell.yaml index d680be46aff..249933560b1 100644 --- a/engine/src/main/resources/swagger/cromwell.yaml +++ b/engine/src/main/resources/swagger/cromwell.yaml @@ -3,6 +3,9 @@ # swagger: '2.0' + +# This line is included when the YAML is served by Cromwell if it sees an environment variable called "SWAGGER_BASE_PATH" +#basePath: ... info: title: Cromwell Server REST API description: Describes the REST API provided by a Cromwell server diff --git a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala index b5126c72a80..6c9bf54d7bf 100644 --- a/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala +++ b/engine/src/main/scala/cromwell/webservice/SwaggerUiHttpService.scala @@ -1,10 +1,12 @@ package cromwell.webservice -import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import akka.http.scaladsl.server.Route import com.typesafe.config.Config import net.ceedubs.ficus.Ficus._ import akka.http.scaladsl.server.Directives._ +import akka.stream.scaladsl.Flow +import akka.util.ByteString /** * Serves up the swagger UI from org.webjars/swagger-ui. @@ -94,6 +96,11 @@ trait SwaggerUiConfigHttpService extends SwaggerUiHttpService { * swagger UI, but defaults to "yaml". This is an alternative to spray-swagger's SwaggerHttpService. */ trait SwaggerResourceHttpService { + + def getBasePathOverride(): Option[String] = { + Option(System.getenv("SWAGGER_BASE_PATH")) + } + /** * @return The directory for the resource under the classpath, and in the url */ @@ -132,10 +139,22 @@ trait SwaggerResourceHttpService { */ final def swaggerResourceRoute: Route = { val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) + + def injectBasePath(basePath: Option[String])(response: HttpResponse): HttpResponse = { + basePath match { + case _ if response.status != StatusCodes.OK => response + case None => response + case Some(base_path) => response.mapEntity { entity => + val swapperFlow: Flow[ByteString, ByteString, Any] = Flow[ByteString].map(byteString => ByteString.apply(byteString.utf8String.replace("#basePath: ...", "basePath: " + base_path))) + entity.transformDataBytes(swapperFlow) + } + } + } + val route = get { swaggerDocsDirective { // Return /uiPath/serviceName.resourceType from the classpath resources. - getFromResource(swaggerDocsPath) + mapResponse(injectBasePath(getBasePathOverride()))(getFromResource(swaggerDocsPath)) } } diff --git a/engine/src/test/resources/swagger/testservice.yaml b/engine/src/test/resources/swagger/testservice.yaml index 186a0b913c0..0ca68741961 100644 --- a/engine/src/test/resources/swagger/testservice.yaml +++ b/engine/src/test/resources/swagger/testservice.yaml @@ -1,4 +1,6 @@ swagger: '2.0' +# This line is included when the YAML is served by Cromwell if it sees an environment variable called "SWAGGER_BASE_PATH" +#basePath: ... info: title: Test Service API description: Test Service API diff --git a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala index 261a6ef3a0e..dcea4c0d2bc 100644 --- a/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala +++ b/engine/src/test/scala/cromwell/webservice/SwaggerUiHttpServiceSpec.scala @@ -75,6 +75,21 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { } } +class OverrideBasePathSwaggerUiHttpServiceSpec extends SwaggerResourceHttpServiceSpec { + override def swaggerServiceName = "testservice" + + override def getBasePathOverride(): Option[String] = Option("/proxy/abc") + + behavior of "SwaggerResourceHttpService" + + it should "inject basePath url into cromwell swagger service" in { + Get("/swagger/testservice.yaml") ~> swaggerResourceRoute ~> check { + status should be(StatusCodes.OK) + responseAs[String] should include("basePath: /proxy/abc") + } + } +} + class NoRedirectRootSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { override def swaggerUiFromRoot = false diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala index 49896dacbda..0f24e803480 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import cats.effect.IO import cloud.nio.impl.drs.{DrsCloudNioFileSystemProvider, DrsPathResolver, MarthaField} import common.exception._ -import cromwell.core.path.DefaultPathBuilder +import cromwell.core.path.{DefaultPathBuilder, Path} import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.syntax.typeable._ @@ -14,6 +14,8 @@ object DrsResolver { private val GcsProtocolLength: Int = 5 // length of 'gs://' + private val DrsLocalizationPathsContainer = "drs_localization_paths" + private def resolveError[A](pathAsString: String)(throwable: Throwable): IO[A] = { IO.raiseError( new RuntimeException( @@ -32,35 +34,39 @@ object DrsResolver { } yield drsFileSystemProvider.drsPathResolver } - private def getGsUriFileNameBondProvider(pathAsString: String, - drsPathResolver: DrsPathResolver - ): IO[(Option[String], Option[String], Option[String])] = { - val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.FileName, MarthaField.BondProvider) - for { - marthaResponse <- drsPathResolver.resolveDrsThroughMartha(pathAsString, fields) - } yield (marthaResponse.gsUri, marthaResponse.fileName, marthaResponse.bondProvider) + case class MarthaLocalizationData(gsUri: Option[String], + fileName: Option[String], + bondProvider: Option[String], + localizationPath: Option[String]) + + private def getMarthaLocalizationData(pathAsString: String, + drsPathResolver: DrsPathResolver): IO[MarthaLocalizationData] = { + val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.FileName, MarthaField.BondProvider, MarthaField.LocalizationPath) + + drsPathResolver.resolveDrsThroughMartha(pathAsString, fields) map { r => + MarthaLocalizationData(r.gsUri, r.fileName, r.bondProvider, r.localizationPath) + } } /** Returns the `gsUri` if it ends in the `fileName` and the `bondProvider` is empty. */ - private def getSimpleGsUri(gsUriOption: Option[String], - fileNameOption: Option[String], - bondProviderOption: Option[String], - ): Option[String] = { - for { - // Only return gsUri that do not use Bond - gsUri <- if (bondProviderOption.isEmpty) gsUriOption else None - // Only return the gsUri if there is no fileName or if gsUri ends in /fileName - if fileNameOption.forall(fileName => gsUri.endsWith(s"/$fileName")) - } yield gsUri + private def getSimpleGsUri(localizationData: MarthaLocalizationData): Option[String] = { + localizationData match { + // `gsUri` not defined so no gsUri can be returned. + case MarthaLocalizationData(None, _, _, _) => None + // `bondProvider` defined, cannot "preresolve" to GCS. + case MarthaLocalizationData(_, _, Some(_), _) => None + // Do not return the simple GS URI if the `fileName` from metadata is mismatched to the filename in the `gsUri`. + case MarthaLocalizationData(Some(gsUri), Some(fileName), _, _) if !gsUri.endsWith(s"/$fileName") => None + // Barring any of the situations above return the `gsUri`. + case MarthaLocalizationData(Some(gsUri), _, _, _) => Option(gsUri) + } } /** Returns the `gsUri` if it ends in the `fileName` and the `bondProvider` is empty. */ def getSimpleGsUri(pathAsString: String, drsPathResolver: DrsPathResolver): IO[Option[String]] = { - val gsUriIO = for { - tuple <- getGsUriFileNameBondProvider(pathAsString, drsPathResolver) - (gsUriOption, fileNameOption, bondProviderOption) = tuple - } yield getSimpleGsUri(gsUriOption, fileNameOption, bondProviderOption) + + val gsUriIO = getMarthaLocalizationData(pathAsString, drsPathResolver) map getSimpleGsUri gsUriIO.handleErrorWith(resolveError(pathAsString)) } @@ -76,35 +82,39 @@ object DrsResolver { def getContainerRelativePath(drsPath: DrsPath): IO[String] = { val pathIO = for { drsPathResolver <- getDrsPathResolver(drsPath) - tuple <- getGsUriFileNameBondProvider(drsPath.pathAsString, drsPathResolver) - (gsUriOption, fileNameOption, _) = tuple - /* - In the DOS/DRS spec file names are safe for file systems but not necessarily the DRS URIs. - Reuse the regex defined for ContentsObject.name, plus add "/" for directory separators. - https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/docs/#_contentsobject - */ - rootPath = DefaultPathBuilder.get(drsPath.pathWithoutScheme.replaceAll("[^/A-Za-z0-9._-]", "_")) - fileName <- getFileName(fileNameOption, gsUriOption) - fullPath = rootPath.resolve(fileName) - fullPathString = fullPath.pathAsString - } yield fullPathString + localizationData <- getMarthaLocalizationData(drsPath.pathAsString, drsPathResolver) + containerRelativePath <- buildContainerRelativePath(localizationData, drsPath) + } yield containerRelativePath.pathAsString pathIO.handleErrorWith(resolveError(drsPath.pathAsString)) } - /** - * Return the file name returned from the martha response or get it from the gsUri - */ - private def getFileName(fileName: Option[String], gsUri: Option[String]): IO[String] = { - fileName match { - case Some(actualFileName) => IO.pure(actualFileName) - case None => - //Currently, Martha only supports resolving DRS paths to GCS paths + // Return the container relative path built from the Martha-specified localization path, file name, or gs URI. + private def buildContainerRelativePath(localizationData: MarthaLocalizationData, drsPath: Path): IO[Path] = { + // Return a relative path constructed from the DRS path minus the leading scheme. + // In the DOS/DRS spec file names are safe for file systems but not necessarily the DRS URIs. + // Reuse the regex defined for ContentsObject.name, plus add "/" for directory separators. + // https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/docs/#_contentsobject + def drsPathRelativePath: Path = + DefaultPathBuilder.get(drsPath.pathWithoutScheme.replaceAll("[^/A-Za-z0-9._-]", "_")) + + localizationData match { + case MarthaLocalizationData(_, _, _, Some(localizationPath)) => + // TDR may return an explicit localization path and if so we should not use the `drsPathRelativePath`. + // We want to end up with something like /cromwell_root/drs_localization_paths/tdr/specified/path/foo.bam. + // Calling code will add the `/cromwell_root/`, so strip any leading slashes to make this a relative path: + val relativeLocalizationPath = if (localizationPath.startsWith("/")) localizationPath.tail else localizationPath + IO.fromTry(DefaultPathBuilder.build(DrsLocalizationPathsContainer).map(_.resolve(relativeLocalizationPath))) + case MarthaLocalizationData(_, Some(fileName), _, _) => + // Paths specified by filename only are made relative to `drsPathRelativePath`. + IO(drsPathRelativePath.resolve(fileName)) + case _ => + // If this logic is forced to fall back on the GCS path there better be a GCS path to fall back on. IO - .fromEither(gsUri.toRight(UrlNotFoundException(GcsScheme))) + .fromEither(localizationData.gsUri.toRight(UrlNotFoundException(GcsScheme))) .map(_.substring(GcsProtocolLength)) .map(DefaultPathBuilder.get(_)) - .map(_.name) + .map(path => drsPathRelativePath.resolve(path.name)) } } } diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala index 4e18e81b54b..b5205e99a32 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderFactorySpec.scala @@ -19,7 +19,7 @@ class DrsPathBuilderFactorySpec extends AnyFlatSpec with CromwellTimeoutSpec wit | class = "cromwell.filesystems.drs.DrsFileSystemConfig" | config { | martha { - | url = "http://matha-url" + | url = "http://martha-url" | } | } | } diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala index ff5b2ecbf6e..990f9eb8b5f 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsResolverSpec.scala @@ -41,6 +41,18 @@ class DrsResolverSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileName) } + it should "find DRS path from a localization path" in { + val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingWithLocalizationPath).get.asInstanceOf[DrsPath] + + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileNameFromLocalizationPath) + } + + it should "find DRS path from all the paths" in { + val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingWithAllThePaths).get.asInstanceOf[DrsPath] + + DrsResolver.getContainerRelativePath(drsPath).unsafeRunSync() should be (MockDrsPaths.gcsRelativePathWithFileNameFromAllThePaths) + } + it should "throw GcsUrlNotFoundException when DRS path doesn't resolve to at least one GCS url" in { val drsPath = drsPathBuilder.build(MockDrsPaths.drsPathResolvingToNoGcsPath).get.asInstanceOf[DrsPath] diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala index 79a55bf1674..76887905387 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala @@ -1,13 +1,13 @@ package cromwell.filesystems.gcs -import java.io.FileNotFoundException - import akka.http.scaladsl.model.StatusCodes import cats.effect.IO import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.cloud.storage.StorageException import cromwell.filesystems.gcs.RequesterPaysErrors.isProjectNotProvidedError +import java.io.FileNotFoundException + object GcsEnhancedRequest { // If the request fails because no project was passed, recover the request, this time setting the project @@ -19,11 +19,11 @@ object GcsEnhancedRequest { // Use NoSuchFileException for better error reporting case e: StorageException if e.getCode == StatusCodes.NotFound.intValue => IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) - case e: GoogleJsonResponseException if isProjectNotProvidedError(e) => + case e: GoogleJsonResponseException if isProjectNotProvidedError(e) => IO(f(true)) case e: GoogleJsonResponseException if e.getStatusCode == StatusCodes.NotFound.intValue => IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}")) - case e => + case e => IO.raiseError(e) }) } diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala index 4a2165d47e5..12b06b01ea5 100644 --- a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala +++ b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala @@ -2,6 +2,7 @@ package cromwell.filesystems.gcs import com.google.api.client.googleapis.json.{GoogleJsonError, GoogleJsonResponseException} import com.google.cloud.storage.StorageException +import org.apache.commons.lang3.StringUtils object RequesterPaysErrors { val BucketIsRequesterPaysErrorCode = 400 @@ -9,15 +10,15 @@ object RequesterPaysErrors { val DoesNotHaveServiceUsePermissionErrorCode = 403 val DoesNotHaveServiceUsePermissionErrorMessage = "does not have serviceusage.services.use" - def isProjectNotProvidedError(storageException: StorageException) = + def isProjectNotProvidedError(storageException: StorageException) = storageException.getCode == BucketIsRequesterPaysErrorCode && - storageException.getMessage == BucketIsRequesterPaysErrorMessage + StringUtils.contains(storageException.getMessage, BucketIsRequesterPaysErrorMessage) def isProjectNotProvidedError(googleJsonError: GoogleJsonError) = googleJsonError.getCode == BucketIsRequesterPaysErrorCode && - googleJsonError.getMessage == BucketIsRequesterPaysErrorMessage + StringUtils.contains(googleJsonError.getMessage, BucketIsRequesterPaysErrorMessage) def isProjectNotProvidedError(googleJsonError: GoogleJsonResponseException) = googleJsonError.getStatusCode == BucketIsRequesterPaysErrorCode && - googleJsonError.getContent == BucketIsRequesterPaysErrorMessage + StringUtils.contains(googleJsonError.getContent, BucketIsRequesterPaysErrorMessage) } diff --git a/processes/release_processes/README.MD b/processes/release_processes/README.MD index d05bbf243a1..ab23d1f37a6 100644 --- a/processes/release_processes/README.MD +++ b/processes/release_processes/README.MD @@ -15,8 +15,8 @@ will need to be on the Broad internal network or VPN to open the following links 1. Tests for various backends supported by Cromwell. Log into Jenkins [here](https://fc-jenkins.dsp-techops.broadinstitute.org), check the tests [here](https://fc-jenkins.dsp-techops.broadinstitute.org/job/cromwell-cron-parent/). 1. Tests for Cromwell in Terra environment. Log into Jenkins [here](https://fc-jenkins.dsp-techops.broadinstitute.org), check the tests [here](https://fc-jenkins.dsp-techops.broadinstitute.org/view/Batch/). 1. [Run the publish script to create a new version of Cromwell](#how-to-publish-a-new-cromwell-version) -1. [Run through the "How to Release Cromwell into Firecloud" process](#how-to-release-cromwell-into-firecloud) -1. [Run through the "How to Deploy Cromwell in CAAS prod" process](#how-to-deploy-cromwell-in-caas-prod) +1. [Run through the "How to Release Cromwell into Firecloud/Terra" process](#how-to-release-cromwell-into-firecloud--terra) +1. [Run through the "How to Deploy Cromwell in CaaS prod" process](#how-to-deploy-cromwell-in-caas-staging-and-caas-prod) ### How to publish a new Cromwell version @@ -40,7 +40,6 @@ The release WDL uses a github token to perform actions on your behalf. This is optional, but I find it useful. Make or copy the following files into some temporary `releases/` directory: -* A cromwell jar file, preferably the most recent Cromwell version. * A copy of the workflow file to run (https://github.com/broadinstitute/cromwell/blob/develop/publish/publish_workflow.wdl) * An inputs json like this: @@ -56,8 +55,8 @@ This is optional, but I find it useful. Make or copy the following files into so #### Make sure Docker will have enough memory -I had to follow the instructions [here](https://docs.docker.com/docker-for-mac/#resources) to increase my Docker memory. -I chose to increase it from 2GB to 8GB; 4GB is not sufficient. +Follow the instructions [here](https://docs.docker.com/docker-for-mac/#resources) to increase Docker memory. +Ensure you have at least 8GB; 4GB is not sufficient. #### Let people know the publish is underway @@ -66,12 +65,13 @@ the release is published. #### Run the `publish_workflow.wdl` Workflow -* Run the Cromwell instance using the Local backend. The publish workflow is quite resource intensive; it's a good idea to - shut down other resource intensive apps before launching it to avoid painfully slow or failed executions. - * In server mode with a persistent backing database is probably a good idea - it will allow call caching to happen if you need to restart for any reason. - Some instructions for using a Dockerized MySQL server and CI config [here](#cromwell-setup-for-publishing). +Run Cromwell in server mode with a persistent backing database, using Docker containers. This allows call caching to happen if you need to restart for any reason. +See instructions for using a Dockerized MySQL server and CI config [here](#cromwell-setup-for-publishing). -* Submit the workflow to Cromwell along with the inputs file. +Note that the publish workflow is quite resource intensive; it's a good idea to +shut down other resource intensive apps before launching it to avoid painfully slow or failed executions. + +Using the Swagger API, submit the workflow to Cromwell along with the inputs file. #### Make sure it all went swimmingly @@ -83,25 +83,27 @@ the release is published. ### How to Release Cromwell into Firecloud / Terra -**Note:** If the Cromwell CHANGELOG indicates that the upgrade might take some time (e.g. because of a database migration), checking in with the release engineer +**Note:** If the Cromwell CHANGELOG indicates that the upgrade might take some time (e.g., because of a database migration), checking in with the release engineer and user support/comms to let them know that the upgrade may involve downtime is also required. You may need to help draft an impact statement and co-ordinate timing the deploy to make sure user impact in minimized. -**Note:** How to accomplish some of these steps might be non-obvious to you (e.g. generating the release notes). +**Note:** How to accomplish some of these steps might be non-obvious to you (e.g., generating the release notes). If so, refer to the additional details in the [full document](https://docs.google.com/document/d/1EEzwemE8IedCplIwL506fiqXr0262Pz4G0x6Cr6V-5E). ![firecloud-develop](firecloud-develop.dot.png) ### How to Deploy Cromwell in CaaS staging and CaaS prod -**Note:** If the Cromwell CHANGELOG indicates that the upgrade might take some time (eg because of a database migration), checking in with the CaaS users +CaaS is "Cromwell as a Service". It is used by a couple of Broad teams (Pipelines and Epigenomics), though the long-term plan is for those teams to migrate to using Terra. + +**Note:** If the Cromwell CHANGELOG indicates that the upgrade might take some time (e.g., because of a database migration), checking in with the CaaS users to let them know that the upgrade is about to happen is a good idea. -Deploying to CAAS is detailed in the [Quick CAAS Deployment Guide](https://docs.google.com/document/d/1s0YC-oohJ7o-OGcgnH_-YBtIEKmLIPTRpG36yvWxUpE) +Deploying to CaaS is detailed in the [Quick CaaS Deployment Guide](https://docs.google.com/document/d/1s0YC-oohJ7o-OGcgnH_-YBtIEKmLIPTRpG36yvWxUpE) ## Bonus Processes -The swagger client library is not part of our core publish/release process but can be performed from time to time, as required. +The Swagger client library is not part of our core publish/release process but can be performed from time to time, as required. ### How to Generate and Publish Swagger Client Library diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 66adf0b41b5..b30cdacb45d 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -11,6 +11,11 @@ object Dependencies { private val ammoniteOpsV = "2.4.0" private val apacheHttpClientV = "4.5.13" private val awsSdkV = "2.17.29" + // We would like to use the BOM to manage Azure SDK versions, but SBT doesn't support it. + // https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/boms/azure-sdk-bom + // https://github.com/sbt/sbt/issues/4531 + private val azureIdentitySdkV = "1.4.0" + private val azureKeyVaultSdkV = "4.3.4" private val betterFilesV = "3.9.1" /* cats-effect, fs2, http4s, and sttp (also to v3) should all be upgraded at the same time to use cats-effect 3.x. @@ -242,10 +247,7 @@ object Dependencies { ) private val liquibaseDependencies = List( - "org.liquibase" % "liquibase-core" % liquibaseV, - // This is to stop liquibase from being so noisy by default - // See: http://stackoverflow.com/questions/20880783/how-to-get-liquibase-to-log-using-slf4j - "com.mattbertolini" % "liquibase-slf4j" % liquibaseSlf4jV + "org.liquibase" % "liquibase-core" % liquibaseV ) private val akkaDependencies = List( @@ -307,7 +309,6 @@ object Dependencies { "cloudwatchlogs", "s3", "sts", - "ecs" ).map(artifactName => "software.amazon.awssdk" % artifactName % awsSdkV) private val googleCloudDependencies = List( @@ -515,7 +516,16 @@ object Dependencies { val bcsBackendDependencies: List[ModuleID] = commonDependencies ++ refinedTypeDependenciesList ++ aliyunBatchComputeDependencies - val tesBackendDependencies: List[ModuleID] = akkaHttpDependencies + val tesBackendAzureDependencies: List[ModuleID] = List( + "com.azure" % "azure-identity" % azureIdentitySdkV + exclude("jakarta.xml.bind", "jakarta.xml.bind-api") + exclude("jakarta.activation", "jakarta.activation-api"), + "com.azure" % "azure-security-keyvault-secrets" % azureKeyVaultSdkV + exclude("jakarta.xml.bind", "jakarta.xml.bind-api") + exclude("jakarta.activation", "jakarta.activation-api") + ) + + val tesBackendDependencies: List[ModuleID] = tesBackendAzureDependencies ++ akkaHttpDependencies val sfsBackendDependencies = List ( "org.lz4" % "lz4-java" % lz4JavaV @@ -547,7 +557,8 @@ object Dependencies { "com.google.cloud" % "google-cloud-storage" % googleCloudStorageV, "org.typelevel" %% "cats-effect" % catsEffectV, "com.iheart" %% "ficus" % ficusV, - "com.softwaremill.sttp" %% "circe" % sttpV + "com.softwaremill.sttp" %% "circe" % sttpV, + "com.github.scopt" %% "scopt" % scoptV, ) ++ circeDependencies ++ catsDependencies ++ slf4jBindingDependencies ++ languageFactoryDependencies val allProjectDependencies: List[ModuleID] = diff --git a/project/Version.scala b/project/Version.scala index 96f52ce5bd9..98751b51319 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -5,7 +5,7 @@ import sbt._ object Version { // Upcoming release, or current if we're on a master / hotfix branch - val cromwellVersion = "70" + val cromwellVersion = "71" /** * Returns true if this project should be considered a snapshot. diff --git a/runConfigurations/Cromwell_server.run.xml b/runConfigurations/Cromwell_server.run.xml index 3cd92753bfa..9baf9832a94 100644 --- a/runConfigurations/Cromwell_server.run.xml +++ b/runConfigurations/Cromwell_server.run.xml @@ -8,6 +8,7 @@ +