diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl b/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl index 56d89094921..dc84d2e346b 100644 --- a/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl +++ b/centaur/src/main/resources/standardTestCases/drs_tests/wf_level_file_size.wdl @@ -1,8 +1,8 @@ version 1.0 workflow wf_level_file_size { - File input1 = "dos://wb-mock-drs-dev.storage.googleapis.com/4a3908ad-1f0b-4e2a-8a92-611f2123e8b0" - File input2 = "dos://wb-mock-drs-dev.storage.googleapis.com/0c8e7bc6-fd76-459d-947b-808b0605beb3" + File input1 = "drs://wb-mock-drs-dev.storage.googleapis.com/4a3908ad-1f0b-4e2a-8a92-611f2123e8b0" + File input2 = "drs://wb-mock-drs-dev.storage.googleapis.com/0c8e7bc6-fd76-459d-947b-808b0605beb3" output { Float fileSize1 = size(input1) diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_hca/drs_usa_hca.inputs b/centaur/src/main/resources/standardTestCases/drs_usa_hca/drs_usa_hca.inputs index f528e7d2364..f82844519f9 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_hca/drs_usa_hca.inputs +++ b/centaur/src/main/resources/standardTestCases/drs_usa_hca/drs_usa_hca.inputs @@ -1,8 +1,8 @@ { # For all below 5 HCA uuids, Martha does not return a service account - "drs_usa_hca.localize_drs_with_usa.file1": "dos://service.staging.explore.data.humancellatlas.org/033c9840-c5cd-438b-b0e4-8e4cd8fc8dc6?version=2019-07-04T104122.106166Z", - "drs_usa_hca.localize_drs_with_usa.file2": "dos://service.staging.explore.data.humancellatlas.org/4defa7b0-46c2-4053-8e99-b827eed1bc96?version=2019-07-04T104122.100969Z", - "drs_usa_hca.localize_drs_with_usa.file3": "dos://service.staging.explore.data.humancellatlas.org/de5dcfc1-5aea-41ba-a7ae-e72c416cb450?version=2019-07-04T104122.092788Z", - "drs_usa_hca.localize_drs_with_usa.file4": "dos://service.staging.explore.data.humancellatlas.org/16dea2c5-e2bd-45bc-b2fd-fcac0daafc48?version=2019-07-04T104122.060634Z", - "drs_usa_hca.localize_drs_with_usa.file5": "dos://service.dev.explore.data.humancellatlas.org/7c800467-9143-402f-b965-4e7cad75c1e6?version=2019-05-26T130511.722646Z" + "drs_usa_hca.localize_drs_with_usa.file1": "drs://service.staging.explore.data.humancellatlas.org/033c9840-c5cd-438b-b0e4-8e4cd8fc8dc6?version=2019-07-04T104122.106166Z", + "drs_usa_hca.localize_drs_with_usa.file2": "drs://service.staging.explore.data.humancellatlas.org/4defa7b0-46c2-4053-8e99-b827eed1bc96?version=2019-07-04T104122.100969Z", + "drs_usa_hca.localize_drs_with_usa.file3": "drs://service.staging.explore.data.humancellatlas.org/de5dcfc1-5aea-41ba-a7ae-e72c416cb450?version=2019-07-04T104122.092788Z", + "drs_usa_hca.localize_drs_with_usa.file4": "drs://service.staging.explore.data.humancellatlas.org/16dea2c5-e2bd-45bc-b2fd-fcac0daafc48?version=2019-07-04T104122.060634Z", + "drs_usa_hca.localize_drs_with_usa.file5": "drs://service.dev.explore.data.humancellatlas.org/7c800467-9143-402f-b965-4e7cad75c1e6?version=2019-05-26T130511.722646Z" } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala index 1daf2da9893..2f36374212a 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala @@ -33,13 +33,13 @@ class DrsCloudNioFileSystemProvider(rootConfig: Config, override def isTransient(exception: Exception): Boolean = false - override def getScheme: String = "dos" + override def getScheme: String = "drs" override def getHost(uriAsString: String): String = { require(uriAsString.startsWith(s"$getScheme://"), s"Scheme does not match $getScheme") /* - * In some cases for a URI, the host name is null. For example, for DRS urls like 'dos://dg.123/123-123-123', + * In some cases for a URI, the host name is null. For example, for DRS urls like 'drs://dg.123/123-123-123', * even though 'dg.123' is a valid host, somehow since it does not conform to URI's standards, uri.getHost returns null. In such * cases, authority is used instead of host. If there is no authority, use an empty string. */ diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala index d4fe08e4655..1a9ec260059 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala @@ -24,7 +24,7 @@ class DrsCloudNioRegularFileAttributes(drsPath: String, drsPathResolver: DrsPath override def fileHash: Option[String] = { drsPathResolver.resolveDrsThroughMartha(drsPath).map(marthaResponse => { - marthaResponse.dos.data_object.checksums.flatMap { + marthaResponse.drs.data_object.checksums.flatMap { _.collectFirst{ case c if c.`type`.equalsIgnoreCase("md5") => c.checksum } } }).unsafeRunSync() @@ -34,7 +34,7 @@ class DrsCloudNioRegularFileAttributes(drsPath: String, drsPathResolver: DrsPath override def lastModifiedTime(): FileTime = { val lastModifiedIO = for { marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath) - lastModifiedInString <- IO.fromEither(marthaResponse.dos.data_object.updated.toRight(throwRuntimeException("updated"))) + lastModifiedInString <- IO.fromEither(marthaResponse.drs.data_object.updated.toRight(throwRuntimeException("updated"))) lastModified <- convertToFileTime(lastModifiedInString) } yield lastModified @@ -45,7 +45,7 @@ class DrsCloudNioRegularFileAttributes(drsPath: String, drsPathResolver: DrsPath override def size(): Long = { val sizeIO = for { marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath) - size <- IO.fromEither(marthaResponse.dos.data_object.size.toRight(throwRuntimeException("size"))) + size <- IO.fromEither(marthaResponse.drs.data_object.size.toRight(throwRuntimeException("size"))) } yield size sizeIO.unsafeRunSync() diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index cc8bbf7dd26..3adaed3152e 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -19,10 +19,14 @@ case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBu implicit lazy val urlDecoder: Decoder[Url] = deriveDecoder implicit lazy val checksumDecoder: Decoder[ChecksumObject] = deriveDecoder - implicit lazy val dataObjectDecoder: Decoder[DosDataObject] = deriveDecoder - implicit lazy val dosObjectDecoder: Decoder[DosObject] = deriveDecoder + implicit lazy val dataObjectDecoder: Decoder[DrsDataObject] = deriveDecoder + implicit lazy val drsObjectDecoder: Decoder[DrsObject] = deriveDecoder implicit lazy val saDataObjectDecoder: Decoder[SADataObject] = deriveDecoder - implicit lazy val marthaResponseDecoder: Decoder[MarthaResponse] = deriveDecoder + // Martha is still returning objects keyed by the obsolete "dos" terminology rather than the current term "drs". + // In order to avoid having Cromwell's case classes use the obsolete terminology that would arise from a derived + // decoder, this `forProduct2` construct instructs Circe to take the value keyed by `dos` and pass that as the + // first argument to `MarthaResponse.apply`, which happens to be the constructor parameter formally named `drs`. + implicit lazy val marthaResponseDecoder: Decoder[MarthaResponse] = Decoder.forProduct2("dos", "googleServiceAccount")(MarthaResponse.apply) private val DrsPathToken = "${drsPath}" @@ -48,7 +52,7 @@ case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBu e => IO.raiseError(new RuntimeException(s"Failed to parse response from Martha into a case class. Error: ${ExceptionUtils.getMessage(e)}")) } } - + private def executeMarthaRequest(httpPost: HttpPost): Resource[IO, HttpResponse]= { for { httpClient <- Resource.fromAutoCloseable(IO(httpClientBuilder.build())) @@ -78,13 +82,13 @@ case class Url(url: String) case class ChecksumObject(checksum: String, `type`: String) -case class DosDataObject(size: Option[Long], +case class DrsDataObject(size: Option[Long], checksums: Option[Array[ChecksumObject]], updated: Option[String], urls: Array[Url]) -case class DosObject(data_object: DosDataObject) +case class DrsObject(data_object: DrsDataObject) case class SADataObject(data: Json) -case class MarthaResponse(dos: DosObject, googleServiceAccount: Option[SADataObject]) +case class MarthaResponse(drs: DrsObject, googleServiceAccount: Option[SADataObject]) diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala index 450532cf6a9..f52b8e5d562 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala @@ -61,7 +61,7 @@ object DrsLocalizerMain extends IOApp { marthaResponse <- resolveDrsThroughMartha(drsUrl, marthaUri) _ = httpBackendConnection.close() // Currently Martha only supports resolving DRS paths to GCS paths - gcsUrl <- extractFirstGcsUrl(marthaResponse.dos.data_object.urls) + gcsUrl <- extractFirstGcsUrl(marthaResponse.drs.data_object.urls) exitState <- downloadFileFromGcs(gcsUrl, marthaResponse.googleServiceAccount.map(_.data.toString), downloadLoc, requesterPaysId) } yield exitState diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/MarthaResponse.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/MarthaResponse.scala index 3d907978e5f..be172bae181 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/MarthaResponse.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/MarthaResponse.scala @@ -6,10 +6,14 @@ import io.circe.generic.semiauto.deriveDecoder object MarthaResponseJsonSupport { implicit val urlFormat: Decoder[Url] = deriveDecoder - implicit val dataObject: Decoder[DosDataObject] = deriveDecoder - implicit val dosObjectFormat: Decoder[DosObject] = deriveDecoder + implicit val dataObject: Decoder[DrsDataObject] = deriveDecoder + implicit val drsObjectFormat: Decoder[DrsObject] = deriveDecoder implicit val googleServiceAccountFormat: Decoder[GoogleServiceAccount] = deriveDecoder - implicit val marthaResponseFormat: Decoder[MarthaResponse] = deriveDecoder + // Martha is still returning objects keyed by the obsolete "dos" terminology rather than the current term "drs". + // In order to avoid having Cromwell's case classes use the obsolete terminology that would arise from a derived + // decoder, this `forProduct2` construct instructs Circe to take the value keyed by `dos` and pass that as the + // first argument to `MarthaResponse.apply`, which happens to be the constructor parameter formally named `drs`. + implicit val marthaResponseFormat: Decoder[MarthaResponse] = Decoder.forProduct2("dos", "googleServiceAccount")(MarthaResponse.apply) implicit val samErrorResponseFormat: Decoder[SamErrorResponse] = deriveDecoder implicit val samErrorResponseCodeFormat: Decoder[SamErrorResponseCode] = deriveDecoder @@ -17,10 +21,10 @@ object MarthaResponseJsonSupport { } case class Url(url: String) -case class DosDataObject(urls: Array[Url]) -case class DosObject(data_object: DosDataObject) +case class DrsDataObject(urls: Array[Url]) +case class DrsObject(data_object: DrsDataObject) case class GoogleServiceAccount(data: Json) -case class MarthaResponse(dos: DosObject, googleServiceAccount: Option[GoogleServiceAccount]) +case class MarthaResponse(drs: DrsObject, googleServiceAccount: Option[GoogleServiceAccount]) case class SamErrorResponse(text: String) diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala index 7d9cbbaeffa..4f93e3ec7c3 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala @@ -87,7 +87,7 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single for { serviceAccountJson <- serviceAccountJsonIo //Currently, Martha only supports resolving DRS paths to GCS paths - url <- IO.fromEither(DrsResolver.extractUrlForScheme(marthaResponse.dos.data_object.urls, GcsScheme)) + url <- IO.fromEither(DrsResolver.extractUrlForScheme(marthaResponse.drs.data_object.urls, GcsScheme)) readableByteChannel <- inputReadChannel(url, GcsScheme, serviceAccountJson, requesterPaysProjectIdOption) } yield readableByteChannel } diff --git a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala index 875bb106998..f84cfc1eb90 100644 --- a/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala +++ b/filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsResolver.scala @@ -31,7 +31,7 @@ object DrsResolver { drsFileSystemProvider <- toIO(drsFileSystemProviderOption, noFileSystemForDrsError) marthaResponse <- drsFileSystemProvider.drsPathResolver.resolveDrsThroughMartha(drsPath.pathAsString) //Currently, Martha only supports resolving DRS paths to GCS paths - relativePath <- IO.fromEither(extractUrlForScheme(marthaResponse.dos.data_object.urls, GcsScheme)) + relativePath <- IO.fromEither(extractUrlForScheme(marthaResponse.drs.data_object.urls, GcsScheme)) .map(_.substring(urlProtocolLength(GcsScheme))) .handleErrorWith(e => IO.raiseError(new RuntimeException(s"Error while resolving DRS path: $drsPath. Error: ${ExceptionUtils.getMessage(e)}"))) } yield relativePath diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala index ae8cfa442c1..ee93ff45581 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/DrsPathBuilderSpec.scala @@ -28,12 +28,12 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi private def pathsToTruncate = Table( ("context", "file", "relative"), - ("dos://bucket/path/to/my/dir", "dos://bucket/path/to/my/dir/file", "file"), - ("dos://bucket/path/to/my/dir", "dos://bucket/path/to/my/dir//file", "file"), - ("dos://bucket/path/to/my//dir", "dos://bucket/path/to/my/dir/file", "dir/file"), - ("dos://bucket/path/to/my//dir", "dos://bucket/path/to/my/dir//file", "dir//file"), - ("dos://bucket/path/to/my/dir", "dos://bucket/path/./to/my/dir/file", "./to/my/dir/file"), - ("dos://bucket/path/to/my/dir/with/file", "dos://bucket/path/to/other/dir/with/file", "other/dir/with/file") + ("drs://bucket/path/to/my/dir", "drs://bucket/path/to/my/dir/file", "file"), + ("drs://bucket/path/to/my/dir", "drs://bucket/path/to/my/dir//file", "file"), + ("drs://bucket/path/to/my//dir", "drs://bucket/path/to/my/dir/file", "dir/file"), + ("drs://bucket/path/to/my//dir", "drs://bucket/path/to/my/dir//file", "dir//file"), + ("drs://bucket/path/to/my/dir", "drs://bucket/path/./to/my/dir/file", "./to/my/dir/file"), + ("drs://bucket/path/to/my/dir/with/file", "drs://bucket/path/to/other/dir/with/file", "other/dir/with/file") ) private def bucket = "mymadeupbucket" @@ -41,71 +41,71 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi private def goodPaths = Seq( GoodPath( description = "a path with spaces", - path = s"dos://$bucket/hello/world/with spaces", + path = s"drs://$bucket/hello/world/with spaces", normalize = false, - pathAsString = s"dos://$bucket/hello/world/with spaces", + pathAsString = s"drs://$bucket/hello/world/with spaces", pathWithoutScheme = s"$bucket/hello/world/with spaces", - parent = s"dos://$bucket/hello/world/", - getParent = s"dos://$bucket/hello/world/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/world/", + getParent = s"drs://$bucket/hello/world/", + root = s"drs://$bucket/", name = "with spaces", - getFileName = s"dos://$bucket/with spaces", + getFileName = s"drs://$bucket/with spaces", getNameCount = 3, isAbsolute = true), GoodPath( description = "a path with non-ascii", - path = s"dos://$bucket/hello/world/with non ascii £€", + path = s"drs://$bucket/hello/world/with non ascii £€", normalize = false, - pathAsString = s"dos://$bucket/hello/world/with non ascii £€", + pathAsString = s"drs://$bucket/hello/world/with non ascii £€", pathWithoutScheme = s"$bucket/hello/world/with non ascii £€", - parent = s"dos://$bucket/hello/world/", - getParent = s"dos://$bucket/hello/world/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/world/", + getParent = s"drs://$bucket/hello/world/", + root = s"drs://$bucket/", name = "with non ascii £€", - getFileName = s"dos://$bucket/with non ascii £€", + getFileName = s"drs://$bucket/with non ascii £€", getNameCount = 3, isAbsolute = true), GoodPath( description = "a gs uri path with encoded characters", - path = s"dos://$bucket/hello/world/encoded%20spaces", + path = s"drs://$bucket/hello/world/encoded%20spaces", normalize = false, - pathAsString = s"dos://$bucket/hello/world/encoded%20spaces", + pathAsString = s"drs://$bucket/hello/world/encoded%20spaces", pathWithoutScheme = s"$bucket/hello/world/encoded%20spaces", - parent = s"dos://$bucket/hello/world/", - getParent = s"dos://$bucket/hello/world/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/world/", + getParent = s"drs://$bucket/hello/world/", + root = s"drs://$bucket/", name = "encoded%20spaces", - getFileName = s"dos://$bucket/encoded%20spaces", + getFileName = s"drs://$bucket/encoded%20spaces", getNameCount = 3, isAbsolute = true), GoodPath( description = "a file at the top of the bucket", - path = s"dos://$bucket/hello", + path = s"drs://$bucket/hello", normalize = false, - pathAsString = s"dos://$bucket/hello", + pathAsString = s"drs://$bucket/hello", pathWithoutScheme = s"$bucket/hello", - parent = s"dos://$bucket/", - getParent = s"dos://$bucket/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/", + getParent = s"drs://$bucket/", + root = s"drs://$bucket/", name = "hello", - getFileName = s"dos://$bucket/hello", + getFileName = s"drs://$bucket/hello", getNameCount = 1, isAbsolute = true), GoodPath( description = "a path ending in /", - path = s"dos://$bucket/hello/world/", + path = s"drs://$bucket/hello/world/", normalize = false, - pathAsString = s"dos://$bucket/hello/world", + pathAsString = s"drs://$bucket/hello/world", pathWithoutScheme = s"$bucket/hello/world", - parent = s"dos://$bucket/hello/", - getParent = s"dos://$bucket/hello/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/", + getParent = s"drs://$bucket/hello/", + root = s"drs://$bucket/", name = "world", - getFileName = s"dos://$bucket/world", + getFileName = s"drs://$bucket/world", getNameCount = 2, isAbsolute = true), @@ -113,57 +113,57 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi GoodPath( description = "a bucket with a path .", - path = s"dos://$bucket/.", + path = s"drs://$bucket/.", normalize = false, - pathAsString = s"dos://$bucket/.", + pathAsString = s"drs://$bucket/.", pathWithoutScheme = s"$bucket/.", parent = null, - getParent = s"dos://$bucket/", - root = s"dos://$bucket/", + getParent = s"drs://$bucket/", + root = s"drs://$bucket/", name = "", - getFileName = s"dos://$bucket/.", + getFileName = s"drs://$bucket/.", getNameCount = 1, isAbsolute = true), GoodPath( description = "a bucket with a path ..", - path = s"dos://$bucket/..", + path = s"drs://$bucket/..", normalize = false, - pathAsString = s"dos://$bucket/..", + pathAsString = s"drs://$bucket/..", pathWithoutScheme = s"$bucket/..", parent = null, - getParent = s"dos://$bucket/", + getParent = s"drs://$bucket/", root = null, name = "", - getFileName = s"dos://$bucket/..", + getFileName = s"drs://$bucket/..", getNameCount = 1, isAbsolute = true), GoodPath( description = "a bucket including . in the path", - path = s"dos://$bucket/hello/./world", + path = s"drs://$bucket/hello/./world", normalize = false, - pathAsString = s"dos://$bucket/hello/./world", + pathAsString = s"drs://$bucket/hello/./world", pathWithoutScheme = s"$bucket/hello/./world", - parent = s"dos://$bucket/hello/", - getParent = s"dos://$bucket/hello/./", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/", + getParent = s"drs://$bucket/hello/./", + root = s"drs://$bucket/", name = "world", - getFileName = s"dos://$bucket/world", + getFileName = s"drs://$bucket/world", getNameCount = 3, isAbsolute = true), GoodPath( description = "a bucket including .. in the path", - path = s"dos://$bucket/hello/../world", + path = s"drs://$bucket/hello/../world", normalize = false, - pathAsString = s"dos://$bucket/hello/../world", + pathAsString = s"drs://$bucket/hello/../world", pathWithoutScheme = s"$bucket/hello/../world", - parent = s"dos://$bucket/", - getParent = s"dos://$bucket/hello/../", - root = s"dos://$bucket/", + parent = s"drs://$bucket/", + getParent = s"drs://$bucket/hello/../", + root = s"drs://$bucket/", name = "world", - getFileName = s"dos://$bucket/world", + getFileName = s"drs://$bucket/world", getNameCount = 3, isAbsolute = true), @@ -171,13 +171,13 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi GoodPath( description = "a bucket with a normalized path .", - path = s"dos://$bucket/.", + path = s"drs://$bucket/.", normalize = true, - pathAsString = s"dos://$bucket/", + pathAsString = s"drs://$bucket/", pathWithoutScheme = s"$bucket/", parent = null, getParent = null, - root = s"dos://$bucket/", + root = s"drs://$bucket/", name = "", getFileName = null, getNameCount = 0, @@ -185,13 +185,13 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi GoodPath( description = "a bucket with a normalized path ..", - path = s"dos://$bucket/..", + path = s"drs://$bucket/..", normalize = true, - pathAsString = s"dos://$bucket/", + pathAsString = s"drs://$bucket/", pathWithoutScheme = s"$bucket/", parent = null, getParent = null, - root = s"dos://$bucket/", + root = s"drs://$bucket/", name = "", getFileName = null, getNameCount = 1, @@ -199,83 +199,83 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi GoodPath( description = "a bucket including . in the normalized path", - path = s"dos://$bucket/hello/./world", + path = s"drs://$bucket/hello/./world", normalize = true, - pathAsString = s"dos://$bucket/hello/world", + pathAsString = s"drs://$bucket/hello/world", pathWithoutScheme = s"$bucket/hello/world", - parent = s"dos://$bucket/hello/", - getParent = s"dos://$bucket/hello/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/hello/", + getParent = s"drs://$bucket/hello/", + root = s"drs://$bucket/", name = "world", - getFileName = s"dos://$bucket/world", + getFileName = s"drs://$bucket/world", getNameCount = 2, isAbsolute = true), GoodPath( description = "a bucket including .. in the normalized path", - path = s"dos://$bucket/hello/../world", + path = s"drs://$bucket/hello/../world", normalize = true, - pathAsString = s"dos://$bucket/world", + pathAsString = s"drs://$bucket/world", pathWithoutScheme = s"$bucket/world", - parent = s"dos://$bucket/", - getParent = s"dos://$bucket/", - root = s"dos://$bucket/", + parent = s"drs://$bucket/", + getParent = s"drs://$bucket/", + root = s"drs://$bucket/", name = "world", - getFileName = s"dos://$bucket/world", + getFileName = s"drs://$bucket/world", getNameCount = 1, isAbsolute = true), GoodPath( description = "a bucket with an underscore", - path = s"dos://hello_underscore/world", + path = s"drs://hello_underscore/world", normalize = true, - pathAsString = s"dos://hello_underscore/world", + pathAsString = s"drs://hello_underscore/world", pathWithoutScheme = s"hello_underscore/world", - parent = s"dos://hello_underscore/", - getParent = s"dos://hello_underscore/", - root = s"dos://hello_underscore/", + parent = s"drs://hello_underscore/", + getParent = s"drs://hello_underscore/", + root = s"drs://hello_underscore/", name = "world", - getFileName = s"dos://hello_underscore/world", + getFileName = s"drs://hello_underscore/world", getNameCount = 1, isAbsolute = true), GoodPath( description = "a bucket named .", - path = s"dos://./hello/world", + path = s"drs://./hello/world", normalize = true, - pathAsString = s"dos://./hello/world", + pathAsString = s"drs://./hello/world", pathWithoutScheme = s"./hello/world", - parent = s"dos://./hello/", - getParent = s"dos://./hello/", - root = s"dos://./", + parent = s"drs://./hello/", + getParent = s"drs://./hello/", + root = s"drs://./", name = "world", - getFileName = s"dos://./world", + getFileName = s"drs://./world", getNameCount = 2, isAbsolute = true), GoodPath( description = "a non ascii bucket name", - path = s"dos://nonasciibucket£€/hello/world", + path = s"drs://nonasciibucket£€/hello/world", normalize = true, - pathAsString = s"dos://nonasciibucket£€/hello/world", + pathAsString = s"drs://nonasciibucket£€/hello/world", pathWithoutScheme = s"nonasciibucket£€/hello/world", - parent = s"dos://nonasciibucket£€/hello/", - getParent = s"dos://nonasciibucket£€/hello/", - root = s"dos://nonasciibucket£€/", + parent = s"drs://nonasciibucket£€/hello/", + getParent = s"drs://nonasciibucket£€/hello/", + root = s"drs://nonasciibucket£€/", name = "world", - getFileName = s"dos://nonasciibucket£€/world", + getFileName = s"drs://nonasciibucket£€/world", getNameCount = 2, isAbsolute = true), GoodPath( description = "an non-absolute path without a host", - path = s"dos://blah/", + path = s"drs://blah/", normalize = false, - pathAsString = s"dos://blah/", + pathAsString = s"drs://blah/", pathWithoutScheme = s"blah/", parent = null, getParent = null, - root = s"dos://blah/", + root = s"drs://blah/", name = "", getFileName = null, getNameCount = 1, @@ -283,13 +283,13 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi GoodPath( description = "an absolute path without a host", - path = s"dos://blah", + path = s"drs://blah", normalize = false, - pathAsString = s"dos://blah/", + pathAsString = s"drs://blah/", pathWithoutScheme = s"blah/", parent = null, getParent = null, - root = s"dos://blah/", + root = s"drs://blah/", name = "", getFileName = null, getNameCount = 1, @@ -297,13 +297,13 @@ class DrsPathBuilderSpec extends TestKitSuite with FlatSpecLike with Matchers wi ) private def badPaths = Seq( - BadPath("an empty path", "", " does not have a dos scheme."), - BadPath("a GCS path", s"gs://$bucket/hello/world", "gs://mymadeupbucket/hello/world does not have a dos scheme."), - BadPath("an bucketless path", "dos://", "Expected authority at index 6: dos://"), - BadPath("a https path", "https://hello/world", "https://hello/world does not have a dos scheme."), - BadPath("a file uri path", "file:///hello/world", "file:///hello/world does not have a dos scheme."), - BadPath("a relative file path", "hello/world", "hello/world does not have a dos scheme."), - BadPath("an absolute file path", "/hello/world", "/hello/world does not have a dos scheme."), + BadPath("an empty path", "", " does not have a drs scheme."), + BadPath("a GCS path", s"gs://$bucket/hello/world", "gs://mymadeupbucket/hello/world does not have a drs scheme."), + BadPath("an bucketless path", "drs://", "Expected authority at index 6: drs://"), + BadPath("a https path", "https://hello/world", "https://hello/world does not have a drs scheme."), + BadPath("a file uri path", "file:///hello/world", "file:///hello/world does not have a drs scheme."), + BadPath("a relative file path", "hello/world", "hello/world does not have a drs scheme."), + BadPath("an absolute file path", "/hello/world", "/hello/world does not have a drs scheme."), ) private def drsReadInterpreter(marthaResponse: MarthaResponse): IO[ReadableByteChannel] = diff --git a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/MockDrsPathResolver.scala b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/MockDrsPathResolver.scala index b52a576a9e4..a3e1b00c1b5 100644 --- a/filesystems/drs/src/test/scala/cromwell/filesystems/drs/MockDrsPathResolver.scala +++ b/filesystems/drs/src/test/scala/cromwell/filesystems/drs/MockDrsPathResolver.scala @@ -28,14 +28,14 @@ class MockDrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBui private def createMarthaResponse(urlArray: Array[Url]): IO[MarthaResponse] = { - val dosDataObject = DosDataObject( + val drsDataObject = DrsDataObject( size = Option(123), checksums = Option(Array(checksumObj)), updated = None, urls = urlArray ) - IO(MarthaResponse(DosObject(dosDataObject), Option(SADataObject(Json.fromString("{}"))))) + IO(MarthaResponse(DrsObject(drsDataObject), Option(SADataObject(Json.fromString("{}"))))) } override def resolveDrsThroughMartha(drsPath: String, serviceAccount: Option[String]): IO[MarthaResponse] = { @@ -62,7 +62,7 @@ class MockDrsCloudNioFileSystemProvider(config: Config, object MockDrsPaths { - private val drsPathPrefix = "dos://drs-host/" + private val drsPathPrefix = "drs://drs-host/" val drsPathResolvingToOneGcsPath = s"$drsPathPrefix/4d427aa3-5640-4f00-81ae-c33443f84acf" diff --git a/supportedBackends/google/pipelines/v2alpha1/src/test/resources/reference.conf b/supportedBackends/google/pipelines/v2alpha1/src/test/resources/reference.conf index 0d57f772594..dca01308ed6 100644 --- a/supportedBackends/google/pipelines/v2alpha1/src/test/resources/reference.conf +++ b/supportedBackends/google/pipelines/v2alpha1/src/test/resources/reference.conf @@ -1,2 +1,2 @@ -drs.localization.docker-image = "somerepo/dos-downloader:tagged" +drs.localization.docker-image = "somerepo/drs-downloader:tagged" drs.localization.command-template = "/path/to/some_executable before args ${drsPath} middle args ${containerPath} ends args" diff --git a/supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesConversionsSpec.scala b/supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesConversionsSpec.scala index 1cbead8ed00..21ee3561d2c 100644 --- a/supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesConversionsSpec.scala +++ b/supportedBackends/google/pipelines/v2alpha1/src/test/scala/cromwell/backend/google/pipelines/v2alpha1/PipelinesConversionsSpec.scala @@ -44,7 +44,7 @@ class PipelinesConversionsSpec extends FlatSpec with Matchers { new DrsCloudNioFileSystemProvider(marthaConfig, fakeCredentials, httpClientBuilder, drsReadInterpreter), None, ) - val drsPath = drsPathBuilder.build("dos://dos.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba").get + val drsPath = drsPathBuilder.build("drs://drs.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba").get val containerRelativePath = DefaultPathBuilder.get("path/to/file.bai") val mount = PipelinesApiWorkingDisk(DiskType.LOCAL, 1) val input = PipelinesApiFileInput("example", drsPath, containerRelativePath, mount) @@ -59,7 +59,7 @@ class PipelinesConversionsSpec extends FlatSpec with Matchers { logging.get("commands") should be(a[java.util.List[_]]) logging.get("commands").asInstanceOf[java.util.List[_]] should contain( """printf '%s %s\n' "$(date -u '+%Y/%m/%d %H:%M:%S')" """ + - """Localizing\ input\ dos://dos.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba\ """ + + """Localizing\ input\ drs://drs.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba\ """ + """-\>\ /cromwell_root/path/to/file.bai""" ) @@ -83,14 +83,14 @@ class PipelinesConversionsSpec extends FlatSpec with Matchers { action.get("commands") should be(a[java.util.List[_]]) action.get("commands").asInstanceOf[java.util.List[_]] should contain theSameElementsAs List( - "dos://dos.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba", + "drs://drs.example.org/aaaabbbb-cccc-dddd-eeee-abcd0000dcba", "/cromwell_root/path/to/file.bai" ) action.get("mounts") should be(a[java.util.List[_]]) action.get("mounts").asInstanceOf[java.util.List[_]] should be (empty) - action.get("imageUri") should be("somerepo/dos-downloader:tagged") + action.get("imageUri") should be("somerepo/drs-downloader:tagged") val actionLabels = action.get("labels").asInstanceOf[java.util.Map[_, _]] actionLabels.keySet.asScala should contain theSameElementsAs List("tag", "inputName")