Accommodate "enhanced" Requester Pays error messages [CROM-6820] (#6556)

broadinstitute · Nov 4, 2021 · c93519c · c93519c
1 parent b360c11
commit c93519c
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 8 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Cromwell Change Log
 
+## 71 Release Notes
+
+### Bug Fixes
+
+* Fixed an issue handling data in Google Cloud Storage buckets with requester pays enabled that could sometimes cause I/O to fail.
+
 ## 70 Release Notes
 
 ### CWL security fix [#6510](https://github.com/broadinstitute/cromwell/pull/6510)

diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsEnhancedRequest.scala
@@ -1,13 +1,13 @@
 package cromwell.filesystems.gcs
 
-import java.io.FileNotFoundException
-
 import akka.http.scaladsl.model.StatusCodes
 import cats.effect.IO
 import com.google.api.client.googleapis.json.GoogleJsonResponseException
 import com.google.cloud.storage.StorageException
 import cromwell.filesystems.gcs.RequesterPaysErrors.isProjectNotProvidedError
 
+import java.io.FileNotFoundException
+
 object GcsEnhancedRequest {
 
   // If the request fails because no project was passed, recover the request, this time setting the project
@@ -19,11 +19,11 @@ object GcsEnhancedRequest {
         // Use NoSuchFileException for better error reporting
         case e: StorageException if e.getCode == StatusCodes.NotFound.intValue =>
           IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}"))
-        case e: GoogleJsonResponseException if isProjectNotProvidedError(e) => 
+        case e: GoogleJsonResponseException if isProjectNotProvidedError(e) =>
           IO(f(true))
         case e: GoogleJsonResponseException if e.getStatusCode == StatusCodes.NotFound.intValue =>
           IO.raiseError(new FileNotFoundException(s"File not found: ${path.pathAsString}"))
-        case e => 
+        case e =>
           IO.raiseError(e)
       })
   }

diff --git a/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala b/filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/RequesterPaysErrors.scala
@@ -2,22 +2,23 @@ package cromwell.filesystems.gcs
 
 import com.google.api.client.googleapis.json.{GoogleJsonError, GoogleJsonResponseException}
 import com.google.cloud.storage.StorageException
+import org.apache.commons.lang3.StringUtils
 
 object RequesterPaysErrors {
   val BucketIsRequesterPaysErrorCode = 400
   val BucketIsRequesterPaysErrorMessage = "Bucket is requester pays bucket but no user project provided."
   val DoesNotHaveServiceUsePermissionErrorCode = 403
   val DoesNotHaveServiceUsePermissionErrorMessage = "does not have serviceusage.services.use"
 
-  def isProjectNotProvidedError(storageException: StorageException) = 
+  def isProjectNotProvidedError(storageException: StorageException) =
     storageException.getCode == BucketIsRequesterPaysErrorCode &&
-    storageException.getMessage == BucketIsRequesterPaysErrorMessage
+    StringUtils.contains(storageException.getMessage, BucketIsRequesterPaysErrorMessage)
 
   def isProjectNotProvidedError(googleJsonError: GoogleJsonError) =
     googleJsonError.getCode == BucketIsRequesterPaysErrorCode &&
-      googleJsonError.getMessage == BucketIsRequesterPaysErrorMessage
+      StringUtils.contains(googleJsonError.getMessage, BucketIsRequesterPaysErrorMessage)
 
   def isProjectNotProvidedError(googleJsonError: GoogleJsonResponseException) =
     googleJsonError.getStatusCode == BucketIsRequesterPaysErrorCode &&
-      googleJsonError.getContent == BucketIsRequesterPaysErrorMessage
+      StringUtils.contains(googleJsonError.getContent, BucketIsRequesterPaysErrorMessage)
 }