[JN-1632] GCS backend for document upload (#1509)

MatthewBemis · connorlbark · web-flow · commit a090cb6f8755 · 2025-03-19T16:36:40.000-04:00
Co-authored-by: Connor Barker &lt;connorlbark@gmail.com&gt;
diff --git a/api-admin/build.gradle b/api-admin/build.gradle
@@ -38,6 +38,7 @@ dependencies {
     implementation 'com.auth0:java-jwt:4.5.0'
     implementation 'org.liquibase:liquibase-core:4.31.0'
     implementation 'io.jsonwebtoken:jjwt-api:0.12.6'
+    implementation 'com.google.guava:guava:33.4.0-jre'
 
     implementation 'net.javacrumbs.shedlock:shedlock-spring:6.3.0'
     implementation 'net.javacrumbs.shedlock:shedlock-provider-jdbc-template:6.2.0'
diff --git a/api-admin/src/main/resources/application-gcp.yml b/api-admin/src/main/resources/application-gcp.yml
@@ -20,6 +20,8 @@ env:
     token: ${sm://${MIXPANEL_TOKEN_SECRET_ID}}
   airtable:
     authToken: ${sm://${AIRTABLE_AUTH_TOKEN_SECRET_ID}}
+  fileUpload:
+    backend: GCSFileStorageBackend
 spring:
   cloud:
     gcp:
diff --git a/api-admin/src/main/resources/application.yml b/api-admin/src/main/resources/application.yml
@@ -48,6 +48,10 @@ env:
   fileUpload:
     backend: LocalFileStorageBackend
     localFileStoragePath: ${LOCAL_FILE_STORAGE_PATH:/tmp/juniper/fileupload}
+    gcsStorageUnscannedBucketName: ${GCS_FILE_STORAGE_UNSCANNED_BUCKET_NAME:}
+    gcsStorageCleanBucketName: ${GCS_FILE_STORAGE_CLEAN_BUCKET_NAME:}
+    gcsStorageQuarantinedBucketName: ${GCS_FILE_STORAGE_QUARANTINED_BUCKET_NAME:}
+    gcsFileStorageProjectId: ${GCS_FILE_STORAGE_PROJECT_ID:}
 
 # Below here is non-deployment-specific
 
diff --git a/api-admin/src/test/resources/application.yml b/api-admin/src/test/resources/application.yml
@@ -18,6 +18,15 @@ env:
   email:
     sendgridApiKey: ${SENDGRID_API_KEY:}
 
+otel:
+  sdk:
+    disabled: true
+  instrumentation:
+    spring-webflux:
+      enabled: false
+    spring-web:
+      enabled: false
+
 # Below here is non-deployment-specific
 
 # When the target is 'local' the write-config.sh script will generate this properties file. It
diff --git a/api-participant/build.gradle b/api-participant/build.gradle
@@ -36,6 +36,7 @@ dependencies {
     implementation 'com.sendgrid:sendgrid-java:4.10.3'
     implementation 'org.liquibase:liquibase-core:4.31.0'
     implementation 'com.auth0:java-jwt:4.5.0'
+    implementation 'com.google.guava:guava:33.4.0-jre'
 
     implementation 'io.jsonwebtoken:jjwt-api:0.12.6'
     runtimeOnly 'io.jsonwebtoken:jjwt-impl:0.12.6',
diff --git a/api-participant/src/main/java/bio/terra/pearl/api/participant/service/file/ParticipantFileExtService.java b/api-participant/src/main/java/bio/terra/pearl/api/participant/service/file/ParticipantFileExtService.java
@@ -3,10 +3,12 @@
 import bio.terra.pearl.api.participant.service.AuthUtilService;
 import bio.terra.pearl.core.model.EnvironmentName;
 import bio.terra.pearl.core.model.file.ParticipantFile;
+import bio.terra.pearl.core.model.file.ScannedParticipantFileDto;
 import bio.terra.pearl.core.model.participant.Enrollee;
 import bio.terra.pearl.core.model.participant.ParticipantUser;
 import bio.terra.pearl.core.service.exception.NotFoundException;
 import bio.terra.pearl.core.service.file.ParticipantFileService;
+import bio.terra.pearl.core.service.file.VirusScanResult;
 import bio.terra.pearl.core.service.file.backends.FileStorageBackend;
 import bio.terra.pearl.core.service.file.backends.FileStorageBackendProvider;
 import java.io.IOException;
@@ -34,7 +36,7 @@ public ParticipantFileExtService(
     this.fileStorageBackend = fileStorageBackendProvider.get();
   }
 
-  public ParticipantFile get(
+  public ScannedParticipantFileDto get(
       String portalShortcode,
       EnvironmentName envName,
       ParticipantUser participantUser,
@@ -43,9 +45,13 @@ public ParticipantFile get(
     authUtilService.authParticipantToPortal(participantUser.getId(), portalShortcode, envName);
     Enrollee enrollee =
         authUtilService.authParticipantUserToEnrollee(participantUser.getId(), enrolleeShortcode);
-    return participantFileService
-        .findByEnrolleeIdAndFileName(enrollee.getId(), fileName)
-        .orElseThrow(() -> new NotFoundException("Could not find file"));
+
+    ParticipantFile file =
+        participantFileService
+            .findByEnrolleeIdAndFileName(enrollee.getId(), fileName)
+            .orElseThrow(() -> new NotFoundException("Could not find file"));
+
+    return participantFileService.attachVirusScanResult(file);
   }
 
   public InputStream downloadFile(
@@ -62,6 +68,13 @@ public InputStream downloadFile(
             .findByEnrolleeIdAndFileName(enrollee.getId(), fileName)
             .orElseThrow(() -> new NotFoundException("Could not find file"));
 
+    VirusScanResult virusScanResult =
+        participantFileService.getVirusScanResult(participantFile.getExternalFileId());
+
+    if (virusScanResult == VirusScanResult.QUARANTINED) {
+      throw new IllegalArgumentException("Virus detected in file");
+    }
+
     return fileStorageBackend.downloadFile(participantFile.getExternalFileId());
   }
 
diff --git a/api-participant/src/main/resources/application-gcp.yml b/api-participant/src/main/resources/application-gcp.yml
@@ -18,6 +18,8 @@ env:
     smartyAuthToken: ${sm://${SMARTY_AUTH_TOKEN_SECRET_ID}}
   mixpanel:
     token: ${sm://${MIXPANEL_TOKEN_SECRET_ID}}
+  fileUpload:
+    backend: GCSFileStorageBackend
 spring:
   cloud:
     gcp:
diff --git a/api-participant/src/main/resources/application.yml b/api-participant/src/main/resources/application.yml
@@ -36,6 +36,10 @@ env:
   fileUpload:
     backend: LocalFileStorageBackend
     localFileStoragePath: ${LOCAL_FILE_STORAGE_PATH:/tmp/juniper/fileupload}
+    gcsStorageUnscannedBucketName: ${GCS_FILE_STORAGE_UNSCANNED_BUCKET_NAME:}
+    gcsStorageCleanBucketName: ${GCS_FILE_STORAGE_CLEAN_BUCKET_NAME:}
+    gcsStorageQuarantinedBucketName: ${GCS_FILE_STORAGE_QUARANTINED_BUCKET_NAME:}
+    gcsFileStorageProjectId: ${GCS_FILE_STORAGE_PROJECT_ID:}
 # Below here is non-deployment-specific
 
 # When the target is 'local' the write-config.sh script will generate this properties file. It
diff --git a/api-participant/src/test/resources/application.yml b/api-participant/src/test/resources/application.yml
@@ -18,6 +18,15 @@ env:
   email:
     sendgridApiKey: ${SENDGRID_API_KEY:}
 
+otel:
+  sdk:
+    disabled: true
+  instrumentation:
+    spring-webflux:
+      enabled: false
+    spring-web:
+      enabled: false
+
 # Below here is non-deployment-specific
 
 # When the target is 'local' the write-config.sh script will generate this properties file. It
diff --git a/core/build.gradle b/core/build.gradle
@@ -50,6 +50,8 @@ dependencies {
     implementation 'com.smartystreets.api:smartystreets-java-sdk:3.16.1'
     implementation 'org.jooq:jooq:3.20.2'
     implementation 'com.mixpanel:mixpanel-java:1.5.3'
+    implementation platform('com.google.cloud:libraries-bom:26.54.0')
+    implementation 'com.google.cloud:google-cloud-storage:2.48.1'
 
     implementation 'com.google.cloud:spring-cloud-gcp-starter-secretmanager:6.0.0'
     implementation 'com.google.cloud:spring-cloud-gcp-starter:6.0.0'
diff --git a/core/src/main/java/bio/terra/pearl/core/dao/file/ParticipantFileDao.java b/core/src/main/java/bio/terra/pearl/core/dao/file/ParticipantFileDao.java
@@ -1,6 +1,6 @@
 package bio.terra.pearl.core.dao.file;
 
-import bio.terra.pearl.core.dao.BaseJdbiDao;
+import bio.terra.pearl.core.dao.BaseMutableJdbiDao;
 import bio.terra.pearl.core.dao.survey.AnswerDao;
 import bio.terra.pearl.core.model.file.ParticipantFile;
 import bio.terra.pearl.core.model.survey.Answer;
@@ -12,7 +12,7 @@
 import java.util.stream.Collectors;
 
 @Component
-public class ParticipantFileDao extends BaseJdbiDao<ParticipantFile> {
+public class ParticipantFileDao extends BaseMutableJdbiDao<ParticipantFile> {
     private final AnswerDao answerDao;
 
     public ParticipantFileDao(Jdbi jdbi, AnswerDao answerDao) {
diff --git a/core/src/main/java/bio/terra/pearl/core/model/file/ScannedParticipantFileDto.java b/core/src/main/java/bio/terra/pearl/core/model/file/ScannedParticipantFileDto.java
@@ -0,0 +1,18 @@
+package bio.terra.pearl.core.model.file;
+
+import bio.terra.pearl.core.service.file.VirusScanResult;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.beans.BeanUtils;
+
+@Getter
+@Setter
+public class ScannedParticipantFileDto extends ParticipantFile {
+    private VirusScanResult virusScanResult;
+
+    public ScannedParticipantFileDto(ParticipantFile participantFile, VirusScanResult virusScanResult) {
+        BeanUtils.copyProperties(participantFile, this);
+
+        this.virusScanResult = virusScanResult;
+    }
+}
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/FileStorageConfig.java b/core/src/main/java/bio/terra/pearl/core/service/file/FileStorageConfig.java
@@ -1,5 +1,7 @@
 package bio.terra.pearl.core.service.file;
 
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
 import lombok.Getter;
 import lombok.Setter;
 import org.springframework.context.annotation.Configuration;
@@ -11,9 +13,17 @@
 public class FileStorageConfig {
     private String defaultBackend;
     private String localFileStoragePath;
+    private String gcsStorageUnscannedBucketName;
+    private String gcsStorageCleanBucketName;
+    private String gcsStorageQuarantinedBucketName;
+    private Storage gcsStorageConfig;
 
     public FileStorageConfig(Environment environment) {
         this.defaultBackend = environment.getProperty("env.fileUpload.backend", "LocalFileStorageBackend");
         this.localFileStoragePath = environment.getProperty("env.fileUpload.localFileStoragePath");
+        this.gcsStorageUnscannedBucketName = environment.getProperty("env.fileUpload.gcsStorageUnscannedBucketName");
+        this.gcsStorageCleanBucketName = environment.getProperty("env.fileUpload.gcsStorageCleanBucketName");
+        this.gcsStorageQuarantinedBucketName = environment.getProperty("env.fileUpload.gcsStorageQuarantinedBucketName");
+        this.gcsStorageConfig = StorageOptions.newBuilder().setProjectId(environment.getProperty("env.fileUpload.gcsFileStorageProjectId")).build().getService();
     }
 }
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/ParticipantFileService.java b/core/src/main/java/bio/terra/pearl/core/service/file/ParticipantFileService.java
@@ -1,19 +1,14 @@
 package bio.terra.pearl.core.service.file;
 
 import bio.terra.pearl.core.dao.file.ParticipantFileDao;
-import bio.terra.pearl.core.model.BaseEntity;
 import bio.terra.pearl.core.model.file.ParticipantFile;
-import bio.terra.pearl.core.model.participant.Enrollee;
-import bio.terra.pearl.core.model.survey.SurveyResponse;
+import bio.terra.pearl.core.model.file.ScannedParticipantFileDto;
 import bio.terra.pearl.core.service.CascadeProperty;
-import bio.terra.pearl.core.service.ImmutableEntityService;
+import bio.terra.pearl.core.service.CrudService;
 import bio.terra.pearl.core.service.exception.NotFoundException;
 import bio.terra.pearl.core.service.file.backends.FileStorageBackend;
 import bio.terra.pearl.core.service.file.backends.FileStorageBackendProvider;
-import bio.terra.pearl.core.service.survey.SurveyResponseService;
-import bio.terra.pearl.core.service.survey.SurveyService;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.context.annotation.Lazy;
 import org.springframework.stereotype.Service;
 
 import java.io.InputStream;
@@ -24,7 +19,7 @@
 
 @Service
 @Slf4j
-public class ParticipantFileService extends ImmutableEntityService<ParticipantFile, ParticipantFileDao> {
+public class ParticipantFileService extends CrudService<ParticipantFile, ParticipantFileDao> {
     private final FileStorageBackend fileStorageBackend;
 
     public ParticipantFileService(ParticipantFileDao dao, FileStorageBackendProvider fileStorageBackendProvider) {
@@ -63,4 +58,14 @@ public void delete(UUID id, Set<CascadeProperty> cascade) {
         }
         super.delete(id, cascade);
     }
+
+    public ScannedParticipantFileDto attachVirusScanResult(ParticipantFile participantFile) {
+        VirusScanResult scanResult = fileStorageBackend.scanResult(participantFile.getExternalFileId());
+
+        return new ScannedParticipantFileDto(participantFile, scanResult);
+    }
+
+    public VirusScanResult getVirusScanResult(UUID fileId) {
+        return fileStorageBackend.scanResult(fileId);
+    }
 }
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/ScannedFile.java b/core/src/main/java/bio/terra/pearl/core/service/file/ScannedFile.java
@@ -0,0 +1,6 @@
+package bio.terra.pearl.core.service.file;
+
+import java.io.InputStream;
+
+public record ScannedFile(InputStream file, VirusScanResult scanResult) {
+}
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/VirusScanResult.java b/core/src/main/java/bio/terra/pearl/core/service/file/VirusScanResult.java
@@ -0,0 +1,7 @@
+package bio.terra.pearl.core.service.file;
+
+public enum VirusScanResult {
+    UNSCANNED,
+    CLEAN,
+    QUARANTINED
+}
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/backends/FileStorageBackend.java b/core/src/main/java/bio/terra/pearl/core/service/file/backends/FileStorageBackend.java
@@ -1,5 +1,7 @@
 package bio.terra.pearl.core.service.file.backends;
 
+import bio.terra.pearl.core.service.file.VirusScanResult;
+
 import java.io.InputStream;
 import java.util.UUID;
 
@@ -16,6 +18,8 @@ public interface FileStorageBackend {
      */
     InputStream downloadFile(UUID uploadedFileId);
 
+    VirusScanResult scanResult(UUID uploadedFileId);
+
     /**
      * Delete file from backend.
      */
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/backends/FileStorageBackendProvider.java b/core/src/main/java/bio/terra/pearl/core/service/file/backends/FileStorageBackendProvider.java
@@ -14,9 +14,11 @@ public class FileStorageBackendProvider {
 
     public FileStorageBackendProvider(
             FileStorageConfig fileStorageConfig,
-            LocalFileStorageBackend localFileStorageBackend) {
+            LocalFileStorageBackend localFileStorageBackend,
+            GCSFileStorageBackend gcsFileStorageBackend) {
         defaultBackend = fileStorageConfig.getDefaultBackend();
         backendMap.put("LocalFileStorageBackend", localFileStorageBackend);
+        backendMap.put("GCSFileStorageBackend", gcsFileStorageBackend);
     }
 
     public FileStorageBackend get() {
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/backends/GCSFileStorageBackend.java b/core/src/main/java/bio/terra/pearl/core/service/file/backends/GCSFileStorageBackend.java
diff --git a/core/src/main/java/bio/terra/pearl/core/service/file/backends/LocalFileStorageBackend.java b/core/src/main/java/bio/terra/pearl/core/service/file/backends/LocalFileStorageBackend.java
diff --git a/core/src/test/java/bio/terra/pearl/core/service/file/backends/GCSFileStorageBackendTest.java b/core/src/test/java/bio/terra/pearl/core/service/file/backends/GCSFileStorageBackendTest.java
diff --git a/terraform/gcp/k8s/environments/dev.yaml b/terraform/gcp/k8s/environments/dev.yaml
diff --git a/terraform/gcp/k8s/environments/prod.yaml b/terraform/gcp/k8s/environments/prod.yaml
diff --git a/terraform/gcp/k8s/templates/admin-deployment.yml b/terraform/gcp/k8s/templates/admin-deployment.yml
diff --git a/terraform/gcp/k8s/templates/participant-deployment.yml b/terraform/gcp/k8s/templates/participant-deployment.yml
diff --git a/terraform/gcp/k8s/values.yaml b/terraform/gcp/k8s/values.yaml
