vault integration - features - fetch credentials of RabbitMQ from vault

pom.xml

@@ -98,6 +98,16 @@
             <artifactId>jsoup</artifactId>
             <version>1.15.3</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-vault-config</artifactId>
+            <version>3.0.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-bootstrap</artifactId>
+            <version>4.0.2</version>
+        </dependency>
 
     </dependencies>
     <dependencyManagement>

src/main/java/com/rabbitMq/rabbitmqscheduler/Controller/JobController.java

@@ -1,5 +1,6 @@
 package com.rabbitMq.rabbitmqscheduler.Controller;
 
+import com.rabbitMq.rabbitmqscheduler.DTO.StopRequest;
 import com.rabbitMq.rabbitmqscheduler.DTO.TransferJobResponse;
 import com.rabbitMq.rabbitmqscheduler.DTO.transferFromODS.RequestFromODS;
 import com.rabbitMq.rabbitmqscheduler.DTO.TransferJobRequest;
@@ -30,4 +31,11 @@ public TransferJobResponse receiveRequest(@RequestBody RequestFromODS odsTransfe
         response.setMessage("Job Submitted");
         return response;
     }
+    @PostMapping(value = "/stopJob")
+    public Boolean stopJob(@RequestBody StopRequest stopRequest) {
+        logger.info("Received request to stop job with id: " + stopRequest.toString());
+        messageSender.sendStopJobRequest(stopRequest);
+        return true;
+    }
+
 }

src/main/java/com/rabbitMq/rabbitmqscheduler/DTO/Properties.java

@@ -0,0 +1,14 @@
+package com.rabbitMq.rabbitmqscheduler.DTO;
+
+import lombok.Data;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@Data
+@ConfigurationProperties("ods")
+public class Properties {
+
+    String AMPQ_PWD;
+    String AMPQ_USER;
+
+}

src/main/java/com/rabbitMq/rabbitmqscheduler/DTO/StopRequest.java

@@ -0,0 +1,10 @@
+package com.rabbitMq.rabbitmqscheduler.DTO;
+
+import lombok.Data;
+
+@Data
+public class StopRequest {
+    Long jobId;
+    String transferNodeName;
+
+}

src/main/java/com/rabbitMq/rabbitmqscheduler/DTO/StopResponses.java

@@ -0,0 +1,19 @@
+package com.rabbitMq.rabbitmqscheduler.DTO;
+
+public class StopResponses {
+    private boolean success;
+    private String errorMessage;
+
+    public StopResponses(boolean success, String errorMessage) {
+        this.success = success;
+        this.errorMessage = errorMessage;
+    }
+
+    public boolean isSuccess() {
+        return success;
+    }
+
+    public String getErrorMessage() {
+        return errorMessage;
+    }
+}

src/main/java/com/rabbitMq/rabbitmqscheduler/RabbitmqSchedulerApplication.java

@@ -1,12 +1,15 @@
 package com.rabbitMq.rabbitmqscheduler;
 
+import com.rabbitMq.rabbitmqscheduler.DTO.Properties;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
 
 
 @SpringBootApplication
 @EnableEurekaClient
+@EnableConfigurationProperties({Properties.class})
 public class RabbitmqSchedulerApplication {
 
 	public static void main(String[] args) {

src/main/java/com/rabbitMq/rabbitmqscheduler/Sender/MessageSender.java

@@ -1,12 +1,15 @@
 package com.rabbitMq.rabbitmqscheduler.Sender;
 
+import com.rabbitMq.rabbitmqscheduler.DTO.StopRequest;
+import com.rabbitMq.rabbitmqscheduler.DTO.StopResponses;
 import com.rabbitMq.rabbitmqscheduler.DTO.TransferJobRequest;
 import com.rabbitMq.rabbitmqscheduler.DTO.TransferParams;
 import com.rabbitMq.rabbitmqscheduler.DTO.transferFromODS.RequestFromODS;
 import com.rabbitMq.rabbitmqscheduler.Enums.EndPointType;
 import lombok.NonNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.amqp.AmqpException;
 import org.springframework.amqp.core.AmqpTemplate;
 import org.springframework.amqp.core.DirectExchange;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -54,6 +57,27 @@ public void sendTransferRequest(TransferJobRequest odsTransferRequest, RequestFr
         }
         logger.info("Processed Job: {}", odsTransferRequest);
     }
+    public StopResponses sendStopJobRequest(StopRequest stopRequest) {
+        String errorMessage = null;
+        boolean success = true;
+
+        if (stopRequest.getTransferNodeName() == null || stopRequest.getTransferNodeName().isEmpty()) {
+            errorMessage = "Failed to send stop job request. Transfer node name" + stopRequest.getTransferNodeName() + " is not provided.";
+            success = false;
+        } else {
+            try {
+                rmqTemplate.convertAndSend(exchange, stopRequest.getTransferNodeName(), stopRequest.getJobId());
+            } catch (AmqpException e) {
+                errorMessage = "Error occurred while sending stop job request: " + e.getMessage();
+                success = false;
+            }
+        }
+
+        return new StopResponses(success, errorMessage);
+
+
+    }
+
 
     /**
      * The Transfer params to send using the routingKey

src/main/java/com/rabbitMq/rabbitmqscheduler/Services/GDriveExpander.java

@@ -26,6 +26,7 @@
 @Service
 public class GDriveExpander extends DestinationChunkSize implements FileExpander{
 
+    Logger logger = LoggerFactory.getLogger(GDriveExpander.class);
     @Value("${gdrive.client.id}")
     private String gDriveClientId;
 
@@ -35,6 +36,7 @@ public class GDriveExpander extends DestinationChunkSize implements FileExpander
     @Value("${gdrive.appname}")
     private String gdriveAppName;
 
+    private final String REQUEST_FILE_FIELDS = "id, name, size, mimeType, modifiedTime, md5Checksum, trashed, parents";
     private Drive client;
 
     @Override
@@ -59,54 +61,76 @@ public void createClient(EndpointCredential credential) {
     public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
         Stack<File> fileListStack = new Stack<>();
         List<EntityInfo> fileInfoList = new ArrayList<>();
-        if(userSelectedResources.isEmpty()){
-            googleDriveLister(fileListStack, fileInfoList, "", "");
-        }else{
             for(EntityInfo fileInfo : userSelectedResources){
                 String fileQuery = "'" + fileInfo.getId() + "' in parents and trashed=false";
-                googleDriveLister(fileListStack, fileInfoList, fileQuery, fileInfo.getId());
-            }
+                EntityInfo info = getMetadataForfile(fileInfo.getId());
+                if(info==null){
+                    fileInfoList.addAll(googleDriveLister(fileListStack, fileQuery, fileInfo.getId()));
+                } else{
+                    fileInfoList.add(info);
+                }
         }
+
         while(!fileListStack.isEmpty()){
             File file = fileListStack.pop();
             String fileQuery = "'" + file.getId() + "' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed=false";
-            googleDriveLister(fileListStack, fileInfoList, fileQuery, file.getId());
+            googleDriveLister(fileListStack, fileQuery, file.getId());
         }
         return fileInfoList;
     }
 
-    private void googleDriveLister(Stack<File> fileListStack, List<EntityInfo> fileInfoList, String fileQuery, String parentFolderId) {
+    private List<EntityInfo> googleDriveLister(Stack<File> fileListStack, String fileQuery, String folderId) {
         FileList fileList;
         String pageToken = "";
+
+        List<EntityInfo> folderFileList = new ArrayList<>();
         try {
             do {
                 fileList = this.client.files().list()
                         .setQ(fileQuery)
                         .setFields("nextPageToken, files(id, name, parents, size, mimeType)")
                         .setPageToken(pageToken)
                         .execute();
+
                 for(File file : fileList.getFiles()){
-                    if(file.getId().equals(parentFolderId)){
+                    if(file.getId().equals(folderId)){//the folder that u query appears in the result
                         continue;
                     }
                     if(file.getMimeType().equals("application/vnd.google-apps.folder")){
                         fileListStack.add(file);
                     }else{
-                        fileInfoList.add(googleFileToEntityInfo(file));
+                        folderFileList.add(getMetadataForfile(file.getId()));
                     }
                 }
                 pageToken = fileList.getNextPageToken();
             } while (pageToken != null);
         } catch (IOException e) {
+            logger.error("Error listing files from Google drive",e);
+        }
+        return folderFileList;
+    }
+
+    public EntityInfo getMetadataForfile(String fileId){
+        try {
+            File file = this.client.files().get(fileId)
+                    .setFields(REQUEST_FILE_FIELDS)
+                    .execute();
+            return googleFileToEntityInfo(file);
+        } catch(IOException e){
             e.printStackTrace();
         }
+       return null;
     }
 
     private EntityInfo googleFileToEntityInfo(File googleFile){
+        if(googleFile.getId() == null || googleFile.getParents() == null || googleFile.getSize() == null || googleFile.getMd5Checksum() == null){
+            return null;
+        }
         EntityInfo entityInfo = new EntityInfo();
         entityInfo.setId(googleFile.getId());
         entityInfo.setSize(googleFile.getSize());
         entityInfo.setPath(String.valueOf(googleFile.getParents()));
+        entityInfo.setChecksum(googleFile.getMd5Checksum());
         return entityInfo;
     }
 }

src/main/resources/application.properties

@@ -19,7 +19,9 @@ ods.rabbitmq.queue=transferqueue
 ods.rabbitmq.routingkey=ods.routing
 spring.rabbitmq.addresses=${RMQ_ADDRESS}
 spring.rabbitmq.port=${AMPQ_PORT:5672}
-spring.rabbitmq.username=${AMPQ_USER:guest}
-spring.rabbitmq.password=${AMPQ_PWD:guest}
 
 dropbox.identifier=${ODS_DROPBOX_IDENTIFIER:OneDataShare-DIDCLab}
+
+#fetching from vault if not passed in boot.sh
+spring.rabbitmq.username=${AMPQ_USER:guest}
+spring.rabbitmq.password=${AMPQ_PWD:guest}

src/main/resources/bootstrap.yaml

@@ -0,0 +1,16 @@
+spring:
+  application:
+    name: RabbitmqSchedulerApplication
+  cloud:
+    vault:
+      scheme: https
+      host: 'vault.onedatashare.org'
+      port: 443
+      token: <place_token_here>
+      authentication: token
+      kv:
+        enabled: true
+        profile-separator: '/'
+        default-context : RabbitmqSchedulerApplication
+        backend: secret
+        profiles: application_properties