Jg/scheduling refactor (#449)

* refactoring the transfer controller to support the new scheduling

* intial scheduling work and refactoring of job submission apis

* including spring profiles

* refactored the public API to have UserTransferOptions Contain the scheduled time

* backends fully up and running with the MetaData API now containing most recent influx information, and scheduler has been modified slightly

* Date picker (#448)

* visuals for calendar added

* added time to date picker

* added constraints for past dates and data will be sent to the backend

* Fixed b to Mb and interval rendering

* added credentialID textbox to stfp/http/ftp (#445)

* added credentialID textbox to stfp/http/ftp

* Credential ID working but some issues with spaced values

---------

Co-authored-by: jacob goldverg <[email protected]>

* removed all gridftp related functions (#443)

* syntax error fix

* request JSON changed

* fixed transfer submission. 2 problems 1 the resourceList was an object, and then there were no default values for the two new paths

* small change

---------

Co-authored-by: Andrew <[email protected]>
Co-authored-by: idkuri <[email protected]>

docker-compose.yml

@@ -16,15 +16,15 @@ services:
     command: mongod --smallfiles --logpath=/dev/null --quiet
 
   # ODS service that runs the Maven container
-  odsbackend:
-  #    image: onedatashare_odsbackend:latest #replace this with "build: ." if u want to build what you have locally. Make sure to update your env appropriately
-    build: .
-    env_file:
-      - .ods.env
-    restart: always
-    ports:
-      - 8080:8080
-    depends_on :
-      - mongodb         # ensuring that the Maven container is started only after the mongo-service is up
+#  odsbackend:
+#  #    image: onedatashare_odsbackend:latest #replace this with "build: ." if u want to build what you have locally. Make sure to update your env appropriately
+#    build: .
+#    env_file:
+#      - .ods.env
+#    restart: always
+#    ports:
+#      - 8080:8080
+#    depends_on :
+#      - mongodb         # ensuring that the Maven container is started only after the mongo-service is up
 volumes: 
   mongodb:

pom.xml

@@ -302,6 +302,11 @@
             <version>3.1.0</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-resolver-dns-native-macos</artifactId>
+            <classifier>osx-aarch_64</classifier>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/org/onedatashare/server/config/EurekaConfig.java

@@ -16,7 +16,7 @@ public class EurekaConfig {
 
     @Bean
     @LoadBalanced
-    public WebClient.Builder webClientBuilder(){
+    public WebClient.Builder webClientBuilder() {
         return WebClient.builder();
     }
 }

src/main/java/org/onedatashare/server/controller/TransferSchedulerController.java

@@ -0,0 +1,64 @@
+package org.onedatashare.server.controller;
+
+import org.onedatashare.server.model.ScheduledTransferJobRequest;
+import org.onedatashare.server.model.TransferJobRequestDTO;
+import org.onedatashare.server.model.request.StopRequest;
+import org.onedatashare.server.service.TransferSchedulerService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.server.ResponseStatusException;
+import reactor.core.publisher.Mono;
+
+import java.security.Principal;
+import java.time.LocalDateTime;
+import java.util.List;
+import java.util.UUID;
+
+@RestController
+@RequestMapping("/api/job")
+public class TransferSchedulerController {
+
+    private final TransferSchedulerService transferSchedulerService;
+
+    public TransferSchedulerController(TransferSchedulerService transferSchedulerService) {
+        this.transferSchedulerService = transferSchedulerService;
+    }
+
+
+    Logger logger = LoggerFactory.getLogger(TransferSchedulerController.class);
+
+    @PostMapping("/schedule")
+    public ResponseEntity<Mono<UUID>> runJob(@RequestBody TransferJobRequestDTO request,
+                                             Principal principal) {
+        logger.debug("Recieved request: " + request.toString());
+        request.setOwnerId(principal.getName());
+        return ResponseEntity.ok(transferSchedulerService.scheduleJob(request));
+    }
+
+    @PostMapping("/stop")
+    public Mono<Void> stopJob(@RequestBody StopRequest stopRequest) {
+        return transferSchedulerService.stopTransferJob(stopRequest)
+                .onErrorResume(e -> Mono.error(new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR,
+                        "Failed to stop job execution")));
+    }
+    @GetMapping("/list")
+    public ResponseEntity<Mono<List<ScheduledTransferJobRequest>>> listScheduledJobs(@RequestParam String userEmail) {
+        return ResponseEntity.ok(this.transferSchedulerService.listScheduledJobs(userEmail));
+    }
+
+    @GetMapping("/details")
+    public ResponseEntity<Mono<TransferJobRequestDTO>> getScheduledJob(@RequestParam UUID jobUuid) {
+        return ResponseEntity.ok(this.transferSchedulerService.getJobDetails(jobUuid));
+    }
+
+    @DeleteMapping("/delete")
+    public ResponseEntity<Void> deleteScheduledJob(@RequestParam UUID jobUuid) {
+        this.transferSchedulerService.deleteScheduledJob(jobUuid);
+        return ResponseEntity.accepted().build();
+    }
+
+}

src/main/java/org/onedatashare/server/model/EntityInfo.java

@@ -0,0 +1,14 @@
+package org.onedatashare.server.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class EntityInfo {
+    protected String id;
+    protected String path;
+    protected long size;
+}

src/main/java/org/onedatashare/server/model/FileDestination.java

@@ -0,0 +1,24 @@
+package org.onedatashare.server.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import org.onedatashare.server.model.core.EndpointType;
+
+import java.io.Serializable;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class FileDestination implements Serializable {
+
+    @NonNull
+    private String credId;
+
+    @NonNull
+    private EndpointType type;
+
+
+    String fileDestinationPath = "";
+}

src/main/java/org/onedatashare/server/model/FileSource.java

@@ -0,0 +1,26 @@
+package org.onedatashare.server.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import org.onedatashare.server.model.core.EndpointType;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class FileSource implements Serializable {
+    @NonNull
+    private String credId;
+    @NonNull
+    public EndpointType type;
+
+    public String fileSourcePath = ""; //can also be the parent Id to the directory to find all data in the infoList
+
+    @NonNull
+    public ArrayList<EntityInfo> resourceList; //a list of files and folders. This will end up being a list of only files with paths fully expanded
+
+}

src/main/java/org/onedatashare/server/model/ScheduledTransferJobRequest.java

@@ -0,0 +1,20 @@
+package org.onedatashare.server.model;
+
+import lombok.Data;
+import org.onedatashare.server.model.request.UserTransferOptions;
+
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import java.util.UUID;
+
+@Data
+public class ScheduledTransferJobRequest implements Serializable {
+
+    LocalDateTime jobStartTime;
+    UUID jobUuid;
+    String ownerId;
+    FileSource source;
+    FileDestination destination;
+    UserTransferOptions options;
+    String transferNodeName;
+}

src/main/java/org/onedatashare/server/model/TransferJobRequestDTO.java

@@ -0,0 +1,18 @@
+package org.onedatashare.server.model;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import lombok.Data;
+import org.onedatashare.server.model.request.UserTransferOptions;
+
+import java.io.Serializable;
+
+@Data
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class TransferJobRequestDTO implements Serializable {
+    private String ownerId;
+    private FileSource source;
+    private FileDestination destination;
+    private UserTransferOptions options;
+    private String transferNodeName;
+
+}

src/main/java/org/onedatashare/server/model/core/Stat.java

@@ -157,7 +157,7 @@ public String path() {
    * Set the files underneath this tree and reset cached values.
    */
   public Stat setFiles(Collection<Stat> fs) {
-    return setFiles(fs.toArray(new Stat[fs.size()]));
+    return setFiles(fs.toArray(new Stat[0]));
   }
 
   public void setFilesList(List<Stat> fs){

src/main/java/org/onedatashare/server/model/request/UserTransferOptions.java

@@ -1,29 +1,32 @@
 /**
- ##**************************************************************
- ##
- ## Copyright (C) 2018-2020, OneDataShare Team, 
- ## Department of Computer Science and Engineering,
- ## University at Buffalo, Buffalo, NY, 14260.
- ## 
- ## Licensed under the Apache License, Version 2.0 (the "License"); you
- ## may not use this file except in compliance with the License.  You may
- ## obtain a copy of the License at
- ## 
- ##    http://www.apache.org/licenses/LICENSE-2.0
- ## 
- ## Unless required by applicable law or agreed to in writing, software
- ## distributed under the License is distributed on an "AS IS" BASIS,
- ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ## See the License for the specific language governing permissions and
- ## limitations under the License.
- ##
- ##**************************************************************
+ * ##**************************************************************
+ * ##
+ * ## Copyright (C) 2018-2020, OneDataShare Team,
+ * ## Department of Computer Science and Engineering,
+ * ## University at Buffalo, Buffalo, NY, 14260.
+ * ##
+ * ## Licensed under the Apache License, Version 2.0 (the "License"); you
+ * ## may not use this file except in compliance with the License.  You may
+ * ## obtain a copy of the License at
+ * ##
+ * ##    http://www.apache.org/licenses/LICENSE-2.0
+ * ##
+ * ## Unless required by applicable law or agreed to in writing, software
+ * ## distributed under the License is distributed on an "AS IS" BASIS,
+ * ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * ## See the License for the specific language governing permissions and
+ * ## limitations under the License.
+ * ##
+ * ##**************************************************************
  */
 
 
 package org.onedatashare.server.model.request;
 
 import lombok.Data;
+import org.springframework.format.annotation.DateTimeFormat;
+
+import java.time.LocalDateTime;
 
 @Data
 public class UserTransferOptions {
@@ -37,4 +40,20 @@ public class UserTransferOptions {
     private Integer parallelThreadCount; //supported
     private Integer pipeSize; //supported
     private Integer chunkSize; //supported
+    @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+    private LocalDateTime scheduledTime;
+
+    public UserTransferOptions() {
+        this.compress = false;
+        this.encrypt = false;
+        this.optimizer = "";
+        this.overwrite = false;
+        this.retry = 0;
+        this.verify = false;
+        this.concurrencyThreadCount = 1;
+        this.pipeSize = 1;
+        this.parallelThreadCount = 1;
+        this.chunkSize = 10 << 1024 << 1024;
+        this.scheduledTime = LocalDateTime.now();
+    }
 }