overwrite flag initial changes for HTTP, Box, DropBox and Gdrive

Author: A-bel

src/main/java/com/onedatashare/scheduler/services/RequestModifier.java

@@ -84,18 +84,18 @@ public List<EntityInfo> selectAndExpandDestination(TransferJobRequest.Destinatio
                 return sftpExpander.expandedDestFileSystem(destination.getFileDestinationPath());
             case http:
                 httpExpander.createClient(destination.getVfsDestCredential());
-                return httpExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath());
+                return httpExpander.expandedDestFileSystem(destination.getFileDestinationPath());
             case box:
                 boxExpander.createClient(destination.getOauthDestCredential());
-                return boxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath());
+                return boxExpander.expandedDestFileSystem(destination.getFileDestinationPath());
             case dropbox:
                 dropBoxExpander.createClient(destination.getOauthDestCredential());
-                return dropBoxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath());
+                return dropBoxExpander.expandedDestFileSystem(destination.getFileDestinationPath());
             case vfs:
                 return selectedResources;
             case gdrive:
                 gDriveExpander.createClient(destination.getOauthDestCredential());
-                return gDriveExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath());
+                return gDriveExpander.expandedDestFileSystem(destination.getFileDestinationPath());
 
         }
         return null;
@@ -174,14 +174,14 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) {
         if (transferJobRequest.getOptions().isOverwrite())
             s.setInfoList(expandedFiles);
         else {
-            //Check files present in destination (given path level only) and remove them from infoList sent from source
+            //Check files present in destination (given path level only)
 
             //can use selectAndExpand but not for vfs
             List<EntityInfo> destExpandedFiles = this.selectAndExpandDestination(d, new ArrayList<>());
 
 
             /*
-            for each file ID at dest, check if present in source list, if no, no action, if yes, remove that file id from src info list
+            for each file ID at dest, check if present in source list, if yes, remove that file id from src info list
             expandedFiles.remove()
             s.setInfoList(expandedFiles);
             */

src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java

@@ -67,6 +67,34 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return transferFiles;
     }
 
+
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        Stack<BoxFolder> travStack = new Stack<>();//this will only hold folders to traverse
+
+        try{
+            //id = 0 represents root folder in box
+            BoxFolder temp = new BoxFolder(this.connection, "0");
+            travStack.push(temp);
+        }catch (BoxAPIException ignored){
+            logger.info("Tried to open {} as a folder but it did not work", "0");
+        }
+
+
+        while(!travStack.isEmpty()){
+            BoxFolder folder = travStack.pop();
+            for(BoxItem.Info child : folder){
+                //ignore folders, if a file, add to list
+                if (child instanceof BoxFile.Info) {
+                    BoxFile.Info fileInfo = (BoxFile.Info) child;
+                    BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID());
+                    destFilesList.add(boxFileToEntityInfo(boxFile));
+                }
+            }
+        }
+        return destFilesList;
+    }
+
     @Override
     public List<EntityInfo> destinationChunkSize(List<EntityInfo> expandedFiles, String basePath, Integer userChunkSize) {
         BoxFolder destinationUploadFolder = new BoxFolder(this.connection, basePath);

src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java

@@ -71,6 +71,21 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return expandedFiles;
     }
 
+    public List<EntityInfo> expandedDestFileSystem(String parentPath) {
+        Stack<Metadata> traversalQueue = new Stack<>();
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        if (parentPath == null || parentPath.isEmpty()) parentPath = "";
+        //Expand all the files.
+
+        List<Metadata> resources = listOp(parentPath);
+        for (Metadata resource : resources) {
+            if (resource instanceof FileMetadata) {
+                destFilesList.add(metaDataToFileInfo((FileMetadata) resource));
+            }
+        }
+        return destFilesList;
+    }
+
     public EntityInfo metaDataToFileInfo(FileMetadata file) {
         EntityInfo fileInfo = new EntityInfo();
         fileInfo.setSize(file.getSize());

src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java

@@ -73,6 +73,15 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return fileInfoList;
     }
 
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        Stack<File> fileListStack = new Stack<>();
+        List<EntityInfo> destFilesList = new ArrayList<>();
+
+        googleDriveLister(fileListStack, destFilesList, "", "");
+
+        return destFilesList;
+    }
+
     private void googleDriveLister(Stack<File> fileListStack, List<EntityInfo> fileInfoList, String fileQuery, String parentFolderId) {
         FileList fileList;
         String pageToken = "";

src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java

@@ -94,6 +94,23 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return filesToSend;
     }
 
+    @SneakyThrows
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        Stack<Element> directoriesToTraverse = new Stack<>();
+        if (basePath.isEmpty()) basePath = "/";
+
+        Document doc = Jsoup.connect(this.credential.getUri() + basePath).get();
+        Elements links = doc.select("body a");
+        for (Element elem : links) {
+            //ignore dir, consider all files and add to list
+            if (!elem.text().endsWith("/")) {
+                destFilesList.add(fromElement(elem));
+            }
+        }
+        return destFilesList;
+    }
+
     public EntityInfo fromElement(Element elem) throws IOException {
         EntityInfo fileInfo = new EntityInfo();
         URL url = new URL(elem.absUrl("href"));