Overwrite checker aj

src/main/java/com/onedatashare/scheduler/services/RequestModifier.java

@@ -69,6 +69,38 @@ public List<EntityInfo> selectAndExpand(TransferJobRequest.Source source, List<E
         return null;
     }
 
+    public List<EntityInfo> selectAndExpandDestination(TransferJobRequest.Destination destination, List<EntityInfo> selectedResources) {
+        //logger.info("The info list in select and expand is \n" + selectedResources.toString());
+        switch (destination.getType()) {
+            case ftp:
+                ftpExpander.createClient(destination.getVfsDestCredential());
+                return ftpExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case s3:
+                s3Expander.createClient(destination.getVfsDestCredential());
+                return s3Expander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case sftp:
+            case scp:
+                sftpExpander.createClient(destination.getVfsDestCredential());
+                return sftpExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case http:
+                httpExpander.createClient(destination.getVfsDestCredential());
+                return httpExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case box:
+                boxExpander.createClient(destination.getOauthDestCredential());
+                return boxExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case dropbox:
+                dropBoxExpander.createClient(destination.getOauthDestCredential());
+                return dropBoxExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+            case vfs:
+                return selectedResources;
+            case gdrive:
+                gDriveExpander.createClient(destination.getOauthDestCredential());
+                return gDriveExpander.expandedDestFileSystem(destination.getFileDestinationPath());
+
+        }
+        return null;
+    }
+
     /**
      * This method is supposed to take a list of Files that are expanded and make sure the chunk size being used is supported by the source/destination
      * So far all what I can tell is that Box has an "optimized" chunk size that we should use to write too.
@@ -137,7 +169,25 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) {
         List<EntityInfo> expandedFiles = this.selectAndExpand(s, odsTransferRequest.getSource().getResourceList());
         logger.info("Expanded files: {}", expandedFiles);
         expandedFiles = this.checkDestinationChunkSize(expandedFiles, d, odsTransferRequest.getOptions().getChunkSize());
-        s.setInfoList(expandedFiles);
+
+        //Overwrite checker
+        if (transferJobRequest.getOptions().isOverwrite())
+            s.setInfoList(expandedFiles);
+        else {
+            //Check files present in destination
+            List<EntityInfo> destExpandedFiles = this.selectAndExpandDestination(d, new ArrayList<>());
+
+            //for each file ID at dest, check if present in source list, if yes, remove that file id from src info list
+            for (EntityInfo destFile : destExpandedFiles) {
+                for (EntityInfo srcFile : expandedFiles) {
+                    if (destFile.getId().equals(srcFile.getId())) {
+                        expandedFiles.remove(srcFile);
+                    }
+                }
+            }
+            s.setInfoList(expandedFiles);
+        }
+
         transferJobRequest.setSource(s);
         transferJobRequest.setDestination(d);
         transferJobRequest.setTransferNodeName(odsTransferRequest.getTransferNodeName());

src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java

@@ -30,29 +30,35 @@ public void createClient(EndpointCredential credential) {
     public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResources, String basePath) {
         List<EntityInfo> transferFiles = new ArrayList<>();
         Stack<BoxFolder> travStack = new Stack<>();//this will only hold folders to traverse
-        if(userSelectedResources.isEmpty()) return new ArrayList<>(); //we need to signal the cancellation of this transferjob request.
-        for(EntityInfo selectedResource : userSelectedResources){
-            boolean isFile = false;
-            try{
-                BoxFile temp = new BoxFile(this.connection, selectedResource.getId());
-                transferFiles.add(boxFileToEntityInfo(temp));
-                isFile = true;
-            }catch (BoxAPIException ignored){
-                logger.info("Tried to open {} as a file but it did not work", selectedResource.toString());
-                isFile = false;
-            }
-            if(!isFile){
-                try{
-                    BoxFolder temp = new BoxFolder(this.connection, selectedResource.getId());
-                    travStack.push(temp);
-                }catch (BoxAPIException ignored){
-                    logger.info("Tried to open {} as a folder but it did not work", selectedResource.toString());
+        if (userSelectedResources.isEmpty()) {
+            //id = 0 represents root folder in box
+            BoxFolder temp = new BoxFolder(this.connection, "0");
+            travStack.push(temp);
+        } else {
+            for (EntityInfo selectedResource : userSelectedResources) {
+                boolean isFile = false;
+                try {
+                    BoxFile temp = new BoxFile(this.connection, selectedResource.getId());
+                    transferFiles.add(boxFileToEntityInfo(temp));
+                    isFile = true;
+                } catch (BoxAPIException ignored) {
+                    logger.info("Tried to open {} as a file but it did not work", selectedResource.toString());
+                    isFile = false;
+                }
+                if (!isFile) {
+                    try {
+                        BoxFolder temp = new BoxFolder(this.connection, selectedResource.getId());
+                        travStack.push(temp);
+                    } catch (BoxAPIException ignored) {
+                        logger.info("Tried to open {} as a folder but it did not work", selectedResource.toString());
+                    }
                 }
             }
         }
-        while(!travStack.isEmpty()){
+
+        while (!travStack.isEmpty()) {
             BoxFolder folder = travStack.pop();
-            for(BoxItem.Info child : folder){
+            for (BoxItem.Info child : folder) {
                 if (child instanceof BoxFile.Info) {
                     BoxFile.Info fileInfo = (BoxFile.Info) child;
                     BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID());
@@ -67,13 +73,41 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return transferFiles;
     }
 
+
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        Stack<BoxFolder> travStack = new Stack<>();//this will only hold folders to traverse
+
+        try {
+            //id = 0 represents root folder in box
+            BoxFolder temp = new BoxFolder(this.connection, "0");
+            travStack.push(temp);
+        } catch (BoxAPIException ignored) {
+            logger.info("Tried to open {} as a folder but it did not work", "0");
+        }
+
+
+        while (!travStack.isEmpty()) {
+            BoxFolder folder = travStack.pop();
+            for (BoxItem.Info child : folder) {
+                //ignore folders, if a file, add to list
+                if (child instanceof BoxFile.Info) {
+                    BoxFile.Info fileInfo = (BoxFile.Info) child;
+                    BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID());
+                    destFilesList.add(boxFileToEntityInfo(boxFile));
+                }
+            }
+        }
+        return destFilesList;
+    }
+
     @Override
     public List<EntityInfo> destinationChunkSize(List<EntityInfo> expandedFiles, String basePath, Integer userChunkSize) {
         BoxFolder destinationUploadFolder = new BoxFolder(this.connection, basePath);
-        for(EntityInfo entityInfo : expandedFiles){
-            if(entityInfo.getSize() < 1024*1024*20){
+        for (EntityInfo entityInfo : expandedFiles) {
+            if (entityInfo.getSize() < 1024 * 1024 * 20) {
                 entityInfo.setChunkSize(Math.toIntExact(entityInfo.getSize()));
-            }else{
+            } else {
                 BoxFileUploadSession.Info uploadSession = destinationUploadFolder.createUploadSession(entityInfo.getId(), entityInfo.getSize());
                 entityInfo.setChunkSize(uploadSession.getPartSize());
             }

src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java

@@ -71,6 +71,21 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return expandedFiles;
     }
 
+    public List<EntityInfo> expandedDestFileSystem(String parentPath) {
+        Stack<Metadata> traversalQueue = new Stack<>();
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        if (parentPath == null || parentPath.isEmpty()) parentPath = "";
+        //Expand all the files.
+
+        List<Metadata> resources = listOp(parentPath);
+        for (Metadata resource : resources) {
+            if (resource instanceof FileMetadata) {
+                destFilesList.add(metaDataToFileInfo((FileMetadata) resource));
+            }
+        }
+        return destFilesList;
+    }
+
     public EntityInfo metaDataToFileInfo(FileMetadata file) {
         EntityInfo fileInfo = new EntityInfo();
         fileInfo.setSize(file.getSize());

src/main/java/com/onedatashare/scheduler/services/expanders/FTPExpander.java

@@ -96,4 +96,39 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         }
         return filesToTransferList;
     }
+
+    @SneakyThrows
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+
+        List<EntityInfo> destFilesList = new LinkedList<>();
+        Stack<FileObject> traversalStack = new Stack<>();
+        FileSystemManager fsm = VFS.getManager();
+        if(basePath.isEmpty() || basePath == null || !basePath.endsWith("/")) basePath += "/";
+
+        FileObject obj = fsm.resolveFile(this.vfsCredential.getUri() + basePath, this.options);
+        //traversalStack.push(obj);
+
+        //get all files/folders from dest path and add to stack
+        traversalStack.addAll(Arrays.asList(obj.getChildren()));
+
+
+        for (int files = Integer.MAX_VALUE; files > 0 && !traversalStack.isEmpty(); --files) {
+            FileObject curr = traversalStack.pop();
+            FileName fileName = curr.getName();
+            URI uri = URI.create(fileName.getURI());
+            logger.info(uri.toString());
+
+            // check all files only. If a file, add it to the list of destFilesList, else if folder, ignore
+            if (curr.getType() == FileType.FILE) {
+
+                //filePath = curr.getPublicURIString().substring(this.vfsCredential.getUri().length()+basePath.length());
+                EntityInfo fileInfo = new EntityInfo();
+                fileInfo.setId(curr.getName().getBaseName());
+                fileInfo.setPath(uri.getPath());
+                fileInfo.setSize(curr.getContent().getSize());
+                destFilesList.add(fileInfo);
+            }
+        }
+        return destFilesList;
+    }
 }

src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java

@@ -73,6 +73,15 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return fileInfoList;
     }
 
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        Stack<File> fileListStack = new Stack<>();
+        List<EntityInfo> destFilesList = new ArrayList<>();
+
+        googleDriveLister(fileListStack, destFilesList, "", "");
+
+        return destFilesList;
+    }
+
     private void googleDriveLister(Stack<File> fileListStack, List<EntityInfo> fileInfoList, String fileQuery, String parentFolderId) {
         FileList fileList;
         String pageToken = "";

src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java

@@ -94,6 +94,23 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return filesToSend;
     }
 
+    @SneakyThrows
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        List<EntityInfo> destFilesList = new ArrayList<>();
+        Stack<Element> directoriesToTraverse = new Stack<>();
+        if (basePath.isEmpty()) basePath = "/";
+
+        Document doc = Jsoup.connect(this.credential.getUri() + basePath).get();
+        Elements links = doc.select("body a");
+        for (Element elem : links) {
+            //ignore dir, consider all files and add to list
+            if (!elem.text().endsWith("/")) {
+                destFilesList.add(fromElement(elem));
+            }
+        }
+        return destFilesList;
+    }
+
     public EntityInfo fromElement(Element elem) throws IOException {
         EntityInfo fileInfo = new EntityInfo();
         URL url = new URL(elem.absUrl("href"));

src/main/java/com/onedatashare/scheduler/services/expanders/S3Expander.java

@@ -67,6 +67,20 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         return traversedFiles;
     }
 
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        List<EntityInfo> traversedFiles = new LinkedList<>();
+        //trim leading forward slashes from base path (s3 doesn't recognise it as root)
+        basePath = StringUtils.stripStart(basePath, "/");
+
+        //expand the whole bucket relative to the basePath
+        ListObjectsV2Result result = this.s3Client.listObjectsV2(createSkeletonPerResource(basePath));
+        traversedFiles.addAll(convertV2ResultToEntityInfoList(result));
+
+        //is this check required before adding to list? --> if(obj.getKey().endsWith("/"))
+
+        return traversedFiles;
+    }
+
     public List<EntityInfo> convertV2ResultToEntityInfoList(ListObjectsV2Result result){
         List<EntityInfo> traversedFiles = new LinkedList<>();
         for(S3ObjectSummary fileInfo : result.getObjectSummaries()){

src/main/java/com/onedatashare/scheduler/services/expanders/SFTPExpander.java

@@ -124,4 +124,40 @@ public List<EntityInfo> expandedFileSystem(List<EntityInfo> userSelectedResource
         }
         return filesToTransferList;
     }
+
+    @SneakyThrows
+    public List<EntityInfo> expandedDestFileSystem(String basePath) {
+        //if(!basePath.endsWith("/")) basePath +="/";
+
+        List<EntityInfo> destFilesList = new LinkedList<>();
+        Stack<ChannelSftp.LsEntry> traversalStack = new Stack<>();
+        HashMap<ChannelSftp.LsEntry, String> entryToFullPath = new HashMap<>();
+        if (basePath.isEmpty()) basePath = channelSftp.pwd();
+        if(!basePath.endsWith("/")) basePath += "/";
+
+        Vector<ChannelSftp.LsEntry> fileVector = channelSftp.ls(basePath);
+        for (ChannelSftp.LsEntry curr : fileVector) {
+            entryToFullPath.put(curr, basePath + curr.getFilename());
+            traversalStack.add(curr);
+        }
+
+
+        while (!traversalStack.isEmpty()) {
+            ChannelSftp.LsEntry curr = traversalStack.pop();
+            String fullPath = entryToFullPath.remove(curr);
+            if (curr.getFilename().equals(".") || curr.getFilename().equals("..")) { //skip these two
+                continue;
+            }
+
+            // if dir skip, if file add to list
+            if (!curr.getAttrs().isDir()) {
+                EntityInfo fileInfo = new EntityInfo();
+                fileInfo.setPath(fullPath);
+                fileInfo.setId(curr.getFilename());
+                fileInfo.setSize(curr.getAttrs().getSize());
+                destFilesList.add(fileInfo);
+            }
+        }
+        return destFilesList;
+    }
 }