From fab13e2d5f9043ea6f1e8a1b8f4624c454289259 Mon Sep 17 00:00:00 2001 From: Abel Jacob Date: Fri, 1 Dec 2023 19:02:32 -0500 Subject: [PATCH 1/4] overwrite flag initial changes for FTP, S3 and SFTP --- .../scheduler/services/RequestModifier.java | 54 ++++++++++++++++++- .../services/expanders/FTPExpander.java | 35 ++++++++++++ .../services/expanders/S3Expander.java | 14 +++++ .../services/expanders/SFTPExpander.java | 36 +++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java index afc5443..0579983 100644 --- a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java +++ b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java @@ -69,6 +69,38 @@ public List selectAndExpand(TransferJobRequest.Source source, List selectAndExpandDestination(TransferJobRequest.Destination destination, List selectedResources) { + //logger.info("The info list in select and expand is \n" + selectedResources.toString()); + switch (destination.getType()) { + case ftp: + ftpExpander.createClient(destination.getVfsDestCredential()); + return ftpExpander.expandedDestFileSystem(destination.getFileDestinationPath()); + case s3: + s3Expander.createClient(destination.getVfsDestCredential()); + return s3Expander.expandedDestFileSystem(destination.getFileDestinationPath()); + case sftp: + case scp: + sftpExpander.createClient(destination.getVfsDestCredential()); + return sftpExpander.expandedDestFileSystem(destination.getFileDestinationPath()); + case http: + httpExpander.createClient(destination.getVfsDestCredential()); + return httpExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + case box: + boxExpander.createClient(destination.getOauthDestCredential()); + return boxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + case dropbox: + dropBoxExpander.createClient(destination.getOauthDestCredential()); + return dropBoxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + case vfs: + return selectedResources; + case gdrive: + gDriveExpander.createClient(destination.getOauthDestCredential()); + return gDriveExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + + } + return null; + } + /** * This method is supposed to take a list of Files that are expanded and make sure the chunk size being used is supported by the source/destination * So far all what I can tell is that Box has an "optimized" chunk size that we should use to write too. @@ -137,7 +169,27 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) { List expandedFiles = this.selectAndExpand(s, odsTransferRequest.getSource().getResourceList()); logger.info("Expanded files: {}", expandedFiles); expandedFiles = this.checkDestinationChunkSize(expandedFiles, d, odsTransferRequest.getOptions().getChunkSize()); - s.setInfoList(expandedFiles); + + //Overwrite checker + if (transferJobRequest.getOptions().isOverwrite()) + s.setInfoList(expandedFiles); + else { + //Check files present in destination (given path level only) and remove them from infoList sent from source + + //can use selectAndExpand but not for vfs + List destExpandedFiles = this.selectAndExpandDestination(d, new ArrayList<>()); + + + /* + for each file ID at dest, check if present in source list, if no, no action, if yes, remove that file id from src info list + expandedFiles.remove() + s.setInfoList(expandedFiles); + */ + + + //TODO: fix box case handling where if no files selected by user (to match other cases) + } + transferJobRequest.setSource(s); transferJobRequest.setDestination(d); transferJobRequest.setTransferNodeName(odsTransferRequest.getTransferNodeName()); diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/FTPExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/FTPExpander.java index a01ab88..84c5b17 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/FTPExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/FTPExpander.java @@ -96,4 +96,39 @@ public List expandedFileSystem(List userSelectedResource } return filesToTransferList; } + + @SneakyThrows + public List expandedDestFileSystem(String basePath) { + + List destFilesList = new LinkedList<>(); + Stack traversalStack = new Stack<>(); + FileSystemManager fsm = VFS.getManager(); + if(basePath.isEmpty() || basePath == null || !basePath.endsWith("/")) basePath += "/"; + + FileObject obj = fsm.resolveFile(this.vfsCredential.getUri() + basePath, this.options); + //traversalStack.push(obj); + + //get all files/folders from dest path and add to stack + traversalStack.addAll(Arrays.asList(obj.getChildren())); + + + for (int files = Integer.MAX_VALUE; files > 0 && !traversalStack.isEmpty(); --files) { + FileObject curr = traversalStack.pop(); + FileName fileName = curr.getName(); + URI uri = URI.create(fileName.getURI()); + logger.info(uri.toString()); + + // check all files only. If a file, add it to the list of destFilesList, else if folder, ignore + if (curr.getType() == FileType.FILE) { + + //filePath = curr.getPublicURIString().substring(this.vfsCredential.getUri().length()+basePath.length()); + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setId(curr.getName().getBaseName()); + fileInfo.setPath(uri.getPath()); + fileInfo.setSize(curr.getContent().getSize()); + destFilesList.add(fileInfo); + } + } + return destFilesList; + } } diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/S3Expander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/S3Expander.java index 35bfab3..5d5054a 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/S3Expander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/S3Expander.java @@ -67,6 +67,20 @@ public List expandedFileSystem(List userSelectedResource return traversedFiles; } + public List expandedDestFileSystem(String basePath) { + List traversedFiles = new LinkedList<>(); + //trim leading forward slashes from base path (s3 doesn't recognise it as root) + basePath = StringUtils.stripStart(basePath, "/"); + + //expand the whole bucket relative to the basePath + ListObjectsV2Result result = this.s3Client.listObjectsV2(createSkeletonPerResource(basePath)); + traversedFiles.addAll(convertV2ResultToEntityInfoList(result)); + + //is this check required before adding to list? --> if(obj.getKey().endsWith("/")) + + return traversedFiles; + } + public List convertV2ResultToEntityInfoList(ListObjectsV2Result result){ List traversedFiles = new LinkedList<>(); for(S3ObjectSummary fileInfo : result.getObjectSummaries()){ diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/SFTPExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/SFTPExpander.java index 6f675ca..780f193 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/SFTPExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/SFTPExpander.java @@ -124,4 +124,40 @@ public List expandedFileSystem(List userSelectedResource } return filesToTransferList; } + + @SneakyThrows + public List expandedDestFileSystem(String basePath) { + //if(!basePath.endsWith("/")) basePath +="/"; + + List destFilesList = new LinkedList<>(); + Stack traversalStack = new Stack<>(); + HashMap entryToFullPath = new HashMap<>(); + if (basePath.isEmpty()) basePath = channelSftp.pwd(); + if(!basePath.endsWith("/")) basePath += "/"; + + Vector fileVector = channelSftp.ls(basePath); + for (ChannelSftp.LsEntry curr : fileVector) { + entryToFullPath.put(curr, basePath + curr.getFilename()); + traversalStack.add(curr); + } + + + while (!traversalStack.isEmpty()) { + ChannelSftp.LsEntry curr = traversalStack.pop(); + String fullPath = entryToFullPath.remove(curr); + if (curr.getFilename().equals(".") || curr.getFilename().equals("..")) { //skip these two + continue; + } + + // if dir skip, if file add to list + if (!curr.getAttrs().isDir()) { + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setPath(fullPath); + fileInfo.setId(curr.getFilename()); + fileInfo.setSize(curr.getAttrs().getSize()); + destFilesList.add(fileInfo); + } + } + return destFilesList; + } } From 35d432eedbaf7da5185a1bb75c972d757b1e8519 Mon Sep 17 00:00:00 2001 From: Abel Jacob Date: Sat, 2 Dec 2023 17:18:45 -0500 Subject: [PATCH 2/4] overwrite flag initial changes for HTTP, Box, DropBox and Gdrive --- .../scheduler/services/RequestModifier.java | 12 ++++---- .../services/expanders/BoxExpander.java | 28 +++++++++++++++++++ .../services/expanders/DropBoxExpander.java | 15 ++++++++++ .../services/expanders/GDriveExpander.java | 9 ++++++ .../services/expanders/HttpExpander.java | 17 +++++++++++ 5 files changed, 75 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java index 0579983..ae198e5 100644 --- a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java +++ b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java @@ -84,18 +84,18 @@ public List selectAndExpandDestination(TransferJobRequest.Destinatio return sftpExpander.expandedDestFileSystem(destination.getFileDestinationPath()); case http: httpExpander.createClient(destination.getVfsDestCredential()); - return httpExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + return httpExpander.expandedDestFileSystem(destination.getFileDestinationPath()); case box: boxExpander.createClient(destination.getOauthDestCredential()); - return boxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + return boxExpander.expandedDestFileSystem(destination.getFileDestinationPath()); case dropbox: dropBoxExpander.createClient(destination.getOauthDestCredential()); - return dropBoxExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + return dropBoxExpander.expandedDestFileSystem(destination.getFileDestinationPath()); case vfs: return selectedResources; case gdrive: gDriveExpander.createClient(destination.getOauthDestCredential()); - return gDriveExpander.expandedFileSystem(selectedResources, destination.getFileDestinationPath()); + return gDriveExpander.expandedDestFileSystem(destination.getFileDestinationPath()); } return null; @@ -174,14 +174,14 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) { if (transferJobRequest.getOptions().isOverwrite()) s.setInfoList(expandedFiles); else { - //Check files present in destination (given path level only) and remove them from infoList sent from source + //Check files present in destination (given path level only) //can use selectAndExpand but not for vfs List destExpandedFiles = this.selectAndExpandDestination(d, new ArrayList<>()); /* - for each file ID at dest, check if present in source list, if no, no action, if yes, remove that file id from src info list + for each file ID at dest, check if present in source list, if yes, remove that file id from src info list expandedFiles.remove() s.setInfoList(expandedFiles); */ diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java index 168cc76..7e2fe6b 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java @@ -67,6 +67,34 @@ public List expandedFileSystem(List userSelectedResource return transferFiles; } + + public List expandedDestFileSystem(String basePath) { + List destFilesList = new ArrayList<>(); + Stack travStack = new Stack<>();//this will only hold folders to traverse + + try{ + //id = 0 represents root folder in box + BoxFolder temp = new BoxFolder(this.connection, "0"); + travStack.push(temp); + }catch (BoxAPIException ignored){ + logger.info("Tried to open {} as a folder but it did not work", "0"); + } + + + while(!travStack.isEmpty()){ + BoxFolder folder = travStack.pop(); + for(BoxItem.Info child : folder){ + //ignore folders, if a file, add to list + if (child instanceof BoxFile.Info) { + BoxFile.Info fileInfo = (BoxFile.Info) child; + BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID()); + destFilesList.add(boxFileToEntityInfo(boxFile)); + } + } + } + return destFilesList; + } + @Override public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize) { BoxFolder destinationUploadFolder = new BoxFolder(this.connection, basePath); diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java index 2080650..13aba83 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/DropBoxExpander.java @@ -71,6 +71,21 @@ public List expandedFileSystem(List userSelectedResource return expandedFiles; } + public List expandedDestFileSystem(String parentPath) { + Stack traversalQueue = new Stack<>(); + List destFilesList = new ArrayList<>(); + if (parentPath == null || parentPath.isEmpty()) parentPath = ""; + //Expand all the files. + + List resources = listOp(parentPath); + for (Metadata resource : resources) { + if (resource instanceof FileMetadata) { + destFilesList.add(metaDataToFileInfo((FileMetadata) resource)); + } + } + return destFilesList; + } + public EntityInfo metaDataToFileInfo(FileMetadata file) { EntityInfo fileInfo = new EntityInfo(); fileInfo.setSize(file.getSize()); diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java index ab08869..0ab6f2b 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/GDriveExpander.java @@ -73,6 +73,15 @@ public List expandedFileSystem(List userSelectedResource return fileInfoList; } + public List expandedDestFileSystem(String basePath) { + Stack fileListStack = new Stack<>(); + List destFilesList = new ArrayList<>(); + + googleDriveLister(fileListStack, destFilesList, "", ""); + + return destFilesList; + } + private void googleDriveLister(Stack fileListStack, List fileInfoList, String fileQuery, String parentFolderId) { FileList fileList; String pageToken = ""; diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java index b0f66db..6f984f1 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/HttpExpander.java @@ -94,6 +94,23 @@ public List expandedFileSystem(List userSelectedResource return filesToSend; } + @SneakyThrows + public List expandedDestFileSystem(String basePath) { + List destFilesList = new ArrayList<>(); + Stack directoriesToTraverse = new Stack<>(); + if (basePath.isEmpty()) basePath = "/"; + + Document doc = Jsoup.connect(this.credential.getUri() + basePath).get(); + Elements links = doc.select("body a"); + for (Element elem : links) { + //ignore dir, consider all files and add to list + if (!elem.text().endsWith("/")) { + destFilesList.add(fromElement(elem)); + } + } + return destFilesList; + } + public EntityInfo fromElement(Element elem) throws IOException { EntityInfo fileInfo = new EntityInfo(); URL url = new URL(elem.absUrl("href")); From 85033af9692fca9f9b49134f63254d2a1641972a Mon Sep 17 00:00:00 2001 From: Abel Jacob Date: Sun, 3 Dec 2023 23:13:53 -0500 Subject: [PATCH 3/4] overwrite flag changes to compare src and dest files and remove from src list --- .../scheduler/services/RequestModifier.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java index ae198e5..9796e09 100644 --- a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java +++ b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java @@ -174,18 +174,18 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) { if (transferJobRequest.getOptions().isOverwrite()) s.setInfoList(expandedFiles); else { - //Check files present in destination (given path level only) - - //can use selectAndExpand but not for vfs + //Check files present in destination List destExpandedFiles = this.selectAndExpandDestination(d, new ArrayList<>()); - - /* - for each file ID at dest, check if present in source list, if yes, remove that file id from src info list - expandedFiles.remove() + //for each file ID at dest, check if present in source list, if yes, remove that file id from src info list + for (EntityInfo destFile : destExpandedFiles) { + for (EntityInfo srcFile : expandedFiles) { + if (destFile.getId().equals(srcFile.getId())) { + expandedFiles.remove(srcFile); + } + } + } s.setInfoList(expandedFiles); - */ - //TODO: fix box case handling where if no files selected by user (to match other cases) } From f97817a432f10255cdbe65742078c11e6185e26a Mon Sep 17 00:00:00 2001 From: Abel Jacob Date: Sun, 10 Dec 2023 21:59:59 -0500 Subject: [PATCH 4/4] Change handling for boxExpander for when user does not select any files --- .../scheduler/services/RequestModifier.java | 2 - .../services/expanders/BoxExpander.java | 58 ++++++++++--------- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java index 9796e09..d7d29f3 100644 --- a/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java +++ b/src/main/java/com/onedatashare/scheduler/services/RequestModifier.java @@ -186,8 +186,6 @@ public TransferJobRequest createRequest(RequestFromODS odsTransferRequest) { } } s.setInfoList(expandedFiles); - - //TODO: fix box case handling where if no files selected by user (to match other cases) } transferJobRequest.setSource(s); diff --git a/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java b/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java index 7e2fe6b..cebf8fe 100644 --- a/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java +++ b/src/main/java/com/onedatashare/scheduler/services/expanders/BoxExpander.java @@ -30,29 +30,35 @@ public void createClient(EndpointCredential credential) { public List expandedFileSystem(List userSelectedResources, String basePath) { List transferFiles = new ArrayList<>(); Stack travStack = new Stack<>();//this will only hold folders to traverse - if(userSelectedResources.isEmpty()) return new ArrayList<>(); //we need to signal the cancellation of this transferjob request. - for(EntityInfo selectedResource : userSelectedResources){ - boolean isFile = false; - try{ - BoxFile temp = new BoxFile(this.connection, selectedResource.getId()); - transferFiles.add(boxFileToEntityInfo(temp)); - isFile = true; - }catch (BoxAPIException ignored){ - logger.info("Tried to open {} as a file but it did not work", selectedResource.toString()); - isFile = false; - } - if(!isFile){ - try{ - BoxFolder temp = new BoxFolder(this.connection, selectedResource.getId()); - travStack.push(temp); - }catch (BoxAPIException ignored){ - logger.info("Tried to open {} as a folder but it did not work", selectedResource.toString()); + if (userSelectedResources.isEmpty()) { + //id = 0 represents root folder in box + BoxFolder temp = new BoxFolder(this.connection, "0"); + travStack.push(temp); + } else { + for (EntityInfo selectedResource : userSelectedResources) { + boolean isFile = false; + try { + BoxFile temp = new BoxFile(this.connection, selectedResource.getId()); + transferFiles.add(boxFileToEntityInfo(temp)); + isFile = true; + } catch (BoxAPIException ignored) { + logger.info("Tried to open {} as a file but it did not work", selectedResource.toString()); + isFile = false; + } + if (!isFile) { + try { + BoxFolder temp = new BoxFolder(this.connection, selectedResource.getId()); + travStack.push(temp); + } catch (BoxAPIException ignored) { + logger.info("Tried to open {} as a folder but it did not work", selectedResource.toString()); + } } } } - while(!travStack.isEmpty()){ + + while (!travStack.isEmpty()) { BoxFolder folder = travStack.pop(); - for(BoxItem.Info child : folder){ + for (BoxItem.Info child : folder) { if (child instanceof BoxFile.Info) { BoxFile.Info fileInfo = (BoxFile.Info) child; BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID()); @@ -72,18 +78,18 @@ public List expandedDestFileSystem(String basePath) { List destFilesList = new ArrayList<>(); Stack travStack = new Stack<>();//this will only hold folders to traverse - try{ + try { //id = 0 represents root folder in box BoxFolder temp = new BoxFolder(this.connection, "0"); travStack.push(temp); - }catch (BoxAPIException ignored){ + } catch (BoxAPIException ignored) { logger.info("Tried to open {} as a folder but it did not work", "0"); } - while(!travStack.isEmpty()){ + while (!travStack.isEmpty()) { BoxFolder folder = travStack.pop(); - for(BoxItem.Info child : folder){ + for (BoxItem.Info child : folder) { //ignore folders, if a file, add to list if (child instanceof BoxFile.Info) { BoxFile.Info fileInfo = (BoxFile.Info) child; @@ -98,10 +104,10 @@ public List expandedDestFileSystem(String basePath) { @Override public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize) { BoxFolder destinationUploadFolder = new BoxFolder(this.connection, basePath); - for(EntityInfo entityInfo : expandedFiles){ - if(entityInfo.getSize() < 1024*1024*20){ + for (EntityInfo entityInfo : expandedFiles) { + if (entityInfo.getSize() < 1024 * 1024 * 20) { entityInfo.setChunkSize(Math.toIntExact(entityInfo.getSize())); - }else{ + } else { BoxFileUploadSession.Info uploadSession = destinationUploadFolder.createUploadSession(entityInfo.getId(), entityInfo.getSize()); entityInfo.setChunkSize(uploadSession.getPartSize()); }