diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/EntityInfo.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/EntityInfo.java index 433514c..1bc3822 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/EntityInfo.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/EntityInfo.java @@ -17,6 +17,10 @@ public class EntityInfo { private long size; private int chunkSize; + private String name; + private String parent; + private String checksum; + @Override public String toString(){ ObjectMapper objectMapper = new ObjectMapper(); diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java index 3c8845a..d60e34e 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java @@ -13,4 +13,18 @@ public EndpointCredential(String accountId){ this.accountId = accountId; } + public static AccountEndpointCredential getAccountCredential(EndpointCredential endpointCredential){ + if(endpointCredential instanceof AccountEndpointCredential){ + return (AccountEndpointCredential) endpointCredential; + }else{ + return null; + } + } + public static OAuthEndpointCredential getOAuthCredential(EndpointCredential endpointCredential){ + if(endpointCredential instanceof OAuthEndpointCredential){ + return (OAuthEndpointCredential) endpointCredential; + }else{ + return null; + } + } } \ No newline at end of file diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/CarbonJobMeasure.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/CarbonJobMeasure.java index 6317b65..9f7220c 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/CarbonJobMeasure.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/CarbonJobMeasure.java @@ -14,6 +14,7 @@ import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -42,6 +43,9 @@ public class CarbonJobMeasure { @Value("${ods.user}") private String odsUser; + @Autowired + private ExpanderService expanderService; + public CarbonJobMeasure(IMap carbonIntensityMap, IMap fileTransferScheduleMap, PmeterParser pmeterParser, ObjectMapper objectMapper) { this.carbonIntensityMap = carbonIntensityMap; this.fileTransferScheduleMap = fileTransferScheduleMap; @@ -61,7 +65,8 @@ public List getPotentialJobsFromMap() { Collection jsonJobs = this.fileTransferScheduleMap.values(potentialJobs); return jsonJobs.stream().map(hazelcastJsonValue -> { try { - return this.objectMapper.readValue(hazelcastJsonValue.getValue(), TransferJobRequest.class); + TransferJobRequest job = this.objectMapper.readValue(hazelcastJsonValue.getValue(), TransferJobRequest.class); + return expanderService.expandFiles(job); } catch (JsonProcessingException e) { logger.error("Json Processing Exception: {}\n With message: {}", e, e.getMessage()); } diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/DestinationChunkSize.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/DestinationChunkSize.java new file mode 100644 index 0000000..e6a313b --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/DestinationChunkSize.java @@ -0,0 +1,25 @@ +package org.onedatashare.transferservice.odstransferservice.service; + +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; + +import java.util.List; + +public abstract class DestinationChunkSize { + + /** + * A class should override this if that protocol needs to get the chunkSize determined by Destination + * @param expandedFiles + * @param basePath + * @return + */ + public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize){ + if(userChunkSize > 15000000){ + userChunkSize = 14900000; + } + for(EntityInfo fileInfo : expandedFiles){ + fileInfo.setChunkSize(userChunkSize); + } + return expandedFiles; + } + +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/ExpanderService.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/ExpanderService.java new file mode 100644 index 0000000..bf138b2 --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/ExpanderService.java @@ -0,0 +1,62 @@ +package org.onedatashare.transferservice.odstransferservice.service; + +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.TransferJobRequest; +import org.onedatashare.transferservice.odstransferservice.service.expanders.*; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class ExpanderService { + + public TransferJobRequest expandFiles(TransferJobRequest job){ + TransferJobRequest.Source source = job.getSource(); + switch(source.getType()){ + case vfs -> { + VfsExpander vfsExpander = new VfsExpander(); + List fileInfo = vfsExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + + case http -> { + HttpExpander httpExpander = new HttpExpander(); + List fileInfo = httpExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + + case box -> { + BoxExpander boxExpander = new BoxExpander(); + List fileInfo = boxExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + case dropbox -> { + DropBoxExpander dropBoxExpander = new DropBoxExpander(); + List fileInfo = dropBoxExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + case gdrive -> { + GDriveExpander gDriveExpander = new GDriveExpander(); + List fileInfo = gDriveExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + case s3 -> { + S3Expander s3Expander = new S3Expander(); + List fileInfo = s3Expander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + + } + case sftp -> { + SFTPExpander sftpExpander = new SFTPExpander(); + List fileInfo = sftpExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + case ftp -> { + FTPExpander ftpExpander = new FTPExpander(); + List fileInfo = ftpExpander.expandedFileSystem(source.getInfoList(), source.getFileSourcePath()); + job.getSource().setInfoList(fileInfo); + } + } + return job; +} +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/BoxExpander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/BoxExpander.java new file mode 100644 index 0000000..ca631c9 --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/BoxExpander.java @@ -0,0 +1,97 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import com.box.sdk.*; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +@Component +public class BoxExpander extends DestinationChunkSize implements FileExpander { + + BoxAPIConnection connection; + Logger logger = LoggerFactory.getLogger(BoxExpander.class); + + @Override + public void createClient(EndpointCredential credential) { + OAuthEndpointCredential oAuthEndpointCredential = EndpointCredential.getOAuthCredential(credential); + connection = new BoxAPIConnection(oAuthEndpointCredential.getToken()); + } + + @Override + public List expandedFileSystem(List userSelectedResources, String basePath) { + List transferFiles = new ArrayList<>(); + Stack travStack = new Stack<>();//this will only hold folders to traverse + if(userSelectedResources.isEmpty()) return new ArrayList<>(); //we need to signal the cancellation of this transferjob request. + for(EntityInfo selectedResource : userSelectedResources){ + boolean isFile = false; + try{ + BoxFile temp = new BoxFile(this.connection, selectedResource.getId()); + transferFiles.add(boxFileToEntityInfo(temp)); + isFile = true; + }catch (BoxAPIException ignored){ + logger.info("Tried to open {} as a file but it did not work", selectedResource.toString()); + isFile = false; + } + if(!isFile){ + try{ + BoxFolder temp = new BoxFolder(this.connection, selectedResource.getId()); + travStack.push(temp); + }catch (BoxAPIException ignored){ + logger.info("Tried to open {} as a folder but it did not work", selectedResource.toString()); + } + } + } + while(!travStack.isEmpty()){ + BoxFolder folder = travStack.pop(); + for(BoxItem.Info child : folder){ + if (child instanceof BoxFile.Info) { + BoxFile.Info fileInfo = (BoxFile.Info) child; + BoxFile boxFile = new BoxFile(this.connection, fileInfo.getID()); + transferFiles.add(boxFileToEntityInfo(boxFile)); + } else if (child instanceof BoxFolder.Info) { + BoxFolder.Info folderInfo = (BoxFolder.Info) child; + BoxFolder childFolder = new BoxFolder(this.connection, folderInfo.getID()); + travStack.push(childFolder); + } + } + } + return transferFiles; + } + + @Override + public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize) { + BoxFolder destinationUploadFolder = new BoxFolder(this.connection, basePath); + for(EntityInfo entityInfo : expandedFiles){ + if(entityInfo.getSize() < 1024*1024*20){ + entityInfo.setChunkSize(Math.toIntExact(entityInfo.getSize())); + }else{ + BoxFileUploadSession.Info uploadSession = destinationUploadFolder.createUploadSession(entityInfo.getId(), entityInfo.getSize()); + entityInfo.setChunkSize(uploadSession.getPartSize()); + } + } + return expandedFiles; + } + + + public EntityInfo boxFileToEntityInfo(BoxFile boxFile) { + BoxFile.Info boxFileInfo = boxFile.getInfo(); + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setId(boxFileInfo.getID()); + fileInfo.setName(boxFileInfo.getName()); + //todo - should this be entire path or just parent id? + fileInfo.setPath(boxFileInfo.getParent().getID()); + fileInfo.setSize(boxFileInfo.getSize()); + //todo - check if etag or sha1 + fileInfo.setChecksum(boxFileInfo.getSha1()); + fileInfo.setParent(boxFileInfo.getParent().getID()); + return fileInfo; + } +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/DropBoxExpander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/DropBoxExpander.java new file mode 100644 index 0000000..56eac5c --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/DropBoxExpander.java @@ -0,0 +1,107 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import com.dropbox.core.DbxException; +import com.dropbox.core.DbxRequestConfig; +import com.dropbox.core.v2.DbxClientV2; +import com.dropbox.core.v2.files.FileMetadata; +import com.dropbox.core.v2.files.FolderMetadata; +import com.dropbox.core.v2.files.Metadata; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Stack; + +@Component +public class DropBoxExpander extends DestinationChunkSize implements FileExpander { + + private DbxClientV2 client; + + @Value("${dropbox.identifier}") + private String odsClientID = "OneDataShare-DIDCLab"; + + + @Override + public void createClient(EndpointCredential credential) { + DbxRequestConfig config = DbxRequestConfig.newBuilder(odsClientID).build(); + this.client = new DbxClientV2(config, ((EndpointCredential.getOAuthCredential(credential))).getToken()); + } + + @Override + public List expandedFileSystem(List userSelectedResources, String parentPath) { + Stack traversalQueue = new Stack<>(); + List expandedFiles = new ArrayList<>(); + if (parentPath == null || parentPath.isEmpty()) parentPath = ""; + //Expand all the files. + if (userSelectedResources == null || userSelectedResources.isEmpty()) { + List resources = listOp(parentPath); + for (Metadata resource : resources) { + if (resource instanceof FileMetadata) { + expandedFiles.add(metaDataToFileInfo((FileMetadata) resource)); + } else if (resource instanceof FolderMetadata) { + traversalQueue.push(resource); + } + } + } else { + for (EntityInfo fileInfo : userSelectedResources) { + List dropBoxFiles = listOp(fileInfo.getPath()); + dropBoxFiles.forEach(metadata -> { + if (metadata instanceof FileMetadata) { + expandedFiles.add(metaDataToFileInfo((FileMetadata) metadata)); + } else if (metadata instanceof FolderMetadata) { + traversalQueue.push(metadata); + } + }); + } + } + while (!traversalQueue.isEmpty()) { + FolderMetadata folderMetadata = (FolderMetadata) traversalQueue.pop(); + List folderList = listOp(folderMetadata.getPathLower()); + for (Metadata res : folderList) { + if (res instanceof FileMetadata) { + expandedFiles.add(metaDataToFileInfo((FileMetadata) res)); + } else if (res instanceof FolderMetadata) { + traversalQueue.push(res); + } + } + } + return expandedFiles; + } + + public EntityInfo metaDataToFileInfo(FileMetadata file) { + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setSize(file.getSize()); + fileInfo.setId(file.getId()); + fileInfo.setPath(file.getPathLower()); + return fileInfo; + } + + public List listOp(String path) { + try { + return this.client.files().listFolderBuilder(path).start().getEntries(); + } catch (DbxException e) {} + try{ + return Collections.singletonList(this.client.files().getMetadata(path)); + } catch (DbxException e){} + return new ArrayList<>(); + } + + @Override + public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize){ + for(EntityInfo fileInfo : expandedFiles){ + if(fileInfo.getSize() < 8L << 20){ + fileInfo.setChunkSize(Long.valueOf(fileInfo.getSize()).intValue()); + }else if(userChunkSize < 4L << 20){ + fileInfo.setChunkSize(10000000); + }else{ + fileInfo.setChunkSize(userChunkSize); + } + } + return expandedFiles; + } +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/FTPExpander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/FTPExpander.java new file mode 100644 index 0000000..465894f --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/FTPExpander.java @@ -0,0 +1,101 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import lombok.SneakyThrows; +import org.apache.commons.vfs2.*; +import org.apache.commons.vfs2.auth.StaticUserAuthenticator; +import org.apache.commons.vfs2.impl.DefaultFileSystemConfigBuilder; +import org.apache.commons.vfs2.provider.ftp.FtpFileSystemConfigBuilder; +import org.apache.commons.vfs2.provider.ftp.FtpFileType; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.net.URI; +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; +import java.util.Stack; + +@Service +public class FTPExpander extends DestinationChunkSize implements FileExpander { + + AccountEndpointCredential vfsCredential; + List infoList; + static final Logger logger = LoggerFactory.getLogger(FTPExpander.class); + FileSystemOptions options; + + public FTPExpander(){ + this.options = generateOpts(); + } + + + + public static FileSystemOptions generateOpts() { + FileSystemOptions opts = new FileSystemOptions(); + FtpFileSystemConfigBuilder.getInstance().setPassiveMode(opts, true); + FtpFileSystemConfigBuilder.getInstance().setFileType(opts, FtpFileType.BINARY); + FtpFileSystemConfigBuilder.getInstance().setAutodetectUtf8(opts, true); + FtpFileSystemConfigBuilder.getInstance().setControlEncoding(opts, "UTF-8"); + return opts; + } + + @Override + public void createClient(EndpointCredential credential) { + this.vfsCredential = EndpointCredential.getAccountCredential(credential); + StaticUserAuthenticator auth = new StaticUserAuthenticator(null, this.vfsCredential.getUsername(), this.vfsCredential.getSecret()); + try { + DefaultFileSystemConfigBuilder.getInstance().setUserAuthenticator(options, auth); + } catch (FileSystemException e) { + e.printStackTrace(); + } + } + + @SneakyThrows + @Override + public List expandedFileSystem(List userSelectedResources, String basePath) { + this.infoList = userSelectedResources; + List filesToTransferList = new LinkedList<>(); + Stack traversalStack = new Stack<>(); + FileSystemManager fsm = VFS.getManager(); + if(basePath.isEmpty() || basePath == null || !basePath.endsWith("/")) basePath += "/"; + if(infoList.isEmpty()){ + FileObject obj = fsm.resolveFile(this.vfsCredential.getUri() + basePath, this.options); + traversalStack.push(obj); + }else{ + for (EntityInfo e : this.infoList) { + logger.info(this.vfsCredential.getUri() + basePath + e.getId()); + FileObject fObject = fsm.resolveFile(this.vfsCredential.getUri() + basePath + e.getId(), this.options); + traversalStack.push(fObject); + } + } + for (int files = Integer.MAX_VALUE; files > 0 && !traversalStack.isEmpty(); --files) { + FileObject curr = traversalStack.pop(); + FileName fileName = curr.getName(); + URI uri = URI.create(fileName.getURI()); + logger.info(uri.toString()); + if (curr.getType() == FileType.FOLDER) { + traversalStack.addAll(Arrays.asList(curr.getChildren())); + //Add empty folders as well + if (curr.getChildren().length == 0) { + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setId(fileName.getBaseName()); + fileInfo.setPath(uri.getPath()); + filesToTransferList.add(fileInfo); + } + } else if (curr.getType() == FileType.FILE) { + + //filePath = curr.getPublicURIString().substring(this.vfsCredential.getUri().length()+basePath.length()); + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setId(curr.getName().getBaseName()); + fileInfo.setPath(uri.getPath()); + fileInfo.setSize(curr.getContent().getSize()); + filesToTransferList.add(fileInfo); + } + } + return filesToTransferList; + } +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/GDriveExpander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/GDriveExpander.java new file mode 100644 index 0000000..a37de29 --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/GDriveExpander.java @@ -0,0 +1,109 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.File; +import com.google.api.services.drive.model.FileList; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + +@Service +public class GDriveExpander extends DestinationChunkSize implements FileExpander { + + @Value("${gdrive.client.id}") + private String gDriveClientId; + + @Value("${gdrive.client.secret}") + private String gDriveClientSecret; + + @Value("${gdrive.appname}") + private String gdriveAppName; + + private Drive client; + + @Override + public void createClient(EndpointCredential credential) { + OAuthEndpointCredential oauthCred = EndpointCredential.getOAuthCredential(credential); + try { + NetHttpTransport transport = GoogleNetHttpTransport.newTrustedTransport(); + GoogleCredential credential1 = new GoogleCredential.Builder().setJsonFactory(GsonFactory.getDefaultInstance()) + .setClientSecrets(gDriveClientId, gDriveClientSecret) + .setTransport(transport).build(); + credential1.setAccessToken(oauthCred.getToken()); + credential1.setRefreshToken(oauthCred.getRefreshToken()); + this.client = new Drive.Builder(transport, GsonFactory.getDefaultInstance(), credential1) + .setApplicationName(gdriveAppName) + .build(); + } catch (GeneralSecurityException | IOException e) { + e.printStackTrace(); + } + } + + @Override + public List expandedFileSystem(List userSelectedResources, String basePath) { + Stack fileListStack = new Stack<>(); + List fileInfoList = new ArrayList<>(); + if(userSelectedResources.isEmpty()){ + googleDriveLister(fileListStack, fileInfoList, "", ""); + }else{ + for(EntityInfo fileInfo : userSelectedResources){ + String fileQuery = "'" + fileInfo.getId() + "' in parents and trashed=false"; + googleDriveLister(fileListStack, fileInfoList, fileQuery, fileInfo.getId()); + } + } + while(!fileListStack.isEmpty()){ + File file = fileListStack.pop(); + String fileQuery = "'" + file.getId() + "' in parents and mimeType != 'application/vnd.google-apps.folder' and trashed=false"; + googleDriveLister(fileListStack, fileInfoList, fileQuery, file.getId()); + } + return fileInfoList; + } + + private void googleDriveLister(Stack fileListStack, List fileInfoList, String fileQuery, String parentFolderId) { + FileList fileList; + String pageToken = ""; + try { + do { + fileList = this.client.files().list() + .setQ(fileQuery) + .setFields("nextPageToken, files(id, name, parents, size, mimeType)") + .setPageToken(pageToken) + .execute(); + for(File file : fileList.getFiles()){ + if(file.getId().equals(parentFolderId)){ + continue; + } + if(file.getMimeType().equals("application/vnd.google-apps.folder")){ + fileListStack.add(file); + }else{ + fileInfoList.add(googleFileToEntityInfo(file)); + } + } + pageToken = fileList.getNextPageToken(); + } while (pageToken != null); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private EntityInfo googleFileToEntityInfo(File googleFile){ + EntityInfo entityInfo = new EntityInfo(); + entityInfo.setId(googleFile.getId()); + entityInfo.setSize(googleFile.getSize()); + entityInfo.setPath(String.valueOf(googleFile.getParents())); + return entityInfo; + } +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/S3Expander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/S3Expander.java new file mode 100644 index 0000000..97bbfde --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/S3Expander.java @@ -0,0 +1,106 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.apache.commons.lang.StringUtils; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.springframework.stereotype.Component; + +import java.util.LinkedList; +import java.util.List; + +@Component +public class S3Expander extends DestinationChunkSize implements FileExpander { + + AmazonS3 s3Client; + String[] regionAndBucket; + + @Override + public void createClient(EndpointCredential cred) { + AccountEndpointCredential credential = EndpointCredential.getAccountCredential(cred); + this.regionAndBucket = credential.getUri().split(":::"); + AWSCredentials credentials = new BasicAWSCredentials(credential.getUsername(), credential.getSecret()); + this.s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(credentials)) + .withRegion(regionAndBucket[0]) + .build(); + } + + @Override + public List expandedFileSystem(List userSelectedResources, String basePath) { + List traversedFiles = new LinkedList<>(); + //trim leading forward slashes from base path (s3 doesn't recognise it as root) + basePath = StringUtils.stripStart(basePath, "/"); + if(userSelectedResources.isEmpty()){//expand the whole bucket relative to the basePath + ListObjectsV2Result result = this.s3Client.listObjectsV2(createSkeletonPerResource(basePath)); + traversedFiles.addAll(convertV2ResultToEntityInfoList(result)); + } + for(EntityInfo userSelectedResource: userSelectedResources){ + //we have a folder/prefix for s3 + if (userSelectedResource.getPath().endsWith("/")){ + ListObjectsV2Request req = createSkeletonPerResource(userSelectedResource.getPath()); + ListObjectsV2Result res = this.s3Client.listObjectsV2(req); + for(S3ObjectSummary obj : res.getObjectSummaries()){ + if(obj.getKey().endsWith("/")) continue; + EntityInfo entityInfo = new EntityInfo(); + entityInfo.setId(obj.getKey()); + entityInfo.setPath(obj.getKey()); + entityInfo.setSize(obj.getSize()); + traversedFiles.add(entityInfo); + } + // the case where the user selected a file + } else if(this.s3Client.doesObjectExist(this.regionAndBucket[1], userSelectedResource.getPath())){ + ObjectMetadata metadata = this.s3Client.getObjectMetadata(this.regionAndBucket[1],userSelectedResource.getPath()); + userSelectedResource.setSize(metadata.getContentLength()); + traversedFiles.add(userSelectedResource); + } + } + + return traversedFiles; + } + + public List convertV2ResultToEntityInfoList(ListObjectsV2Result result){ + List traversedFiles = new LinkedList<>(); + for(S3ObjectSummary fileInfo : result.getObjectSummaries()){ + EntityInfo entityInfo = new EntityInfo(); + entityInfo.setId(fileInfo.getKey()); + entityInfo.setPath(fileInfo.getKey()); + entityInfo.setSize(fileInfo.getSize()); + traversedFiles.add(entityInfo); + } + return traversedFiles; + } + + public ListObjectsV2Request createSkeletonPerResource(String path){ + if(path.isEmpty()){ + return new ListObjectsV2Request() + .withBucketName(regionAndBucket[1]); + }else{ + return new ListObjectsV2Request() + .withBucketName(regionAndBucket[1]) + .withPrefix(path); + } + } + + @Override + public List destinationChunkSize(List expandedFiles, String basePath, Integer userChunkSize) { + for (EntityInfo fileInfo : expandedFiles) { + if(userChunkSize < 5000000){ + fileInfo.setChunkSize(10000000); + }else{ + fileInfo.setChunkSize(userChunkSize); + } + } + return expandedFiles; + } +} diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/SFTPExpander.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/SFTPExpander.java new file mode 100644 index 0000000..f5bc27e --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/expanders/SFTPExpander.java @@ -0,0 +1,129 @@ +package org.onedatashare.transferservice.odstransferservice.service.expanders; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import lombok.SneakyThrows; +import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; +import org.onedatashare.transferservice.odstransferservice.model.credential.AccountEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.model.credential.EndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.DestinationChunkSize; +import org.springframework.stereotype.Component; + +import java.util.*; + +@Component +public class SFTPExpander extends DestinationChunkSize implements FileExpander { + + AccountEndpointCredential credential; + ChannelSftp channelSftp; + List infoList; + + @SneakyThrows + @Override + public void createClient(EndpointCredential cred) { + this.credential = EndpointCredential.getAccountCredential(cred); + Session jschSession = null; + boolean connected = false; + JSch jsch = new JSch(); + String[] typeAndUri = credential.getUri().split("://"); //split out the sftp partof the string + String host = ""; + String port = "22"; + if(typeAndUri[1].contains(":")){ + String[] hostAndPort = typeAndUri[1].split(":"); + host = hostAndPort[0]; + port = hostAndPort[1]; + } + try { + jsch.addIdentity("randomName", credential.getSecret().getBytes(), null, null); + jschSession = jsch.getSession(credential.getUsername(), host, Integer.parseInt(port)); + jschSession.setConfig("StrictHostKeyChecking", "no"); + jschSession.connect(); + connected = true; + } catch (JSchException ignored) { + connected = false; + } + if (!connected) { + try { + jschSession = jsch.getSession(credential.getUsername(), host, Integer.parseInt(port)); + jschSession.setConfig("StrictHostKeyChecking", "no"); + jschSession.setPassword(credential.getSecret()); + jschSession.connect(); + connected = true; + } catch (JSchException ignored) { + connected = false; + } + } + assert jschSession != null; +// if (!jschSession.isConnected()) { +// throw new JSchException("Unable to authenticate with the password/pem file"); +// } + ChannelSftp channelSftp = (ChannelSftp) jschSession.openChannel("sftp"); + channelSftp.connect(); + this.channelSftp = channelSftp; + } + + @SneakyThrows + @Override + public List expandedFileSystem(List userSelectedResources, String basePath) { + //if(!basePath.endsWith("/")) basePath +="/"; + this.infoList = userSelectedResources; + List filesToTransferList = new LinkedList<>(); + Stack traversalStack = new Stack<>(); + HashMap entryToFullPath = new HashMap<>(); + if (basePath.isEmpty()) basePath = channelSftp.pwd(); + if(!basePath.endsWith("/")) basePath += "/"; + if (userSelectedResources.isEmpty()) { + Vector fileVector = channelSftp.ls(basePath); + for (ChannelSftp.LsEntry curr : fileVector) { + entryToFullPath.put(curr, basePath + curr.getFilename()); + traversalStack.add(curr); + } + } else { + for (EntityInfo e : userSelectedResources) { + String path = basePath + e.getPath(); + Vector fileVector = channelSftp.ls(path); + if(fileVector.size() == 1) { + ChannelSftp.LsEntry curr = fileVector.get(0); + entryToFullPath.put(curr, path); + traversalStack.add(fileVector.get(0)); + }else{ + for (ChannelSftp.LsEntry curr : fileVector) { + entryToFullPath.put(curr, path + curr.getFilename()); + traversalStack.add(curr); + } + } + } + } + + while (!traversalStack.isEmpty()) { + ChannelSftp.LsEntry curr = traversalStack.pop(); + String fullPath = entryToFullPath.remove(curr); + if (curr.getFilename().equals(".") || curr.getFilename().equals("..")) { //skip these two + continue; + } + if (curr.getAttrs().isDir()) { + Vector children = channelSftp.ls(fullPath); + if (children.size() == 0) {//this should include the empty directory + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setId(curr.getFilename()); + fileInfo.setSize(curr.getAttrs().getSize()); + fileInfo.setPath(fullPath); + } else { + for (ChannelSftp.LsEntry f : children) { + entryToFullPath.put(f, fullPath + "/" + f.getFilename()); + traversalStack.add(f); + } + } + } else if (!curr.getAttrs().isDir()) { + EntityInfo fileInfo = new EntityInfo(); + fileInfo.setPath(fullPath); + fileInfo.setId(curr.getFilename()); + fileInfo.setSize(curr.getAttrs().getSize()); + filesToTransferList.add(fileInfo); + } + } + return filesToTransferList; + } +}