From b7d57749db153e20cae7e4fced3c24bec14f2cf9 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:40:05 -0400 Subject: [PATCH 01/12] reauthentication changes --- .../odstransferservice/config/DataSourceConfig.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/config/DataSourceConfig.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/config/DataSourceConfig.java index 723a4a6d..6ee5252d 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/config/DataSourceConfig.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/config/DataSourceConfig.java @@ -4,9 +4,11 @@ import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.repository.support.JobRepositoryFactoryBean; import org.springframework.beans.factory.annotation.Value; +import org.springframework.cloud.client.loadbalancer.LoadBalanced; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.web.client.RestTemplate; import javax.sql.DataSource; @@ -31,5 +33,12 @@ public JobRepository roachRepository(DataSource dataSource, PlatformTransactionM factory.setMaxVarCharLength(1000); return factory.getObject(); } + + @LoadBalanced + @Bean + public RestTemplate credentialTemplate(){ + RestTemplate credenTemplate = new RestTemplate(); + return credenTemplate; + } } From e4b9ebdefbcba7dd55e556da9843117f58598671 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:40:36 -0400 Subject: [PATCH 02/12] reauthentication changes --- .../odstransferservice/constant/ODSConstants.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/constant/ODSConstants.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/constant/ODSConstants.java index e465a825..87ed892c 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/constant/ODSConstants.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/constant/ODSConstants.java @@ -17,6 +17,10 @@ public class ODSConstants { public static final String FILE_SIZE = "fileSize"; public static final String FILE_PATH = "filePath"; public static final String FILE_ID = "file_id"; + + public static final String DROPBOX= "dropbox"; + public static final String GOOGLEDRIVE = "gdrive"; + public static final String BOX= "box"; public static final String TIME = "time"; public static final String SOURCE_ACCOUNT_ID_PASS = "sourceAccountIdPass"; public static final String SOURCE_HOST = "sourceURI"; From 733eef36ad35e63ce8292dd08237c12e6c2b2e00 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:40:48 -0400 Subject: [PATCH 03/12] reauthentication changes --- .../model/credential/EndpointCredential.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java index 3a029fdb..fd3b54fe 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/model/credential/EndpointCredential.java @@ -1,8 +1,12 @@ package org.onedatashare.transferservice.odstransferservice.model.credential; +import lombok.Getter; + /** * Base class for storing one user credential */ + +@Getter public class EndpointCredential{ protected String accountId; From 7eb51be1155946de0b51bb921ab0101010c1ff26 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:41:07 -0400 Subject: [PATCH 04/12] reauthentication changes --- .../odstransferservice/service/JobControl.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java index 13d6092c..33dcefe1 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/JobControl.java @@ -105,6 +105,9 @@ public class JobControl extends DefaultBatchConfigurer { @Autowired JobRepository roachRepository; + @Autowired + AuthenticateCredentials authenticateCredentials; + @Lazy @Bean @@ -161,18 +164,18 @@ protected AbstractItemCountingItemStreamItemReader getRightReader(End amazonS3Reader.setPool(connectionBag.getS3ReaderPool()); return amazonS3Reader; case box: - BoxReader boxReader = new BoxReader(request.getSource().getOauthSourceCredential(), fileInfo); + BoxReader boxReader = new BoxReader(request.getSource().getOauthSourceCredential(), fileInfo,this.authenticateCredentials); boxReader.setMaxRetry(this.request.getOptions().getRetry()); return boxReader; case dropbox: - DropBoxReader dropBoxReader = new DropBoxReader(request.getSource().getOauthSourceCredential(), fileInfo); + DropBoxReader dropBoxReader = new DropBoxReader(request.getSource().getOauthSourceCredential(), fileInfo, this.authenticateCredentials); return dropBoxReader; case scp: SCPReader reader = new SCPReader(fileInfo); reader.setPool(connectionBag.getSftpReaderPool()); return reader; case gdrive: - GDriveReader dDriveReader = new GDriveReader(request.getSource().getOauthSourceCredential(), fileInfo); + GDriveReader dDriveReader = new GDriveReader(request.getSource().getOauthSourceCredential(), fileInfo, this.authenticateCredentials); return dDriveReader; } return null; @@ -208,11 +211,11 @@ protected ItemWriter getRightWriter(EndpointType type, EntityInfo fil BoxWriterSmallFile boxWriterSmallFile = new BoxWriterSmallFile(request.getDestination().getOauthDestCredential(), fileInfo, this.metricsCollector, this.influxCache); return boxWriterSmallFile; } else { - BoxWriterLargeFile boxWriterLargeFile = new BoxWriterLargeFile(request.getDestination().getOauthDestCredential(), fileInfo, this.metricsCollector, this.influxCache); + BoxWriterLargeFile boxWriterLargeFile = new BoxWriterLargeFile(request.getDestination().getOauthDestCredential(), fileInfo, this.metricsCollector, this.influxCache, this.authenticateCredentials); return boxWriterLargeFile; } case dropbox: - DropBoxChunkedWriter dropBoxChunkedWriter = new DropBoxChunkedWriter(request.getDestination().getOauthDestCredential(), this.metricsCollector, this.influxCache); + DropBoxChunkedWriter dropBoxChunkedWriter = new DropBoxChunkedWriter(request.getDestination().getOauthDestCredential(), this.metricsCollector, this.influxCache, this.authenticateCredentials); return dropBoxChunkedWriter; case scp: SCPWriter scpWriter = new SCPWriter(fileInfo, this.metricsCollector, this.influxCache); @@ -223,7 +226,7 @@ protected ItemWriter getRightWriter(EndpointType type, EntityInfo fil GDriveSimpleWriter writer = new GDriveSimpleWriter(request.getDestination().getOauthDestCredential(),fileInfo); return writer; }else{ - GDriveResumableWriter writer = new GDriveResumableWriter(request.getDestination().getOauthDestCredential(),fileInfo); + GDriveResumableWriter writer = new GDriveResumableWriter(request.getDestination().getOauthDestCredential(),fileInfo, this.authenticateCredentials); writer.setPool(connectionBag.getGoogleDriveWriterPool()); return writer; } From 77f7e8ba179ac9c8b1150a3e85ed353b1c9c7fbf Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:41:20 -0400 Subject: [PATCH 05/12] reauthentication changes --- .../service/step/box/BoxReader.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxReader.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxReader.java index ae58c6dc..64e8ace7 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxReader.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxReader.java @@ -2,10 +2,13 @@ import com.box.sdk.BoxAPIConnection; import com.box.sdk.BoxFile; +import com.box.sdk.BoxUser; +import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; import org.onedatashare.transferservice.odstransferservice.model.FilePart; import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.service.FilePartitioner; import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility; import org.slf4j.Logger; @@ -17,6 +20,8 @@ import java.io.ByteArrayOutputStream; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; + public class BoxReader extends AbstractItemCountingItemStreamItemReader { private OAuthEndpointCredential credential; @@ -27,14 +32,24 @@ public class BoxReader extends AbstractItemCountingItemStreamItemReader Date: Sun, 7 May 2023 12:41:32 -0400 Subject: [PATCH 06/12] reauthentication changes --- .../service/step/box/BoxWriterLargeFile.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java index c5a64c9e..cce75746 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java @@ -1,12 +1,11 @@ package org.onedatashare.transferservice.odstransferservice.service.step.box; -import com.box.sdk.BoxAPIConnection; -import com.box.sdk.BoxFileUploadSession; -import com.box.sdk.BoxFileUploadSessionPart; -import com.box.sdk.BoxFolder; +import com.box.sdk.*; +import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.service.InfluxCache; import org.onedatashare.transferservice.odstransferservice.service.cron.MetricsCollector; import org.onedatashare.transferservice.odstransferservice.service.step.ODSBaseWriter; @@ -26,6 +25,7 @@ import java.util.List; import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.DEST_BASE_PATH; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; /** * This class is responsible for writing to Box using the chunked upload approach & small file upload @@ -33,7 +33,7 @@ */ public class BoxWriterLargeFile extends ODSBaseWriter implements ItemWriter { - private final OAuthEndpointCredential credential; + private OAuthEndpointCredential credential; private BoxAPIConnection boxAPIConnection; EntityInfo fileInfo; private HashMap fileMap; @@ -42,8 +42,11 @@ public class BoxWriterLargeFile extends ODSBaseWriter implements ItemWriter(); this.parts = new ArrayList<>(); this.credential = oAuthDestCredential; + this.authenticateCredentials = authenticateCredentials; + BoxUser user = new BoxUser(this.boxAPIConnection, this.credential.getAccountId()); + BoxUser.Info userInfo = user.getInfo("login"); + this.usersEmail = userInfo.getLogin(); } @BeforeStep @@ -58,6 +65,7 @@ public void beforeStep(StepExecution stepExecution) { this.destinationBasePath = stepExecution.getJobParameters().getString(DEST_BASE_PATH); //path to place the files this.boxFolder = new BoxFolder(this.boxAPIConnection, this.destinationBasePath); this.stepExecution = stepExecution; + this.ownerId = stepExecution.getJobParameters().getString(OWNER_ID); } /** @@ -111,6 +119,10 @@ private boolean ready(String fileName) { */ @Override public void write(List items) throws NoSuchAlgorithmException { + if(this.credential.isTokenExpires()){ + this.credential = authenticateCredentials.checkExpiryAndGenerateNew( usersEmail, ODSConstants.BOX, ownerId); + this.boxAPIConnection = new BoxAPIConnection(this.credential.getToken()); + } String fileName = items.get(0).getFileName(); prepareForUpload(fileName); BoxFileUploadSession session = this.fileMap.get(fileName); From 9e4dd518d69769663cd7a2e5db09aac929e87df8 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:41:47 -0400 Subject: [PATCH 07/12] reauthentication changes --- .../service/step/dropbox/DropBoxReader.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxReader.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxReader.java index 519826d5..6f4855cc 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxReader.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxReader.java @@ -4,10 +4,12 @@ import com.dropbox.core.v2.DbxClientV2; import com.dropbox.core.v2.files.DownloadBuilder; import com.dropbox.core.v2.files.Metadata; +import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; import org.onedatashare.transferservice.odstransferservice.model.FilePart; import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.service.FilePartitioner; import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility; import org.slf4j.Logger; @@ -20,13 +22,14 @@ import java.io.ByteArrayOutputStream; import java.nio.file.Paths; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.SOURCE_BASE_PATH; public class DropBoxReader extends AbstractItemCountingItemStreamItemReader { Logger logger = LoggerFactory.getLogger(DropBoxReader.class); private final EntityInfo fileInfo; - private final OAuthEndpointCredential credential; + private OAuthEndpointCredential credential; private String sBasePath; private FilePartitioner partitioner; @@ -34,12 +37,18 @@ public class DropBoxReader extends AbstractItemCountingItemStreamItemReader Date: Sun, 7 May 2023 12:42:00 -0400 Subject: [PATCH 08/12] reauthentication changes --- .../step/dropbox/DropBoxChunkedWriter.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxChunkedWriter.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxChunkedWriter.java index bcd5f546..d895e7e1 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxChunkedWriter.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/dropbox/DropBoxChunkedWriter.java @@ -1,13 +1,17 @@ package org.onedatashare.transferservice.odstransferservice.service.step.dropbox; +import com.box.sdk.BoxUser; import com.dropbox.core.DbxException; import com.dropbox.core.v2.DbxClientV2; import com.dropbox.core.v2.files.CommitInfo; import com.dropbox.core.v2.files.FileMetadata; import com.dropbox.core.v2.files.UploadSessionCursor; +import com.dropbox.core.v2.users.FullAccount; import lombok.SneakyThrows; +import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.service.InfluxCache; import org.onedatashare.transferservice.odstransferservice.service.cron.MetricsCollector; import org.onedatashare.transferservice.odstransferservice.service.step.ODSBaseWriter; @@ -25,10 +29,11 @@ import java.util.List; import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.DEST_BASE_PATH; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; public class DropBoxChunkedWriter extends ODSBaseWriter implements ItemWriter { - private final OAuthEndpointCredential credential; + private OAuthEndpointCredential credential; private String destinationPath; private DbxClientV2 client; String sessionId; @@ -37,10 +42,16 @@ public class DropBoxChunkedWriter extends ODSBaseWriter implements ItemWriter items) throws Exception { + if(this.credential.isTokenExpires()){ + this.credential = authenticateCredentials.checkExpiryAndGenerateNew( usersEmail, ODSConstants.DROPBOX, ownerId); + this.client = new DbxClientV2(ODSUtility.dbxRequestConfig, this.credential.getToken()); + } for(DataChunk chunk : items){ this.client.files().uploadSessionAppendV2(cursor).uploadAndFinish(new ByteArrayInputStream(chunk.getData())); this.cursor = new UploadSessionCursor(sessionId, chunk.getStartPosition() + chunk.getSize()); From 341a560a2f3521dcd8e3954c6b4b5e8c17a9e221 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:42:13 -0400 Subject: [PATCH 09/12] reauthentication changes --- .../step/googleDrive/GDriveReader.java | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveReader.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveReader.java index 9cdf1fe6..998bca99 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveReader.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveReader.java @@ -1,40 +1,58 @@ package org.onedatashare.transferservice.odstransferservice.service.step.googleDrive; +import com.box.sdk.BoxUser; import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.About; import com.google.api.services.drive.model.File; +import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; import org.onedatashare.transferservice.odstransferservice.model.FilePart; import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.service.FilePartitioner; import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.annotation.BeforeStep; import org.springframework.batch.item.support.AbstractItemCountingItemStreamItemReader; import org.springframework.util.ClassUtils; import java.io.ByteArrayOutputStream; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; + public class GDriveReader extends AbstractItemCountingItemStreamItemReader { Logger logger = LoggerFactory.getLogger(GDriveReader.class); private final EntityInfo fileInfo; - private final OAuthEndpointCredential credential; + private OAuthEndpointCredential credential; private final FilePartitioner partitioner; private Drive client; private String fileName; private File file; - public GDriveReader(OAuthEndpointCredential credential, EntityInfo fileInfo){ + String usersEmail; + String ownerId; + + private AuthenticateCredentials authenticateCredentials; + + public GDriveReader(OAuthEndpointCredential credential, EntityInfo fileInfo,AuthenticateCredentials authenticateCredentials){ this.credential = credential; this.fileInfo = fileInfo; this.partitioner = new FilePartitioner(fileInfo.getChunkSize()); this.setName(ClassUtils.getShortName(GDriveReader.class)); logger.info(credential.toString()); + this.authenticateCredentials = authenticateCredentials; } @Override protected DataChunk doRead() throws Exception { + if(this.credential.isTokenExpires()){ + this.credential = authenticateCredentials.checkExpiryAndGenerateNew( usersEmail, ODSConstants.GOOGLEDRIVE, ownerId); + doOpen(); + } FilePart filePart = this.partitioner.nextPart(); if(filePart == null || filePart.getSize() == 0) return null; Drive.Files.Get get = this.client.files().get(fileInfo.getId()); @@ -55,6 +73,13 @@ protected void doOpen() throws Exception { this.fileName = file.getName(); this.partitioner.createParts(this.fileInfo.getSize(), this.fileName); logger.info(file.toString()); + About about = client.about().get().execute(); + this.usersEmail = about.getUser().getEmailAddress(); + } + + @BeforeStep + public void beforeStep(StepExecution stepExecution){ + this.ownerId = stepExecution.getJobParameters().getString(OWNER_ID); } @Override From 95aaba0b4293ebc0fb3f12b52d9bc0e1df9ff6bc Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:42:26 -0400 Subject: [PATCH 10/12] reauthentication changes --- .../googleDrive/GDriveResumableWriter.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveResumableWriter.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveResumableWriter.java index ec59c9ff..2518c4cb 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveResumableWriter.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/googleDrive/GDriveResumableWriter.java @@ -1,6 +1,7 @@ package org.onedatashare.transferservice.odstransferservice.service.step.googleDrive; import com.google.api.services.drive.Drive; +import com.google.api.services.drive.model.About; import com.google.api.services.drive.model.File; import org.apache.commons.pool2.ObjectPool; import org.codehaus.jettison.json.JSONException; @@ -11,6 +12,7 @@ import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; import org.onedatashare.transferservice.odstransferservice.pools.GDriveConnectionPool; import org.onedatashare.transferservice.odstransferservice.pools.HttpConnectionPool; +import org.onedatashare.transferservice.odstransferservice.service.AuthenticateCredentials; import org.onedatashare.transferservice.odstransferservice.utility.GDriveHelper; import org.onedatashare.transferservice.odstransferservice.utility.ODSUtility; import org.slf4j.Logger; @@ -36,12 +38,13 @@ import java.util.*; import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.GOOGLE_DRIVE_MIN_BYTES; +import static org.onedatashare.transferservice.odstransferservice.constant.ODSConstants.OWNER_ID; public class GDriveResumableWriter implements ItemWriter,SetPool { Logger logger = LoggerFactory.getLogger(GDriveResumableWriter.class); - private final OAuthEndpointCredential credential; + private OAuthEndpointCredential credential; private final EntityInfo fileInfo; private String basePath; private String fileName; @@ -50,12 +53,18 @@ public class GDriveResumableWriter implements ItemWriter,SetPool { private DataChunk readyChunk; + String usersEmail; + String ownerId; + + private AuthenticateCredentials authenticateCredentials; + private boolean failed; private boolean success; - public GDriveResumableWriter(OAuthEndpointCredential credential, EntityInfo fileInfo) { + public GDriveResumableWriter(OAuthEndpointCredential credential, EntityInfo fileInfo, AuthenticateCredentials authenticateCredentials) { this.credential = credential; this.fileInfo = fileInfo; + this.authenticateCredentials = authenticateCredentials; } @BeforeStep @@ -64,10 +73,11 @@ public void beforeStep(StepExecution stepExecution) throws IOException, GeneralS this.readyChunk = null; this.failed=false; this.success=false; + this.ownerId = stepExecution.getJobParameters().getString(OWNER_ID); } @BeforeWrite - public void beforeWrite(List items) throws IOException, JSONException, InterruptedException { + public void beforeWrite(List items) throws IOException, JSONException, InterruptedException, GeneralSecurityException { if(this.utility == null || this.utility.getSessionUri() == null){ this.logger.debug("Initializing resumable upload"); this.utility = GDriveHelper.builder() @@ -81,11 +91,21 @@ public void beforeWrite(List items) throws IOException, JSO throw new IOException("Unable to get the Location header from google drive. Error response code: "+status); } } + Drive client = ODSUtility.authenticateDriveClient(this.credential); + if(client!=null) { + About about = client.about().get().execute(); + if(about!= null) { + this.usersEmail = about.getUser().getEmailAddress(); + } + } } @Override public void write(List dataChunkList) { try { + if(this.credential.isTokenExpires()){ + this.credential = authenticateCredentials.checkExpiryAndGenerateNew( usersEmail, ODSConstants.GOOGLEDRIVE, ownerId); + } int chunkIndex = 0; while (chunkIndex < dataChunkList.size()) { DataChunk currentChunk = dataChunkList.get(chunkIndex); From 9ccf16af4c6f9fea362c0546f55f35d16cb24cc0 Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:51:16 -0400 Subject: [PATCH 11/12] reauthentication changes --- .../service/step/box/BoxWriterLargeFile.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java index cce75746..c9fa8aef 100644 --- a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/step/box/BoxWriterLargeFile.java @@ -1,6 +1,10 @@ package org.onedatashare.transferservice.odstransferservice.service.step.box; -import com.box.sdk.*; +import com.box.sdk.BoxAPIConnection; +import com.box.sdk.BoxFileUploadSession; +import com.box.sdk.BoxFileUploadSessionPart; +import com.box.sdk.BoxFolder; +import com.box.sdk.BoxUser; import org.onedatashare.transferservice.odstransferservice.constant.ODSConstants; import org.onedatashare.transferservice.odstransferservice.model.DataChunk; import org.onedatashare.transferservice.odstransferservice.model.EntityInfo; From ec3d4d799daac26d17d05cd4b918535409edd2bd Mon Sep 17 00:00:00 2001 From: SravyaPo <99841571+SravyaPo@users.noreply.github.com> Date: Sun, 7 May 2023 12:52:52 -0400 Subject: [PATCH 12/12] reauthentication changes --- .../service/AuthenticateCredentials.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 src/main/java/org/onedatashare/transferservice/odstransferservice/service/AuthenticateCredentials.java diff --git a/src/main/java/org/onedatashare/transferservice/odstransferservice/service/AuthenticateCredentials.java b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/AuthenticateCredentials.java new file mode 100644 index 00000000..82c2788c --- /dev/null +++ b/src/main/java/org/onedatashare/transferservice/odstransferservice/service/AuthenticateCredentials.java @@ -0,0 +1,22 @@ +package org.onedatashare.transferservice.odstransferservice.service; + +import org.onedatashare.transferservice.odstransferservice.model.credential.OAuthEndpointCredential; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + + +@Service +public class AuthenticateCredentials { + RestTemplate credentialTemplate; + String baseUrl = "http://EndPointCredentialService/{}/{}/{}"; + + public AuthenticateCredentials(RestTemplate credentialTemplate){ + this.credentialTemplate = credentialTemplate; + } + public OAuthEndpointCredential checkExpiryAndGenerateNew(String userId, String type, String accountId){ + String endpoint = baseUrl.replaceFirst("\\{}", userId) + .replaceFirst("\\{}", type) + .replaceFirst("\\{}", accountId); + return this.credentialTemplate.getForObject(endpoint, OAuthEndpointCredential.class); + } +}