Skip to content

Commit 9ecdf11

Browse files
qqmyersqqmyers
committed
Merge remote-tracking branch 'GDCC/AWSv2' into dev-prov-awsv2
2 parents 03d29b6 + 2deb1e4 commit 9ecdf11

File tree

16 files changed

+1173
-1035
lines changed

16 files changed

+1173
-1035
lines changed

‎doc/release-notes/11360-AwsSdkV2.x.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
## Update to AWS SDK v2.x for S3
2+
The v1 AWS SDK Dataverse used for S3 is in maintenance mode and will reach it's end of life in Dec. 2025.
3+
With this release, Dataverse has switched to v2.x of the SDK. Older versions of localstack, e.g. v2.3.2 appear to be incompatible and users of localstack should update (v.4.2.0 was tested).

‎docker-compose-dev.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ services:
209209
dev_localstack:
210210
container_name: "dev_localstack"
211211
hostname: "localstack"
212-
image: localstack/localstack:2.3.2
212+
image: localstack/localstack:4.2.0
213213
restart: on-failure
214214
ports:
215215
- "127.0.0.1:4566:4566"

‎modules/dataverse-parent/pom.xml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,13 @@
3333
<scope>import</scope>
3434
</dependency>
3535
<dependency>
36-
<groupId>com.amazonaws</groupId>
37-
<artifactId>aws-java-sdk-bom</artifactId>
36+
<groupId>software.amazon.awssdk</groupId>
37+
<artifactId>bom</artifactId>
3838
<version>${aws.version}</version>
3939
<type>pom</type>
4040
<scope>import</scope>
4141
</dependency>
42+
4243
<dependency>
4344
<groupId>com.google.cloud</groupId>
4445
<artifactId>libraries-bom</artifactId>
@@ -152,7 +153,7 @@
152153
<payara.version>6.2025.3</payara.version>
153154
<postgresql.version>42.7.4</postgresql.version>
154155
<solr.version>9.8.1</solr.version>
155-
<aws.version>1.12.748</aws.version>
156+
<aws.version>2.31.3</aws.version>
156157
<google.library.version>26.30.0</google.library.version>
157158

158159
<!-- Basic libs, logging -->

‎pom.xml‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,18 @@
7676
<groupId>org.apache.geronimo.specs</groupId>
7777
<artifactId>geronimo-javamail_1.4_spec</artifactId>
7878
</exclusion>
79+
<exclusion>
80+
<groupId>org.apache.geronimo.specs</groupId>
81+
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
82+
</exclusion>
83+
<exclusion>
84+
<groupId>org.codehaus.woodstox</groupId>
85+
<artifactId>wstx-asl</artifactId>
86+
</exclusion>
87+
<exclusion>
88+
<groupId>org.codehaus.woodstox</groupId>
89+
<artifactId>woodstox-core-asl</artifactId>
90+
</exclusion>
7991
</exclusions>
8092
</dependency>
8193
<!-- Dependency for Apache Abdera and Apache Tika. Tika needs newer version. -->
@@ -166,10 +178,24 @@
166178
</exclusion>
167179
</exclusions>
168180
</dependency>
169-
170181
<dependency>
171-
<groupId>com.amazonaws</groupId>
172-
<artifactId>aws-java-sdk-s3</artifactId>
182+
<groupId>software.amazon.awssdk</groupId>
183+
<artifactId>s3</artifactId>
184+
<!-- no version here as managed by BOM above! -->
185+
</dependency>
186+
<dependency>
187+
<groupId>software.amazon.awssdk</groupId>
188+
<artifactId>s3-transfer-manager</artifactId>
189+
<!-- no version here as managed by BOM above! -->
190+
</dependency>
191+
<!--dependency>
192+
<groupId>software.amazon.awssdk</groupId>
193+
<artifactId>apache-client</artifactId-->
194+
<!-- no version here as managed by BOM above! -->
195+
<!--/dependency-->
196+
<dependency>
197+
<groupId>software.amazon.awssdk</groupId>
198+
<artifactId>netty-nio-client</artifactId>
173199
<!-- no version here as managed by BOM above! -->
174200
</dependency>
175201
<dependency>
@@ -180,7 +206,6 @@
180206
<dependency>
181207
<groupId>com.google.code.gson</groupId>
182208
<artifactId>gson</artifactId>
183-
<version>2.9.1</version>
184209
<scope>compile</scope>
185210
</dependency>
186211
<!-- Should be refactored and moved to transitive section above once on Java EE 8 (makes WAR smaller) -->

‎scripts/zipdownload/pom.xml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
<artifactId>postgresql</artifactId>
2424
</dependency>
2525
<dependency>
26-
<groupId>com.amazonaws</groupId>
27-
<artifactId>aws-java-sdk-s3</artifactId>
26+
<groupId>software.amazon.awssdk</groupId>
27+
<artifactId>s3</artifactId>
28+
<!-- no version here as managed by BOM above! -->
2829
</dependency>
2930
</dependencies>
3031
<build>

‎scripts/zipdownload/src/main/java/edu/harvard/iq/dataverse/custom/service/util/DirectAccessUtil.java‎

Lines changed: 21 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@
2020

2121
package edu.harvard.iq.dataverse.custom.service.util;
2222

23-
import com.amazonaws.SdkClientException;
24-
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
25-
import com.amazonaws.services.s3.AmazonS3;
26-
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
27-
import com.amazonaws.services.s3.model.GetObjectRequest;
28-
import com.amazonaws.services.s3.model.ObjectMetadata;
23+
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
24+
import software.amazon.awssdk.core.ResponseInputStream;
25+
import software.amazon.awssdk.regions.Region;
26+
import software.amazon.awssdk.services.s3.S3Client;
27+
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
28+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
29+
import software.amazon.awssdk.services.s3.model.S3Exception;
30+
2931
import java.io.File;
3032
import java.io.FileInputStream;
3133
import java.io.IOException;
@@ -38,9 +40,9 @@
3840
*
3941
* @author Leonid Andreev
4042
*/
41-
public class DirectAccessUtil implements java.io.Serializable {
43+
public class DirectAccessUtil implements java.io.Serializable {
4244

43-
private AmazonS3 s3 = null;
45+
private S3Client s3 = null;
4446

4547
public InputStream openDirectAccess(String storageLocation) {
4648
InputStream inputStream = null;
@@ -57,31 +59,17 @@ public InputStream openDirectAccess(String storageLocation) {
5759
String bucket = storageLocation.substring(0, storageLocation.indexOf('/'));
5860
String key = storageLocation.substring(storageLocation.indexOf('/') + 1);
5961

60-
//System.out.println("bucket: "+bucket);
61-
//System.out.println("key: "+key);
62-
63-
/* commented-out code below is for looking up S3 metatadata
64-
properties, such as size, etc. prior to making an access call:
65-
ObjectMetadata objectMetadata = null;
66-
long fileSize = 0L;
67-
try {
68-
objectMetadata = s3.getObjectMetadata(bucket, key);
69-
fileSize = objectMetadata.getContentLength();
70-
//System.out.println("byte size: "+objectMetadata.getContentLength());
71-
} catch (SdkClientException sce) {
72-
System.err.println("Cannot get S3 object metadata " + key + " from bucket " + bucket);
73-
}*/
74-
7562
try {
76-
inputStream = s3.getObject(new GetObjectRequest(bucket, key)).getObjectContent();
77-
} catch (SdkClientException sce) {
63+
ResponseInputStream<GetObjectResponse> s3Object = s3.getObject(GetObjectRequest.builder()
64+
.bucket(bucket)
65+
.key(key)
66+
.build());
67+
inputStream = s3Object;
68+
} catch (S3Exception se) {
7869
System.err.println("Cannot get S3 object " + key + " from bucket " + bucket);
7970
}
8071

8172
} else if (storageLocation.startsWith("file://")) {
82-
// This could be a static method; since no reusable client/maintainable
83-
// state is required
84-
8573
storageLocation = storageLocation.substring(7);
8674

8775
try {
@@ -98,14 +86,13 @@ public InputStream openDirectAccess(String storageLocation) {
9886
private void createOrReuseAwsClient() {
9987
if (this.s3 == null) {
10088
try {
101-
AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard();
102-
s3CB.setCredentials(new ProfileCredentialsProvider("default"));
103-
this.s3 = s3CB.build();
104-
89+
this.s3 = S3Client.builder()
90+
.region(Region.US_EAST_1) // You may want to make this configurable
91+
.credentialsProvider(ProfileCredentialsProvider.create("default"))
92+
.build();
10593
} catch (Exception e) {
106-
System.err.println("cannot instantiate an S3 client");
94+
System.err.println("Cannot instantiate an S3 client: " + e.getMessage());
10795
}
10896
}
10997
}
110-
11198
}

‎src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1836,26 +1836,6 @@ public String getRsyncScriptFilename() {
18361836
return rsyncScriptFilename;
18371837
}
18381838

1839-
@Deprecated
1840-
public void requestDirectUploadUrl() {
1841-
1842-
S3AccessIO<?> s3io = FileUtil.getS3AccessForDirectUpload(dataset);
1843-
if (s3io == null) {
1844-
FacesContext.getCurrentInstance().addMessage(uploadComponentId, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadWarning"), "Direct upload not supported for this dataset"));
1845-
}
1846-
String url = null;
1847-
String storageIdentifier = null;
1848-
try {
1849-
url = s3io.generateTemporaryS3UploadUrl();
1850-
storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation());
1851-
} catch (IOException io) {
1852-
logger.warning(io.getMessage());
1853-
FacesContext.getCurrentInstance().addMessage(uploadComponentId, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadWarning"), "Issue in connecting to S3 store for direct upload"));
1854-
}
1855-
1856-
PrimeFaces.current().executeScript("uploadFileDirectly('" + url + "','" + storageIdentifier + "')");
1857-
}
1858-
18591839
public void requestDirectUploadUrls() {
18601840

18611841
Map<String, String> paramMap = FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap();

0 commit comments

Comments
 (0)