Skip to content

Commit 491eb6f

Browse files
author
Denver
committed
Merge pull request #78 from rpmoore/master
Metadata Support
2 parents d79ef83 + fcc4947 commit 491eb6f

23 files changed

+515
-30
lines changed

‎build.gradle‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
allprojects {
1717
group = 'com.spectralogic.ds3'
18-
version = '1.1.0-RC7'
18+
version = '1.1.0-RC8'
1919
}
2020

2121
subprojects {
@@ -32,14 +32,17 @@ subprojects {
3232
dependencies {
3333
compile 'org.slf4j:slf4j-api:1.7.12'
3434
compile 'org.slf4j:jcl-over-slf4j:1.7.12'
35-
testCompile 'org.mockito:mockito-all:1.9.5'
36-
testCompile 'junit:junit:4.11'
35+
testCompile ('org.mockito:mockito-core:1.9.5') {
36+
exclude group: 'org.hamcrest'
37+
}
38+
39+
testCompile 'junit:junit:4.12'
3740
testCompile 'org.slf4j:slf4j-simple:1.7.12'
3841
}
3942
}
4043

4144
task wrapper(type: Wrapper) {
42-
gradleVersion = '2.4'
45+
gradleVersion = '2.6'
4346
}
4447

4548
project(':ds3-sdk-integration') {

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/Metadata_Test.java‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package com.spectralogic.ds3client.integration;
2+
3+
import com.google.common.collect.ImmutableMultimap;
4+
import com.google.common.collect.Lists;
5+
import com.spectralogic.ds3client.Ds3Client;
6+
import com.spectralogic.ds3client.commands.*;
7+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
8+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
9+
import com.spectralogic.ds3client.networking.Metadata;
10+
import com.spectralogic.ds3client.serializer.XmlProcessingException;
11+
import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel;
12+
import org.apache.commons.io.IOUtils;
13+
import org.junit.AfterClass;
14+
import org.junit.BeforeClass;
15+
import org.junit.Test;
16+
17+
import java.io.IOException;
18+
import java.nio.ByteBuffer;
19+
import java.nio.channels.SeekableByteChannel;
20+
import java.security.SignatureException;
21+
import java.util.List;
22+
import java.util.Map;
23+
import java.util.Set;
24+
25+
import static org.hamcrest.CoreMatchers.*;
26+
import static org.junit.Assert.assertFalse;
27+
import static org.junit.Assert.assertThat;
28+
import static org.junit.Assert.assertTrue;
29+
30+
public class Metadata_Test {
31+
private static Ds3Client client;
32+
33+
@BeforeClass
34+
public static void startup() {
35+
client = Util.fromEnv();
36+
}
37+
38+
@AfterClass
39+
public static void teardown() throws IOException {
40+
client.close();
41+
}
42+
43+
@Test
44+
public void singleMetadataField() throws IOException, SignatureException, XmlProcessingException {
45+
46+
final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value"));
47+
48+
final List<String> values = metadata.get("name");
49+
assertThat(values, is(notNullValue()));
50+
assertFalse(values.isEmpty());
51+
assertThat(values.size(), is(1));
52+
assertThat(values.get(0), is("value"));
53+
54+
}
55+
56+
@Test
57+
public void multipleMetadataFields() throws IOException, SignatureException, XmlProcessingException {
58+
59+
final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "key2", "value2"));
60+
61+
final Set<String> keys = metadata.keys();
62+
63+
assertThat(keys.size(), is(2));
64+
assertTrue(keys.contains("name"));
65+
assertTrue(keys.contains("key2"));
66+
67+
final List<String> values = metadata.get("name");
68+
assertThat(values, is(notNullValue()));
69+
assertFalse(values.isEmpty());
70+
assertThat(values.size(), is(1));
71+
assertThat(values.get(0), is("value"));
72+
73+
}
74+
75+
/*
76+
//TODO There is currently a limitation in BP where it does not handle metadata values that have the same key.
77+
@Test
78+
public void multipleMetadataFieldsForSameKey() throws XmlProcessingException, SignatureException, IOException {
79+
80+
final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "name", "value2"));
81+
82+
final Set<String> keys = metadata.keys();
83+
84+
assertThat(keys.size(), is(1));
85+
assertTrue(keys.contains("name"));
86+
87+
final List<String> values = metadata.get("name");
88+
assertThat(values, is(notNullValue()));
89+
assertFalse(values.isEmpty());
90+
assertThat(values, hasItem("value"));
91+
}
92+
93+
@Test
94+
public void multipleMetadataFieldsForSameKeyDifferentCase() throws XmlProcessingException, SignatureException, IOException {
95+
96+
final Metadata metadata = processMetadataRequest("metadataBucket", ImmutableMultimap.of("name", "value", "Name", "value2"));
97+
98+
final Set<String> keys = metadata.keys();
99+
100+
assertThat(keys.size(), is(1));
101+
assertTrue(keys.contains("name"));
102+
103+
final List<String> values = metadata.get("name");
104+
assertThat(values, is(notNullValue()));
105+
assertFalse(values.isEmpty());
106+
assertThat(values, hasItem("value"));
107+
}
108+
*/
109+
110+
private static Metadata processMetadataRequest(final String bucketName, final ImmutableMultimap<String, String> metadata) throws IOException, SignatureException, XmlProcessingException {
111+
final Ds3ClientHelpers wrapper = Ds3ClientHelpers.wrap(client);
112+
wrapper.ensureBucketExists(bucketName);
113+
114+
try {
115+
final Ds3Object obj = new Ds3Object("obj.txt", 1024);
116+
final BulkPutResponse putResponse = client.bulkPut(new BulkPutRequest(bucketName, Lists.newArrayList(obj)));
117+
final PutObjectRequest request = new PutObjectRequest(bucketName, obj.getName(), putResponse.getResult().getJobId(), 1024, 0, buildRandomChannel(1024));
118+
119+
for (final Map.Entry<String, String> entry : metadata.entries()) {
120+
request.withMetaData(entry.getKey(), entry.getValue());
121+
}
122+
123+
final PutObjectResponse putObjResponse = client.putObject(request);
124+
125+
assertThat(putObjResponse.getStatusCode(), is(200));
126+
127+
final HeadObjectResponse response = client.headObject(new HeadObjectRequest(bucketName, obj.getName()));
128+
129+
assertThat(response.getStatusCode(), is(200));
130+
131+
return response.getMetadata();
132+
133+
} finally {
134+
Util.deleteAllContents(client, bucketName);
135+
}
136+
}
137+
138+
private static SeekableByteChannel buildRandomChannel(int length) throws IOException {
139+
140+
final byte[] randomData = IOUtils.toByteArray(new RandomDataInputStream(System.currentTimeMillis(), length));
141+
final ByteBuffer randomBuffer = ByteBuffer.wrap(randomData);
142+
143+
final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel(length);
144+
channel.write(randomBuffer);
145+
146+
return channel;
147+
}
148+
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,17 @@ GetObjectResponse getObject(GetObjectRequest request)
144144
PutObjectResponse putObject(PutObjectRequest request)
145145
throws IOException, SignatureException;
146146

147+
/**
148+
* Performs an HTTP HEAD for an object. If the object exists then a successful response is returned and any metadata
149+
* associated with the object.
150+
* @param request The Object to perform the HTTP HEAD for.
151+
* @return If successful a response is returned with any metadata that was assigned with the object was created.
152+
* @throws IOException
153+
* @throws SignatureException
154+
*/
155+
HeadObjectResponse headObject(HeadObjectRequest request)
156+
throws IOException, SignatureException;
157+
147158
/**
148159
* Primes the Ds3 appliance for a Bulk Get. This does not perform the gets for each individual files. See
149160
* {@link #getObject(GetObjectRequest)} for performing the get.

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,11 @@ public PutObjectResponse putObject(final PutObjectRequest request) throws IOExce
9090
return new PutObjectResponse(this.netClient.getResponse(request));
9191
}
9292

93+
@Override
94+
public HeadObjectResponse headObject(final HeadObjectRequest request) throws IOException, SignatureException {
95+
return new HeadObjectResponse(this.netClient.getResponse(request));
96+
}
97+
9398
@Override
9499
public BulkGetResponse bulkGet(final BulkGetRequest request) throws IOException, SignatureException {
95100
return new BulkGetResponse(this.netClient.getResponse(request));

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/HeadersImpl.java‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,29 +15,41 @@
1515

1616
package com.spectralogic.ds3client;
1717

18+
import com.google.common.collect.ImmutableMultimap;
19+
import com.google.common.collect.Lists;
20+
import com.google.common.collect.Sets;
1821
import com.spectralogic.ds3client.networking.Headers;
1922
import org.apache.http.Header;
2023

21-
public class HeadersImpl implements Headers {
24+
import java.util.List;
25+
import java.util.Set;
2226

23-
private final Header[] headers;
27+
class HeadersImpl implements Headers {
2428

25-
HeadersImpl(final Header[] allHeaders){
26-
this.headers = allHeaders;
27-
}
29+
private final ImmutableMultimap<String, String> headers;
2830

29-
@Override
30-
public String get(final String key) {
31-
return findHeader(key);
31+
32+
HeadersImpl(final Header[] allHeaders){
33+
this.headers = toMultiMap(allHeaders);
3234
}
3335

34-
private String findHeader(final String key) {
36+
private static ImmutableMultimap<String, String> toMultiMap(final Header[] headers){
37+
final ImmutableMultimap.Builder<String, String> builder = ImmutableMultimap.builder();
3538

3639
for (final Header header : headers) {
37-
if (header.getName().equalsIgnoreCase(key)) {
38-
return header.getValue();
39-
}
40+
builder.put(header.getName(), header.getValue());
4041
}
41-
return null;
42+
43+
return builder.build();
44+
}
45+
46+
@Override
47+
public List<String> get(final String key) {
48+
return Lists.newArrayList(headers.get(key));
49+
}
50+
51+
@Override
52+
public Set<String> keys() {
53+
return Sets.newHashSet(headers.keySet());
4254
}
4355
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,22 @@
1818
import com.google.common.collect.ImmutableSet;
1919
import com.spectralogic.ds3client.models.Error;
2020
import com.spectralogic.ds3client.networking.FailedRequestException;
21+
import com.spectralogic.ds3client.networking.Headers;
2122
import com.spectralogic.ds3client.networking.WebResponse;
2223
import com.spectralogic.ds3client.serializer.XmlOutput;
2324

2425
import org.apache.commons.io.IOUtils;
26+
import org.slf4j.Logger;
27+
import org.slf4j.LoggerFactory;
2528

2629
import java.io.IOException;
2730
import java.io.InputStream;
2831
import java.io.StringWriter;
32+
import java.util.List;
2933

3034
public abstract class AbstractResponse implements Ds3Response{
35+
final private static Logger LOG = LoggerFactory.getLogger(AbstractResponse.class);
36+
3137
final public static String UTF8 = "UTF-8";
3238

3339
final private WebResponse response;
@@ -36,14 +42,23 @@ public abstract class AbstractResponse implements Ds3Response{
3642
public AbstractResponse(final WebResponse response) throws IOException {
3743
this.response = response;
3844
if (response != null) {
39-
this.md5 = this.response.getHeaders().get("Content-MD5");
45+
this.md5 = getFirstHeaderValue(this.response.getHeaders(), "Content-MD5");
4046
}
4147
else {
4248
this.md5 = null;
4349
}
4450
this.processResponse();
4551
}
4652

53+
final protected String getFirstHeaderValue(final Headers headers, final String key) {
54+
final List<String> valueList = headers.get(key);
55+
if (valueList == null || valueList.isEmpty()) {
56+
return null;
57+
} else {
58+
return valueList.get(0);
59+
}
60+
}
61+
4762
protected abstract void processResponse() throws IOException;
4863

4964
public WebResponse getResponse() {
@@ -81,6 +96,7 @@ private static Error parseErrorResponse(final String responseString) {
8196
return XmlOutput.fromXml(responseString, Error.class);
8297
} catch (final IOException e) {
8398
// It's likely the response string is not in a valid error format.
99+
LOG.error("Failed to parse error response", e);
84100
return null;
85101
}
86102
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ private static Objects parseChunk(final WebResponse webResponse) throws IOExcept
7373
}
7474

7575
private static int parseRetryAfter(final WebResponse webResponse) {
76-
final String retryAfter = webResponse.getHeaders().get("Retry-After");
76+
final String retryAfter = webResponse.getHeaders().get("Retry-After").get(0);
7777
if (retryAfter == null) {
7878
throw new RetryAfterExpectedException();
7979
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ private static MasterObjectList parseMasterObjectList(final WebResponse webRespo
7474
}
7575

7676
private static int parseRetryAfter(final WebResponse webResponse) {
77-
final String retryAfter = webResponse.getHeaders().get("Retry-After");
77+
final String retryAfter = webResponse.getHeaders().get("Retry-After").get(0);
7878
if (retryAfter == null) {
7979
throw new RetryAfterExpectedException();
8080
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/GetObjectResponse.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package com.spectralogic.ds3client.commands;
1717

18+
import com.spectralogic.ds3client.networking.Metadata;
1819
import com.spectralogic.ds3client.networking.WebResponse;
1920
import com.spectralogic.ds3client.utils.IOUtils;
2021

@@ -28,6 +29,10 @@ public GetObjectResponse(final WebResponse response, final WritableByteChannel d
2829
download(destinationChannel, bufferSize);
2930
}
3031

32+
public Metadata getMetadata() {
33+
return new MetadataImpl(this.getResponse().getHeaders());
34+
}
35+
3136
@Override
3237
protected void processResponse() throws IOException {
3338
this.checkStatusCode(200);

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/HeadObjectRequest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.spectralogic.ds3client.commands;
2+
3+
import com.spectralogic.ds3client.HttpVerb;
4+
5+
public class HeadObjectRequest extends AbstractRequest {
6+
7+
private final String bucketName;
8+
private final String objectName;
9+
10+
public HeadObjectRequest(final String bucketName, final String objectName) {
11+
this.bucketName = bucketName;
12+
this.objectName = objectName;
13+
}
14+
15+
@Override
16+
public String getPath() {
17+
return "/" + bucketName + "/" + objectName;
18+
}
19+
20+
@Override
21+
public HttpVerb getVerb() {
22+
return HttpVerb.HEAD;
23+
}
24+
}

0 commit comments

Comments
 (0)