Skip to content

Commit d603a7b

Browse files
author
shabtaisharon
authored
Merge pull request #325 from rpmoore/lazy_iterator
Lazy iterator and other small improvements
2 parents 17e2616 + 60dc564 commit d603a7b

File tree

15 files changed

+440
-180
lines changed

15 files changed

+440
-180
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ subprojects {
4141
}
4242

4343
task wrapper(type: Wrapper) {
44-
gradleVersion = '2.14'
44+
gradleVersion = '3.0'
4545
}
4646

4747
project(':ds3-sdk-integration') {

‎ds3-sdk-integration/src/main/java/com/spectralogic/ds3client/integration/Util.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,13 @@
2626
import com.spectralogic.ds3client.models.bulk.Ds3Object;
2727
import com.spectralogic.ds3client.serializer.XmlProcessingException;
2828
import com.spectralogic.ds3client.utils.ResourceUtils;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
2931

3032
import java.io.IOException;
3133
import java.net.URISyntaxException;
3234
import java.nio.file.Files;
3335
import java.nio.file.Path;
34-
import java.security.SignatureException;
3536
import java.util.ArrayList;
3637
import java.util.List;
3738

@@ -40,6 +41,7 @@
4041

4142

4243
public class Util {
44+
private static final Logger LOG = LoggerFactory.getLogger(Util.class);
4345
public static final String RESOURCE_BASE_NAME = "books/";
4446
public static final String[] BOOKS = {"beowulf.txt", "sherlock_holmes.txt", "tale_of_two_cities.txt", "ulysses.txt"};
4547

@@ -57,19 +59,20 @@ public static Ds3Client insecureFromEnv() {
5759
return builder.build();
5860
}
5961

60-
public static void assumeVersion1_2(final Ds3Client client) throws IOException, SignatureException {
62+
public static void assumeVersion1_2(final Ds3Client client) throws IOException {
6163
final int majorVersion = Integer.parseInt(client.getSystemInformationSpectraS3(
6264
new GetSystemInformationSpectraS3Request()).getSystemInformationResult().getBuildInformation().getVersion().split("\\.")[0]);
6365
assumeThat(majorVersion, is(1));
6466
}
6567

66-
public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException {
67-
68+
public static void loadBookTestData(final Ds3Client client, final String bucketName) throws IOException, XmlProcessingException, URISyntaxException {
69+
LOG.info("Loading test data...");
6870
getLoadJob(client, bucketName, RESOURCE_BASE_NAME)
6971
.transfer(new ResourceObjectPutter(RESOURCE_BASE_NAME));
72+
LOG.info("Finished loading test data...");
7073
}
7174

72-
public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, SignatureException, XmlProcessingException, URISyntaxException {
75+
public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final String bucketName, final String resourceBaseName) throws IOException, XmlProcessingException, URISyntaxException {
7376
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
7477

7578
final List<Ds3Object> objects = new ArrayList<>();
@@ -85,7 +88,7 @@ public static Ds3ClientHelpers.Job getLoadJob(final Ds3Client client, final Stri
8588
.startWriteJob(bucketName, objects);
8689
}
8790

88-
public static void loadBookTestDataWithPrefix(final Ds3Client client, final String bucketName, final String prefix) throws XmlProcessingException, SignatureException, IOException, URISyntaxException {
91+
public static void loadBookTestDataWithPrefix(final Ds3Client client, final String bucketName, final String prefix) throws XmlProcessingException, IOException, URISyntaxException {
8992
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
9093

9194
final List<Ds3Object> objects = new ArrayList<>();
@@ -100,7 +103,7 @@ public static void loadBookTestDataWithPrefix(final Ds3Client client, final Stri
100103
helpers.startWriteJob(bucketName, objects).transfer(new PrefixAdderObjectChannelBuilder(new ResourceObjectPutter(RESOURCE_BASE_NAME), prefix));
101104
}
102105

103-
public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException, SignatureException {
106+
public static void deleteAllContents(final Ds3Client client, final String bucketName) throws IOException {
104107
final Ds3ClientHelpers helpers = Ds3ClientHelpers.wrap(client);
105108

106109
final Iterable<Contents> objects = helpers.listObjects(bucketName);

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/LazyIterator_Test.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2015 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.integration;
17+
18+
import com.spectralogic.ds3client.Ds3Client;
19+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
20+
import com.spectralogic.ds3client.helpers.LazyObjectIterable;
21+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
22+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
23+
import com.spectralogic.ds3client.models.ChecksumType;
24+
import com.spectralogic.ds3client.models.Contents;
25+
import com.spectralogic.ds3client.serializer.XmlProcessingException;
26+
import org.junit.AfterClass;
27+
import org.junit.BeforeClass;
28+
import org.junit.Test;
29+
import org.slf4j.Logger;
30+
import org.slf4j.LoggerFactory;
31+
32+
import java.io.IOException;
33+
import java.net.URISyntaxException;
34+
import java.security.SignatureException;
35+
import java.util.Iterator;
36+
import java.util.UUID;
37+
38+
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
39+
import static com.spectralogic.ds3client.integration.Util.loadBookTestData;
40+
import static org.hamcrest.CoreMatchers.is;
41+
import static org.hamcrest.CoreMatchers.notNullValue;
42+
import static org.junit.Assert.assertFalse;
43+
import static org.junit.Assert.assertThat;
44+
import static org.junit.Assert.assertTrue;
45+
46+
public class LazyIterator_Test {
47+
private static final Logger LOG = LoggerFactory.getLogger(LazyIterator_Test.class);
48+
49+
private static final Ds3Client CLIENT = Util.fromEnv();
50+
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(CLIENT);
51+
private static final String TEST_ENV_NAME = "lazy_iterator_test";
52+
private static TempStorageIds envStorageIds;
53+
private static UUID envDataPolicyId;
54+
private static final int retries = 5;
55+
56+
@BeforeClass
57+
public static void startup() throws IOException, SignatureException {
58+
LOG.info("Starting test Setup...");
59+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, CLIENT);
60+
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, CLIENT);
61+
LOG.info("Finished test Setup...");
62+
}
63+
64+
@AfterClass
65+
public static void teardown() throws IOException, SignatureException {
66+
LOG.info("Starting test teardown...");
67+
TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, CLIENT);
68+
CLIENT.close();
69+
LOG.info("Finished test teardown...");
70+
}
71+
72+
@Test
73+
public void emptyTest() throws IOException, SignatureException {
74+
HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
75+
try {
76+
final String prefix = "";
77+
final String nextMarker = null;
78+
final int maxKeys = 100;
79+
80+
81+
final LazyObjectIterable iterable = new LazyObjectIterable(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, retries);
82+
final Iterator<Contents> iterator = iterable.iterator();
83+
84+
assertFalse(iterator.hasNext());
85+
86+
} finally {
87+
deleteAllContents(CLIENT, TEST_ENV_NAME);
88+
}
89+
}
90+
91+
@Test
92+
public void singlePageTest() throws IOException, XmlProcessingException, SignatureException, URISyntaxException {
93+
HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
94+
loadBookTestData(CLIENT, TEST_ENV_NAME);
95+
try {
96+
final String prefix = "";
97+
final String nextMarker = null;
98+
final int maxKeys = 100;
99+
100+
final LazyObjectIterable iterable = new LazyObjectIterable(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, retries);
101+
final Iterator<Contents> iterator = iterable.iterator();
102+
103+
assertTrue(iterator.hasNext());
104+
assertThat(iterator.next(), is(notNullValue()));
105+
assertTrue(iterator.hasNext());
106+
assertThat(iterator.next(), is(notNullValue()));
107+
assertTrue(iterator.hasNext());
108+
assertThat(iterator.next(), is(notNullValue()));
109+
assertTrue(iterator.hasNext());
110+
assertThat(iterator.next(), is(notNullValue()));
111+
assertFalse(iterator.hasNext());
112+
} finally {
113+
deleteAllContents(CLIENT, TEST_ENV_NAME);
114+
}
115+
}
116+
117+
@Test
118+
public void multiPageTest() throws IOException, URISyntaxException, XmlProcessingException {
119+
HELPERS.ensureBucketExists(TEST_ENV_NAME, envDataPolicyId);
120+
loadBookTestData(CLIENT, TEST_ENV_NAME);
121+
try {
122+
final String prefix = "";
123+
final String nextMarker = null;
124+
final int maxKeys = 2;
125+
126+
final LazyObjectIterable iterable = new LazyObjectIterable(CLIENT, TEST_ENV_NAME, prefix, nextMarker, maxKeys, retries);
127+
final Iterator<Contents> iterator = iterable.iterator();
128+
129+
assertTrue(iterator.hasNext());
130+
assertThat(iterator.next(), is(notNullValue()));
131+
assertTrue(iterator.hasNext());
132+
assertThat(iterator.next(), is(notNullValue()));
133+
assertTrue(iterator.hasNext());
134+
assertThat(iterator.next(), is(notNullValue()));
135+
assertTrue(iterator.hasNext());
136+
assertThat(iterator.next(), is(notNullValue()));
137+
assertFalse(iterator.hasNext());
138+
} finally {
139+
deleteAllContents(CLIENT, TEST_ENV_NAME);
140+
}
141+
}
142+
}

0 commit comments

Comments
 (0)