Skip to content

Commit 8655b95

Browse files
RachelTuckerRachelTucker
authored
Merge pull request #195 from scribe/CLI-210
Cli 210
2 parents 4d95540 + 2a7a377 commit 8655b95

File tree

11 files changed

+172
-26
lines changed

11 files changed

+172
-26
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ subprojects {
5252
compile 'ch.qos.logback:logback-core:1.2.3'
5353
compile 'ch.qos.logback:logback-classic:1.2.3'
5454
compile 'com.google.guava:guava:23.0-jre'
55-
compile 'com.spectralogic.ds3:ds3-sdk:5.0.2'
55+
compile 'com.spectralogic.ds3:ds3-sdk:5.0.3'
5656
compile 'com.google.inject:guice:4.2.0'
5757

5858
testCompile ('org.mockito:mockito-core:1.10.19') {

‎ds3-cli-certification/src/test/java/com/spectralogic/ds3cli/certification/Certification_Test.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ public void test_8_1_versioning() throws Exception {
371371
final Long fileSize = 1024L;
372372
final String bucketName = CertificationUtil.getBucketName(testDescription);
373373
boolean success = false;
374-
final UUID dataPolicy = setupDataPolicy(testDescription, false, ChecksumType.Type.MD5, client);
374+
final UUID dataPolicy = setupDataPolicy(testDescription, false, ChecksumType.Type.MD5, VersioningLevel.NONE, client);
375375
final TempStorageIds tempStorageIds = TempStorageUtil.setup(testDescription, dataPolicy, client);
376376

377377

‎ds3-cli-helpers/src/test/java/com/spectralogic/ds3cli/helpers/TempStorageUtil.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,11 @@ public static UUID setupDataPolicy(
104104
final String testSetName,
105105
final boolean withEndToEndCrcRequired,
106106
final ChecksumType.Type checksumType,
107+
final VersioningLevel versioningLevel,
107108
final Ds3Client client) throws IOException {
108109
final PutDataPolicySpectraS3Response dataPolicyResponse = createDataPolicy(
109110
testSetName + DATA_POLICY_NAME,
110-
VersioningLevel.NONE,
111+
versioningLevel,
111112
checksumType,
112113
withEndToEndCrcRequired,
113114
false,

‎ds3-cli-integration/src/test/java/com/spectralogic/ds3cli/integration/FeatureIntegration_Test.java‎

Lines changed: 52 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,13 @@
4343
import com.spectralogic.ds3client.commands.GetBucketResponse;
4444
import com.spectralogic.ds3client.commands.GetObjectRequest;
4545
import com.spectralogic.ds3client.commands.GetObjectResponse;
46-
import com.spectralogic.ds3client.commands.spectrads3.CancelJobSpectraS3Request;
46+
import com.spectralogic.ds3client.commands.spectrads3.*;
4747
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
4848
import com.spectralogic.ds3client.helpers.FileObjectPutter;
4949
import com.spectralogic.ds3client.helpers.FolderNameFilter;
5050
import com.spectralogic.ds3client.models.ChecksumType;
5151
import com.spectralogic.ds3client.models.Contents;
52+
import com.spectralogic.ds3client.models.VersioningLevel;
5253
import com.spectralogic.ds3client.models.bulk.Ds3Object;
5354
import com.spectralogic.ds3client.utils.Platform;
5455
import com.spectralogic.ds3client.utils.ResourceUtils;
@@ -90,10 +91,11 @@ public class FeatureIntegration_Test {
9091
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client);
9192
private static final String TEST_ENV_NAME = "FeatureIntegration_Test";
9293
private static TempStorageIds envStorageIds;
94+
private static UUID envDataPolicyId;
9395

9496
@BeforeClass
9597
public static void startup() throws IOException {
96-
final UUID envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
98+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, VersioningLevel.NONE, client);
9799
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, client);
98100
}
99101

@@ -260,7 +262,7 @@ public void getCompletedJob() throws Exception {
260262

261263
final String expectedBeginning = "JobId: " + readJob.getJobId() + " | Name: ";
262264
final String expectedMiddle = "Status: COMPLETED | Bucket: " + bucketName + " | Type: GET | Priority:";
263-
final String expectedEnding = " | Total Size: " + objSize + " | Total Transferred: " ;
265+
final String expectedEnding = " | Total Size: " + objSize + " | Total Transferred: ";
264266

265267
assertTrue(getJobResponse.getMessage().startsWith(expectedBeginning));
266268
assertTrue(getJobResponse.getMessage().contains(expectedMiddle));
@@ -546,7 +548,7 @@ public void putBulkObjectWithPrefix() throws Exception {
546548
"-c", "put_bulk",
547549
"-b", bucketName,
548550
"-d", Util.RESOURCE_BASE_NAME,
549-
"-p" , prefix,
551+
"-p", prefix,
550552
"--sync"});
551553
final CommandResponse response = Util.command(client, args);
552554
assertThat(response.getMessage(), is(String.format("SUCCESS: Wrote all the files in %s to bucket %s", "." + File.separator + "src" + File.separator + "test" + File.separator + "resources" + File.separator + "books", bucketName)));
@@ -720,6 +722,49 @@ public void metadataPut() throws Exception {
720722
}
721723
}
722724

725+
@Test
726+
public void testPutObjectWithVersion() throws Exception {
727+
final String bucketName = "test_put_object_with_version";
728+
final UUID setupDataPolicy = TempStorageUtil.setupDataPolicy(bucketName, false, ChecksumType.Type.CRC_32, VersioningLevel.KEEP_MULTIPLE_VERSIONS, client);
729+
final TempStorageIds tempStorageIds = TempStorageUtil.setup(bucketName, setupDataPolicy, client);
730+
731+
try {
732+
client.putBucketSpectraS3(new PutBucketSpectraS3Request(bucketName).withDataPolicyId(setupDataPolicy.toString()));
733+
734+
final Arguments args = new Arguments(new String[]{"--http", "-c", "put_object", "-b", bucketName, "-o", "src/test/resources/books/beowulf.txt"});
735+
final CommandResponse response = Util.command(client, args);
736+
final CommandResponse responseAgain = Util.command(client, args);
737+
738+
assertThat(response.getReturnCode(), is(0));
739+
assertThat(responseAgain.getReturnCode(), is(0));
740+
741+
final Arguments getBucket = new Arguments(new String[]{"--http", "-c", "get_bucket", "-b", bucketName, "-v"});
742+
final CommandResponse getBucketResponse = Util.command(client, getBucket);
743+
final String bucketMessage = getBucketResponse.getMessage();
744+
final String[] lines = bucketMessage.split("\\n");
745+
final String line9 = lines[9];
746+
final String line10 = lines[10];
747+
final String[] line9Split = line9.split("\\|");
748+
final String[] line10Split = line10.split("\\|");
749+
final String versionIdOne = line9Split[6].trim();
750+
final String versionIdTwo = line10Split[6].trim();
751+
assertThat(versionIdOne, not(versionIdTwo));
752+
753+
final Arguments getFirst = new Arguments(new String[]{"--http", "-c", "get_object", "-b", bucketName, "-o", "src/test/resources/books/beowulf.txt", "--versionId", versionIdOne});
754+
final Arguments getSecond = new Arguments(new String[]{"--http", "-c", "get_object", "-b", bucketName, "-o", "src/test/resources/books/beowulf.txt", "--versionId", versionIdTwo});
755+
final CommandResponse getFirstResponse = Util.command(client, getFirst);
756+
final CommandResponse getSecondResponse = Util.command(client, getSecond);
757+
758+
assertThat(getFirstResponse.getReturnCode(), is(0));
759+
assertThat(getSecondResponse.getReturnCode(), is(0));
760+
} finally {
761+
client.deleteBucketSpectraS3(new DeleteBucketSpectraS3Request(bucketName).withForce(true));
762+
TempStorageUtil.teardown(bucketName, tempStorageIds, client);
763+
client.modifyUserSpectraS3(new ModifyUserSpectraS3Request("Administrator")
764+
.withDefaultDataPolicyId(envDataPolicyId));
765+
}
766+
}
767+
723768
@Test
724769
public void testPutObjectWithFileMetadata() throws Exception {
725770
Assume.assumeFalse(Platform.isWindows());
@@ -832,7 +877,7 @@ public void getPhysicalPlacement() throws Exception {
832877
Util.createBucket(client, bucketName);
833878
Util.loadBookTestData(client, bucketName);
834879

835-
final Arguments args = new Arguments(new String[]{"--http", "-c", "get_physical_placement", "-b", bucketName, "-o", "beowulf.txt" });
880+
final Arguments args = new Arguments(new String[]{"--http", "-c", "get_physical_placement", "-b", bucketName, "-o", "beowulf.txt"});
836881
final CommandResponse response = Util.command(client, args);
837882
assertTrue(response.getMessage().contains("| Object Name | ID | In Cache | Length | Offset | Latest | Version |"));
838883
assertTrue(response.getMessage().contains("| beowulf.txt |"));
@@ -901,7 +946,7 @@ public void getBulkWithPipeMissingFile() throws Exception {
901946
}
902947
}
903948

904-
@Test (expected = com.spectralogic.ds3client.helpers.JobRecoveryNotActiveException.class)
949+
@Test(expected = com.spectralogic.ds3client.helpers.JobRecoveryNotActiveException.class)
905950
public void recoverGetBulk() throws Exception {
906951
assumeThat(Util.getBlackPearlVersion(client), greaterThan(1.2));
907952

@@ -1084,7 +1129,7 @@ private Path createAFile(final String fileName) throws Exception {
10841129
final Path filePath = Paths.get(fileName);
10851130
final Path createdFilePath = Files.createFile(filePath);
10861131

1087-
write(createdFilePath, new byte[] { 0 });
1132+
write(createdFilePath, new byte[]{0});
10881133

10891134
return createdFilePath;
10901135
}

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/ArgumentFactory.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ public final class ArgumentFactory {
5757
public static final Option PREFIX = Option.builder("p").hasArg(true).argName("prefix")
5858
.desc("Used with get operations to restore only objects whose names start with prefix "
5959
+ "and with put operations, to prepend a prefix to object name(s)").build();
60+
public static final Option VERSION = Option.builder("v").argName("version")
61+
.desc("Show version information").build();
62+
public static final Option VERSIONID = Option.builder().longOpt("versionId").desc("Version ID value").hasArg(true)
63+
.build();
6064
public static final Option OBJECT_NAME = Option.builder("o").hasArg(true).argName("objectFileName")
6165
.desc("The name of the object to be retrieved or stored").build();
6266
public static final Option ID = Option.builder("i").hasArg(true).argName("optionId")

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/Arguments.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,8 @@ public int getBufferSize() throws BadArgumentException {
306306

307307
// convenience getters for public options
308308
public String getBucket() { return this.getOptionValue(BUCKET.getOpt()); }
309+
public boolean getVersion() { return this.optionExists(VERSION.getOpt());}
310+
public String getVersionId() { return this.getOptionValue(VERSIONID.getLongOpt());}
309311
public String getDirectory() { return this.getOptionValue(DIRECTORY.getOpt()); }
310312
public String getObjectName() { return this.getOptionValue(OBJECT_NAME.getOpt()); }
311313
public boolean isForce() { return this.optionExists(FORCE.getLongOpt()); }

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetBucket.java‎

Lines changed: 25 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import com.spectralogic.ds3cli.models.GetBucketResult;
2626
import com.spectralogic.ds3cli.views.cli.GetBucketView;
2727
import com.spectralogic.ds3cli.views.json.DataView;
28+
import com.spectralogic.ds3client.commands.GetBucketRequest;
29+
import com.spectralogic.ds3client.commands.GetBucketResponse;
2830
import com.spectralogic.ds3client.commands.spectrads3.GetBucketSpectraS3Request;
2931
import com.spectralogic.ds3client.commands.spectrads3.GetBucketSpectraS3Response;
3032
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
@@ -34,23 +36,28 @@
3436
import com.spectralogic.ds3client.utils.Guard;
3537
import org.apache.commons.cli.Option;
3638

39+
import java.util.List;
40+
3741
import static com.spectralogic.ds3cli.ArgumentFactory.BUCKET;
3842
import static com.spectralogic.ds3cli.ArgumentFactory.PREFIX;
43+
import static com.spectralogic.ds3cli.ArgumentFactory.VERSION;
3944

4045
public class GetBucket extends CliCommand<GetBucketResult> {
4146

4247
private final static ImmutableList<Option> requiredArgs = ImmutableList.of(BUCKET);
43-
private final static ImmutableList<Option> optionalArgs = ImmutableList.of(PREFIX);
48+
private final static ImmutableList<Option> optionalArgs = ImmutableList.of(PREFIX, VERSION);
4449

4550
private String bucket;
4651
private String prefix;
52+
private boolean version;
4753

4854
@Override
4955
public CliCommand init(final Arguments args) throws Exception {
5056
processCommandOptions(requiredArgs, optionalArgs, args);
5157

5258
this.bucket = args.getBucket();
5359
this.prefix = args.getPrefix();
60+
this.version = args.getVersion();
5461
return this;
5562
}
5663

@@ -59,21 +66,22 @@ public GetBucketResult call() throws Exception {
5966

6067
try {
6168
// GetBucketDetail to get both name and id
62-
final GetBucketSpectraS3Response response = getClient().getBucketSpectraS3(new GetBucketSpectraS3Request(bucket));
69+
GetBucketRequest getBucketRequest = new GetBucketRequest(bucket);
70+
getBucketRequest.withVersions(version);
71+
getBucketRequest.withPrefix(prefix);
72+
final GetBucketSpectraS3Request getBucketSpectraS3Request = new GetBucketSpectraS3Request(bucket);
73+
final GetBucketSpectraS3Response response = getClient().getBucketSpectraS3(getBucketSpectraS3Request);
6374
final Bucket bucketDetails = response.getBucketResult();
64-
// helper.listObjects only takes bucket name
65-
this.bucket = response.getBucketResult().getName();
75+
final GetBucketResponse bucket = getClient().getBucket(getBucketRequest);
6676

67-
final Ds3ClientHelpers helper = Ds3ClientHelpers.wrap(getClient());
68-
final Iterable<Contents> objects;
69-
if (this.prefix == null) {
70-
objects = helper.listObjects(bucket);
71-
}
72-
else {
73-
objects = helper.listObjects(bucket, this.prefix);
77+
final List<Contents> contents;
78+
if (version) {
79+
contents = bucket.getListBucketResult().getVersionedObjects();
80+
} else {
81+
contents = bucket.getListBucketResult().getObjects();
7482
}
7583

76-
return new GetBucketResult(bucketDetails, objects);
84+
return new GetBucketResult(bucketDetails, contents);
7785
} catch(final FailedRequestException e) {
7886
if(e.getStatusCode() == 404) {
7987
throw new CommandException("Error: Unknown bucket.", e);
@@ -88,7 +96,11 @@ public View<GetBucketResult> getView() {
8896
case JSON:
8997
return new DataView();
9098
default:
91-
return new com.spectralogic.ds3cli.views.cli.GetBucketView();
99+
if(version) {
100+
return new com.spectralogic.ds3cli.views.cli.GetVersionedBucketView();
101+
} else {
102+
return new com.spectralogic.ds3cli.views.cli.GetBucketView();
103+
}
92104
}
93105
}
94106

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/command/GetObject.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public class GetObject extends CliCommand<DefaultResult> {
5151

5252
private final static ImmutableList<Option> requiredArgs = ImmutableList.of(BUCKET, OBJECT_NAME);
5353
private final static ImmutableList<Option> optionalArgs = ImmutableList.of(DIRECTORY, SYNC,
54-
FORCE, NUMBER_OF_THREADS, PRIORITY, RANGE_OFFSET, RANGE_LENGTH, DISCARD, FILE_METADATA);
54+
FORCE, NUMBER_OF_THREADS, PRIORITY, RANGE_OFFSET, RANGE_LENGTH, DISCARD, FILE_METADATA, VERSIONID);
5555

5656
private String bucketName;
5757
private String objectName;
@@ -63,10 +63,12 @@ public class GetObject extends CliCommand<DefaultResult> {
6363
private long rangeLength = 0L;
6464
private boolean discard;
6565
private boolean restoreMetadata;
66+
private String versionId;
6667

6768
@Override
6869
public CliCommand init(final Arguments args) throws Exception {
6970
processCommandOptions(requiredArgs, optionalArgs, args);
71+
this.versionId = args.getVersionId();
7072
this.bucketName = args.getBucket();
7173
this.priority = args.getPriority();
7274
this.objectName = args.getObjectName();
@@ -116,7 +118,11 @@ public DefaultResult call() throws Exception {
116118
ds3Obj = new PartialDs3Object(this.objectName.replace("\\", "/"),
117119
Range.byLength(this.rangeOffset, this.rangeLength));
118120
} else {
119-
ds3Obj = new Ds3Object(this.objectName.replace("\\", "/"));
121+
if (Guard.isStringNullOrEmpty(versionId)) {
122+
ds3Obj = new Ds3Object(this.objectName.replace("\\", "/"));
123+
} else {
124+
ds3Obj = new Ds3Object(this.objectName.replace("\\", "/"), versionId);
125+
}
120126
}
121127

122128
if (this.sync && FileUtils.fileExists(filePath)) {

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/util/Guard.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@ public static String nullGuard(final String message) {
2929
return message;
3030
}
3131

32+
public static String nullGuard(final Object o) {
33+
if (o == null) {
34+
return "N/A";
35+
} else {
36+
return o.toString();
37+
}
38+
}
39+
3240
public static String nullGuardToString(final Object o) {
3341
// default alternate
3442
return nullGuardToString(o, "N/A");

‎ds3_java_cli/src/main/java/com/spectralogic/ds3cli/views/cli/GetBucketView.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public String render(final GetBucketResult br) {
3939
}
4040
this.contents = br.getResult();
4141
initTable(ImmutableList.of("File Name", "Size", "Owner", "Last Modified", "ETag"));
42-
setTableDataAlignment(ImmutableList.of(ASCIITable.ALIGN_LEFT, ASCIITable.ALIGN_RIGHT, ASCIITable.ALIGN_RIGHT ,ASCIITable.ALIGN_LEFT, ASCIITable.ALIGN_RIGHT ));
42+
setTableDataAlignment(ImmutableList.of(ASCIITable.ALIGN_LEFT, ASCIITable.ALIGN_RIGHT, ASCIITable.ALIGN_RIGHT ,ASCIITable.ALIGN_LEFT, ASCIITable.ALIGN_RIGHT));
4343

4444
final String bucketContents = ASCIITable.getInstance().getTable(getHeaders(), formatTableContents());
4545

0 commit comments

Comments
 (0)