Skip to content

Commit 6ba6823

Browse files
rpmoorerpmoore
committed
Minor refactoring for JobPartTracker. Added a workaround for a NPE that was detected when the NodeId reported back in a chunk is missing.
1 parent 5494e2c commit 6ba6823

File tree

7 files changed

+128
-52
lines changed

7 files changed

+128
-52
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
allprojects {
1717
group = 'com.spectralogic.ds3'
18-
version = '3.0.3'
18+
version = '3.0.4'
1919
}
2020

2121
subprojects {

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package com.spectralogic.ds3client.helpers;
1717

18+
import com.google.common.collect.ImmutableMap;
1819
import com.google.common.util.concurrent.Futures;
1920
import com.google.common.util.concurrent.ListenableFuture;
2021
import com.google.common.util.concurrent.ListeningExecutorService;
@@ -61,15 +62,16 @@ public void transferChunks(
6162
final Iterable<Objects> chunks)
6263
throws SignatureException, IOException, XmlProcessingException {
6364
LOG.debug("Getting ready to process chunks");
64-
final Map<UUID, JobNode> nodeMap = buildNodeMap(nodes);
65+
final ImmutableMap<UUID, JobNode> nodeMap = buildNodeMap(nodes);
6566
LOG.debug("Starting executor service");
6667
final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(maxParallelRequests));
6768
LOG.debug("Executor service started");
6869
try {
6970
final List<ListenableFuture<?>> tasks = new ArrayList<>();
7071
for (final Objects chunk : chunks) {
7172
LOG.debug("Processing parts for chunk: " + chunk.getChunkId().toString());
72-
final Ds3Client client = mainClient.newForNode(nodeMap.get(chunk.getNodeId()));
73+
74+
final Ds3Client client = getClient(nodeMap, chunk.getNodeId(), mainClient);
7375
for (final BulkObject ds3Object : chunk.getObjects()) {
7476
final ObjectPart part = new ObjectPart(ds3Object.getOffset(), ds3Object.getLength());
7577
if (this.partTracker.containsPart(ds3Object.getName(), part)) {
@@ -93,12 +95,23 @@ public Object call() throws Exception {
9395
}
9496
}
9597

96-
private static Map<UUID, JobNode> buildNodeMap(final Iterable<JobNode> nodes) {
97-
final Map<UUID, JobNode> nodeMap = new HashMap<>();
98-
for(final JobNode node: nodes) {
98+
private static Ds3Client getClient(final ImmutableMap<UUID, JobNode> nodeMap, final UUID nodeId, final Ds3Client mainClient) {
99+
final JobNode jobNode = nodeMap.get(nodeId);
100+
101+
if (jobNode == null) {
102+
LOG.warn("The jobNode was not found, returning the existing client");
103+
return mainClient;
104+
}
105+
106+
return mainClient.newForNode(jobNode);
107+
}
108+
109+
private static ImmutableMap<UUID, JobNode> buildNodeMap(final Iterable<JobNode> nodes) {
110+
final ImmutableMap.Builder<UUID, JobNode> nodeMap = ImmutableMap.builder();
111+
for (final JobNode node: nodes) {
99112
nodeMap.put(node.getId(), node);
100113
}
101-
return nodeMap;
114+
return nodeMap.build();
102115
}
103116

104117
private static void executeWithExceptionHandling(final List<ListenableFuture<?>> tasks)

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java‎

Lines changed: 7 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -15,51 +15,16 @@
1515

1616
package com.spectralogic.ds3client.helpers;
1717

18-
import java.util.Map;
19-
2018
/**
2119
* This class manages parts for all of the objects in the job. It aggregates
2220
* ObjectPartTracker implementations, which manage the parts for a single
2321
* object.
2422
*/
25-
public class JobPartTracker {
26-
private final Map<String, ObjectPartTracker> trackers;
27-
28-
public JobPartTracker(final Map<String, ObjectPartTracker> trackers) {
29-
this.trackers = trackers;
30-
}
31-
32-
public void completePart(final String key, final ObjectPart objectPart) {
33-
trackers.get(key).completePart(objectPart);
34-
}
35-
36-
public boolean containsPart(final String key, final ObjectPart objectPart) {
37-
return trackers.get(key).containsPart(objectPart);
38-
}
39-
40-
public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) {
41-
for (final ObjectPartTracker tracker : this.trackers.values()) {
42-
tracker.attachDataTransferredListener(listener);
43-
}
44-
return this;
45-
}
46-
47-
public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) {
48-
for (final ObjectPartTracker tracker : this.trackers.values()) {
49-
tracker.attachObjectCompletedListener(listener);
50-
}
51-
return this;
52-
}
53-
54-
public void removeDataTransferredListener(final DataTransferredListener listener) {
55-
for (final ObjectPartTracker tracker : this.trackers.values()) {
56-
tracker.removeDataTransferredListener(listener);
57-
}
58-
}
59-
60-
public void removeObjectCompletedListener(final ObjectCompletedListener listener) {
61-
for (final ObjectPartTracker tracker : this.trackers.values()) {
62-
tracker.removeObjectCompletedListener(listener);
63-
}
64-
}
23+
public interface JobPartTracker {
24+
void completePart(final String key, final ObjectPart objectPart);
25+
boolean containsPart(final String key, final ObjectPart objectPart);
26+
JobPartTracker attachDataTransferredListener(final DataTransferredListener listener);
27+
JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener);
28+
void removeDataTransferredListener(final DataTransferredListener listener);
29+
void removeObjectCompletedListener(final ObjectCompletedListener listener);
6530
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public static JobPartTracker buildPartTracker(final Iterable<BulkObject> objects
3131
for (final BulkObject bulkObject : Preconditions.checkNotNull(objects)) {
3232
multimap.put(bulkObject.getName(), new ObjectPart(bulkObject.getOffset(), bulkObject.getLength()));
3333
}
34-
return new JobPartTracker(new HashMap<>(Maps.transformEntries(
34+
return new JobPartTrackerImpl(new HashMap<>(Maps.transformEntries(
3535
multimap.asMap(),
3636
new BuildObjectPartTrackerFromObjectPartGroup()
3737
)));

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerImpl.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package com.spectralogic.ds3client.helpers;
2+
3+
import java.util.Map;
4+
5+
public class JobPartTrackerImpl implements JobPartTracker {
6+
7+
private final Map<String, ObjectPartTracker> trackers;
8+
9+
public JobPartTrackerImpl(final Map<String, ObjectPartTracker> trackers) {
10+
this.trackers = trackers;
11+
}
12+
13+
public void completePart(final String key, final ObjectPart objectPart) {
14+
trackers.get(key).completePart(objectPart);
15+
}
16+
17+
public boolean containsPart(final String key, final ObjectPart objectPart) {
18+
return trackers.get(key).containsPart(objectPart);
19+
}
20+
21+
public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) {
22+
for (final ObjectPartTracker tracker : this.trackers.values()) {
23+
tracker.attachDataTransferredListener(listener);
24+
}
25+
return this;
26+
}
27+
28+
public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) {
29+
for (final ObjectPartTracker tracker : this.trackers.values()) {
30+
tracker.attachObjectCompletedListener(listener);
31+
}
32+
return this;
33+
}
34+
35+
public void removeDataTransferredListener(final DataTransferredListener listener) {
36+
for (final ObjectPartTracker tracker : this.trackers.values()) {
37+
tracker.removeDataTransferredListener(listener);
38+
}
39+
}
40+
41+
public void removeObjectCompletedListener(final ObjectCompletedListener listener) {
42+
for (final ObjectPartTracker tracker : this.trackers.values()) {
43+
tracker.removeObjectCompletedListener(listener);
44+
}
45+
}
46+
}

‎ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/ChunkTransferrer_Test.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
package com.spectralogic.ds3client.helpers;
16+
17+
import com.google.common.collect.ImmutableList;
18+
import com.spectralogic.ds3client.Ds3Client;
19+
import com.spectralogic.ds3client.models.JobNode;
20+
import com.spectralogic.ds3client.models.Objects;
21+
import com.spectralogic.ds3client.serializer.XmlProcessingException;
22+
import org.junit.Test;
23+
24+
import java.io.IOException;
25+
import java.security.SignatureException;
26+
import java.util.ArrayList;
27+
import java.util.UUID;
28+
29+
import static org.mockito.Matchers.any;
30+
import static org.mockito.Mockito.mock;
31+
import static org.mockito.Mockito.times;
32+
import static org.mockito.Mockito.verify;
33+
34+
public class ChunkTransferrer_Test {
35+
36+
@Test
37+
public void nullJobNode() throws XmlProcessingException, SignatureException, IOException {
38+
final Ds3Client client = mock(Ds3Client.class);
39+
40+
final ChunkTransferrer chunkTransferrer = new ChunkTransferrer(null, client, null, 1);
41+
final ImmutableList.Builder<Objects> objectsBuilder = ImmutableList.builder();
42+
43+
final Objects objects = new Objects();
44+
objects.setChunkId(UUID.randomUUID());
45+
objects.setChunkNumber(0);
46+
objectsBuilder.add(objects);
47+
48+
chunkTransferrer.transferChunks(new ArrayList<JobNode>(), objectsBuilder.build());
49+
50+
verify(client, times(0)).newForNode((JobNode) any());
51+
}
52+
}

‎ds3-sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public void trackerCallsTrackers() {
3030
final Map<String, ObjectPartTracker> trackers = new HashMap<>();
3131
trackers.put("foo", fooTracker);
3232
trackers.put("bar", barTracker);
33-
final JobPartTracker jobPartTracker = new JobPartTracker(trackers);
33+
final JobPartTracker jobPartTracker = new JobPartTrackerImpl(trackers);
3434

3535
final ObjectPart[] partsRemoved = {
3636
new ObjectPart(10, 11),
@@ -65,7 +65,7 @@ public void TrackerEventsForward()
6565
final List<Long> sizes = new ArrayList<>();
6666
final List<String> objects = new ArrayList<>();
6767

68-
final JobPartTracker jobPartTracker = new JobPartTracker(trackers);
68+
final JobPartTracker jobPartTracker = new JobPartTrackerImpl(trackers);
6969
jobPartTracker.attachDataTransferredListener(new DataTransferredListener() {
7070
@Override
7171
public void dataTransferred(final long size) {

0 commit comments

Comments
 (0)