Skip to content
This repository was archived by the owner on Feb 26, 2025. It is now read-only.

Commit e40295b

Browse files
jarnaizjarnaiz
committed
integrated PR #47 and #34 
1 parent 927f24a commit e40295b

File tree

11 files changed

+182
-31
lines changed

11 files changed

+182
-31
lines changed

‎src/main/java/com/upplication/s3fs/S3FileAttributes.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@ public class S3FileAttributes implements BasicFileAttributes {
1111
private final boolean directory;
1212
private final boolean regularFile;
1313
private final String key;
14+
private long cacheCreated;
1415

1516
public S3FileAttributes(String key, FileTime lastModifiedTime, long size, boolean isDirectory, boolean isRegularFile) {
1617
this.key = key;
1718
this.lastModifiedTime = lastModifiedTime;
1819
this.size = size;
1920
directory = isDirectory;
2021
regularFile = isRegularFile;
22+
23+
cacheCreated = System.currentTimeMillis();
2124
}
2225

2326
@Override
@@ -69,4 +72,14 @@ public Object fileKey() {
6972
public String toString() {
7073
return format("[%s: lastModified=%s, size=%s, isDirectory=%s, isRegularFile=%s]", key, lastModifiedTime, size, directory, regularFile);
7174
}
75+
76+
public long getCacheCreated() {
77+
return cacheCreated;
78+
}
79+
80+
// for testing
81+
82+
public void setCacheCreated(long time) {
83+
this.cacheCreated = time;
84+
}
7285
}

‎src/main/java/com/upplication/s3fs/S3FileSystem.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,14 @@ public class S3FileSystem extends FileSystem implements Comparable<S3FileSystem>
3030
private final String key;
3131
private final AmazonS3 client;
3232
private final String endpoint;
33+
private int cache;
3334

34-
public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) {
35+
public S3FileSystem(S3FileSystemProvider provider, String key, AmazonS3 client, String endpoint) {
3536
this.provider = provider;
3637
this.key = key;
3738
this.client = client;
3839
this.endpoint = endpoint;
40+
this.cache = 60000; // 1 minute cache for the s3Path
3941
}
4042

4143
@Override
@@ -183,4 +185,8 @@ public boolean equals(Object obj) {
183185
public int compareTo(S3FileSystem o) {
184186
return key.compareTo(o.getKey());
185187
}
188+
189+
public int getCache() {
190+
return cache;
191+
}
186192
}

‎src/main/java/com/upplication/s3fs/S3FileSystemProvider.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import com.google.common.collect.ImmutableList;
4646
import com.google.common.collect.ImmutableSet;
4747
import com.google.common.collect.Sets;
48+
import com.upplication.s3fs.util.Cache;
4849
import com.upplication.s3fs.util.S3Utils;
4950

5051
/**
@@ -81,6 +82,7 @@ public class S3FileSystemProvider extends FileSystemProvider {
8182
private static final ConcurrentMap<String, S3FileSystem> fileSystems = new ConcurrentHashMap<>();
8283

8384
private S3Utils s3Utils = new S3Utils();
85+
private Cache cache = new Cache();
8486

8587
@Override
8688
public String getScheme() {
@@ -441,8 +443,17 @@ public <V extends FileAttributeView> V getFileAttributeView(Path path, Class<V>
441443
public <A extends BasicFileAttributes> A readAttributes(Path path, Class<A> type, LinkOption... options) throws IOException {
442444
S3Path s3Path = toS3Path(path);
443445
if (type == BasicFileAttributes.class) {
444-
return type.cast(s3Utils.getS3FileAttributes(s3Path));
446+
if (cache.isInTime(s3Path.getFileSystem().getCache(), s3Path.getFileAttributes())) {
447+
A result = type.cast(s3Path.getFileAttributes());
448+
s3Path.setFileAttributes(null);
449+
return result;
450+
} else {
451+
S3FileAttributes attrs = s3Utils.getS3FileAttributes(s3Path);
452+
s3Path.setFileAttributes(attrs);
453+
return type.cast(attrs);
454+
}
445455
}
456+
446457
throw new UnsupportedOperationException(format("only %s supported", BasicFileAttributes.class));
447458
}
448459

@@ -534,9 +545,17 @@ public boolean isOpen(S3FileSystem s3FileSystem) {
534545

535546
/**
536547
* only 4 testing
537-
* @return
538548
*/
549+
539550
protected static ConcurrentMap<String, S3FileSystem> getFilesystems() {
540551
return fileSystems;
541552
}
553+
554+
public Cache getCache() {
555+
return cache;
556+
}
557+
558+
public void setCache(Cache cache) {
559+
this.cache = cache;
560+
}
542561
}

‎src/main/java/com/upplication/s3fs/S3Iterator.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.upplication.s3fs;
22

3+
import java.nio.file.NoSuchFileException;
34
import java.nio.file.Path;
45
import java.util.ArrayList;
56
import java.util.Arrays;
@@ -8,13 +9,12 @@
89
import java.util.List;
910
import java.util.NoSuchElementException;
1011
import java.util.Set;
11-
12-
import com.amazonaws.services.s3.AmazonS3;
1312
import com.amazonaws.services.s3.model.ListObjectsRequest;
1413
import com.amazonaws.services.s3.model.ObjectListing;
1514
import com.amazonaws.services.s3.model.S3ObjectSummary;
1615
import com.google.common.collect.Lists;
1716
import com.google.common.collect.Sets;
17+
import com.upplication.s3fs.util.S3Utils;
1818

1919
/**
2020
* S3 iterator over folders at first level.
@@ -32,6 +32,8 @@ public class S3Iterator implements Iterator<Path> {
3232
private int size;
3333
private boolean incremental;
3434

35+
private S3Utils s3Utils = new S3Utils();
36+
3537
public S3Iterator(S3Path path) {
3638
this(path, false);
3739
}
@@ -138,7 +140,7 @@ private void parseObjectListing(String key, List<S3Path> listPath, ObjectListing
138140
String immediateDescendantKey = getImmediateDescendant(key, objectSummaryKey);
139141
if (immediateDescendantKey != null) {
140142
S3Path descendentPart = new S3Path(fileSystem, fileStore, fileSystem.key2Parts(immediateDescendantKey));
141-
143+
descendentPart.setFileAttributes(s3Utils.toS3FileAttributes(objectSummary, descendentPart.getKey()));
142144
if (!listPath.contains(descendentPart)) {
143145
listPath.add(descendentPart);
144146
}

‎src/main/java/com/upplication/s3fs/S3Path.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ public class S3Path implements Path {
5757
*/
5858
private S3FileSystem fileSystem;
5959

60+
/**
61+
* S3FileAttributes cache
62+
*/
63+
private S3FileAttributes fileAttributes;
64+
6065
/**
6166
* path must be a string of the form "/{bucket}", "/{bucket}/{key}" or just
6267
* "{key}".
@@ -431,6 +436,14 @@ public int hashCode() {
431436
return result;
432437
}
433438

439+
public S3FileAttributes getFileAttributes() {
440+
return fileAttributes;
441+
}
442+
443+
public void setFileAttributes(S3FileAttributes fileAttributes) {
444+
this.fileAttributes = fileAttributes;
445+
}
446+
434447
// ~ helpers methods
435448

436449
private static Function<String, String> strip(final String... strs) {
@@ -455,9 +468,9 @@ public boolean apply(@Nullable String input) {
455468
};
456469
}
457470

458-
/*
459-
* delete redundant "/" and empty parts
460-
*/
471+
/*
472+
* delete redundant "/" and empty parts
473+
*/
461474
private abstract static class KeyParts {
462475
private static ImmutableList<String> parse(String[] parts) {
463476
return ImmutableList.copyOf(filter(transform(Arrays.asList(parts), strip("/")), notEmpty()));

‎src/main/java/com/upplication/s3fs/util/Cache.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package com.upplication.s3fs.util;
2+
3+
import com.upplication.s3fs.S3FileAttributes;
4+
5+
public class Cache {
6+
7+
/**
8+
* check if the cache of the S3FileAttributes is still valid
9+
* @param cache int cache time of the fileAttributes in milliseconds
10+
* @param fileAttributes S3FileAttributes to check if is still valid, can be null
11+
* @return true or false, if cache are -1 and fileAttributes are not null then always return true
12+
*/
13+
public boolean isInTime(int cache, S3FileAttributes fileAttributes){
14+
if (fileAttributes == null) {
15+
return false;
16+
}
17+
18+
if (cache == -1) {
19+
return true;
20+
}
21+
22+
return getCurrentTime() - cache <= fileAttributes.getCacheCreated();
23+
}
24+
25+
public long getCurrentTime(){
26+
return System.currentTimeMillis();
27+
}
28+
}

‎src/main/java/com/upplication/s3fs/util/S3Utils.java‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515
public class S3Utils {
1616

1717
/**
18-
* Get the {@link com.amazonaws.services.s3.model.S3ObjectSummary} that represent this Path or her first child if this path not exists
19-
* @param s3Path {@link com.upplication.s3fs.S3Path}
20-
* @return {@link com.amazonaws.services.s3.model.S3ObjectSummary}
21-
* @throws java.nio.file.NoSuchFileException if not found the path and any child
18+
* Get the {@link S3ObjectSummary} that represent this Path or her first child if this path not exists
19+
* @param s3Path {@link S3Path}
20+
* @return {@link S3ObjectSummary}
21+
* @throws NoSuchFileException if not found the path and any child
2222
*/
2323
public S3ObjectSummary getS3ObjectSummary(S3Path s3Path) throws NoSuchFileException {
2424
String key = s3Path.getKey();
@@ -61,9 +61,19 @@ public S3ObjectSummary getS3ObjectSummary(S3Path s3Path) throws NoSuchFileExcept
6161
* @return S3FileAttributes
6262
*/
6363
public S3FileAttributes getS3FileAttributes(S3Path s3Path) throws NoSuchFileException {
64-
String key = s3Path.getKey();
6564
S3ObjectSummary objectSummary = getS3ObjectSummary(s3Path);
65+
return toS3FileAttributes(objectSummary, s3Path.getKey());
66+
}
6667

68+
/**
69+
* convert S3ObjectSummary to S3FileAttributes
70+
* @param objectSummary S3ObjectSummary mandatory not null, the real objectSummary with
71+
* exactly the same key than the key param or the immediate descendant
72+
* if it is a virtual directory
73+
* @param key String the real key that can be exactly equal than the objectSummary or
74+
* @return S3FileAttributes
75+
*/
76+
public S3FileAttributes toS3FileAttributes(S3ObjectSummary objectSummary, String key) {
6777
// parse the data to BasicFileAttributes.
6878
FileTime lastModifiedTime = null;
6979
if (objectSummary.getLastModified() != null){
@@ -77,8 +87,7 @@ public S3FileAttributes getS3FileAttributes(S3Path s3Path) throws NoSuchFileExce
7787
if (key.endsWith("/") && resolvedKey.equals(key) ||
7888
resolvedKey.equals(key + "/")) {
7989
directory = true;
80-
}
81-
else if (key.isEmpty()) { // is a bucket (no key)
90+
} else if (key.isEmpty()) { // is a bucket (no key)
8291
directory = true;
8392
resolvedKey = "/";
8493
}
@@ -88,8 +97,9 @@ else if (!resolvedKey.equals(key) && resolvedKey.startsWith(key)) { // is a dire
8897
size = 0;
8998
// delete extra part
9099
resolvedKey = key + "/";
91-
} else
100+
} else {
92101
regularFile = true;
102+
}
93103
return new S3FileAttributes(resolvedKey, lastModifiedTime, size, directory, regularFile);
94104
}
95-
}
105+
}

‎src/test/java/com/upplication/s3fs/CacheTest.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package com.upplication.s3fs;
2+
3+
import com.upplication.s3fs.util.Cache;
4+
import org.junit.Test;
5+
6+
import static org.junit.Assert.assertFalse;
7+
import static org.junit.Assert.assertTrue;
8+
import static org.mockito.Mockito.doReturn;
9+
import static org.mockito.Mockito.spy;
10+
11+
public class CacheTest {
12+
13+
@Test
14+
public void cacheIsInclusive() {
15+
Cache cache = spy(new Cache());
16+
doReturn(300L).when(cache).getCurrentTime();
17+
18+
S3FileAttributes attributes = new S3FileAttributes("key", null, 0, false, true);
19+
attributes.setCacheCreated(0);
20+
21+
boolean result = cache.isInTime(300, attributes);
22+
assertTrue(result);
23+
}
24+
25+
@Test
26+
public void outOfTime() {
27+
Cache cache = spy(new Cache());
28+
doReturn(200L).when(cache).getCurrentTime();
29+
30+
S3FileAttributes attributes = new S3FileAttributes("key", null, 0, false, true);
31+
attributes.setCacheCreated(0);
32+
33+
boolean result = cache.isInTime(100, attributes);
34+
assertFalse(result);
35+
}
36+
37+
@Test
38+
public void infinite() {
39+
Cache cache = spy(new Cache());
40+
doReturn(200L).when(cache).getCurrentTime();
41+
42+
S3FileAttributes attributes = new S3FileAttributes("key", null, 0, false, true);
43+
attributes.setCacheCreated(100);
44+
45+
boolean result = cache.isInTime(-1, attributes);
46+
assertTrue(result);
47+
}
48+
}

‎src/test/java/com/upplication/s3fs/S3FileSystemProviderTest.java‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,14 @@
3030
import java.util.Set;
3131
import java.util.concurrent.TimeUnit;
3232

33-
import com.upplication.s3fs.util.IOUtils;
33+
import com.upplication.s3fs.util.*;
3434
import org.junit.After;
3535
import org.junit.Before;
3636
import org.junit.Test;
3737

3838
import com.amazonaws.services.s3.model.AccessControlList;
3939
import com.amazonaws.services.s3.model.Owner;
4040
import com.google.common.collect.ImmutableMap;
41-
import com.upplication.s3fs.util.AmazonS3ClientMock;
42-
import com.upplication.s3fs.util.AmazonS3MockFactory;
43-
import com.upplication.s3fs.util.MockBucket;
4441
import org.mockito.ArgumentMatcher;
4542

4643
public class S3FileSystemProviderTest extends S3UnitTestBase {
@@ -525,7 +522,7 @@ public void anotherOutputStream() throws IOException {
525522
public void seekable() throws IOException {
526523
// fixtures
527524
AmazonS3ClientMock client = AmazonS3MockFactory.getAmazonClientMock();
528-
client.bucket("bucketA").dir("dir").file("dir/file");
525+
client.bucket("bucketA").dir("dir").file("dir/file", "".getBytes());
529526

530527
Path base = createNewS3FileSystem().getPath("/bucketA", "dir");
531528

@@ -999,8 +996,9 @@ public void getFileAttributeView() {
999996
@Test
1000997
public void readAttributesFileEmpty() throws IOException {
1001998
// fixtures
999+
final String content = "";
10021000
AmazonS3ClientMock client = AmazonS3MockFactory.getAmazonClientMock();
1003-
client.bucket("bucketA").dir("dir").file("dir/file1");
1001+
client.bucket("bucketA").dir("dir").file("dir/file1", content.getBytes());
10041002

10051003
Path file1 = createNewS3FileSystem().getPath("/bucketA/dir/file1");
10061004

@@ -1115,6 +1113,25 @@ public void readAttributesDirectoryNotExistsAtAmazon() throws IOException {
11151113
assertEquals(fileAttributes.lastAccessTime().to(TimeUnit.SECONDS), fileAttributes.lastModifiedTime().to(TimeUnit.SECONDS));
11161114
}
11171115

1116+
@Test
1117+
public void readAttributesRegenerateCacheWhenNotExists() throws IOException {
1118+
// fixtures
1119+
final String content = "";
1120+
AmazonS3ClientMock client = AmazonS3MockFactory.getAmazonClientMock();
1121+
client.bucket("bucketA").dir("dir").file("dir/file1", content.getBytes());
1122+
1123+
Path file1 = createNewS3FileSystem().getPath("/bucketA/dir/file1");
1124+
// create the cache
1125+
s3fsProvider.readAttributes(file1, BasicFileAttributes.class);
1126+
assertNotNull(((S3Path) file1).getFileAttributes());
1127+
1128+
s3fsProvider.readAttributes(file1, BasicFileAttributes.class);
1129+
assertNull(((S3Path) file1).getFileAttributes());
1130+
1131+
s3fsProvider.readAttributes(file1, BasicFileAttributes.class);
1132+
assertNotNull(((S3Path) file1).getFileAttributes());
1133+
}
1134+
11181135
@Test(expected = NoSuchFileException.class)
11191136
public void readAttributesFileNotExists() throws IOException {
11201137
// fixtures

0 commit comments

Comments
 (0)