Skip to content

Commit e95ab23

Browse files
author
Samrat
committed
Native Flink s3 fileSystem
1 parent 04f3047 commit e95ab23

File tree

7 files changed

+719
-0
lines changed

7 files changed

+719
-0
lines changed

‎flink-filesystems/flink-s3-fs-native/pom.xml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
21+
22+
<modelVersion>4.0.0</modelVersion>
23+
24+
<parent>
25+
<groupId>org.apache.flink</groupId>
26+
<artifactId>flink-filesystems</artifactId>
27+
<version>2.2-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>flink-s3-fs-native</artifactId>
31+
<name>Flink : FileSystems : S3 FS Native</name>
32+
<packaging>jar</packaging>
33+
34+
<properties>
35+
<aws.sdk.v2.version>2.25.64</aws.sdk.v2.version>
36+
<japicmp.skip>true</japicmp.skip>
37+
</properties>
38+
39+
<dependencies>
40+
<!-- Flink core FileSystem API -->
41+
<dependency>
42+
<groupId>org.apache.flink</groupId>
43+
<artifactId>flink-core</artifactId>
44+
<version>${project.version}</version>
45+
<scope>provided</scope>
46+
</dependency>
47+
48+
<!-- AWS SDK v2 (S3 + STS + auth) -->
49+
<dependency>
50+
<groupId>software.amazon.awssdk</groupId>
51+
<artifactId>s3</artifactId>
52+
<version>${aws.sdk.v2.version}</version>
53+
</dependency>
54+
<dependency>
55+
<groupId>software.amazon.awssdk</groupId>
56+
<artifactId>sts</artifactId>
57+
<version>${aws.sdk.v2.version}</version>
58+
</dependency>
59+
<dependency>
60+
<groupId>software.amazon.awssdk</groupId>
61+
<artifactId>auth</artifactId>
62+
<version>${aws.sdk.v2.version}</version>
63+
</dependency>
64+
65+
<!-- test -->
66+
<dependency>
67+
<groupId>org.apache.flink</groupId>
68+
<artifactId>flink-test-utils</artifactId>
69+
<version>${project.version}</version>
70+
<scope>test</scope>
71+
</dependency>
72+
</dependencies>
73+
74+
<build>
75+
<plugins>
76+
<plugin>
77+
<groupId>org.apache.maven.plugins</groupId>
78+
<artifactId>maven-deploy-plugin</artifactId>
79+
<configuration>
80+
<skip>true</skip>
81+
</configuration>
82+
</plugin>
83+
</plugins>
84+
</build>
85+
86+
</project>
87+
88+

‎flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3/nativev2/NativeS3Options.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.flink.fs.s3.nativev2;
20+
21+
import org.apache.flink.configuration.ConfigOption;
22+
import org.apache.flink.configuration.ConfigOptions;
23+
import org.apache.flink.configuration.MemorySize;
24+
25+
/** Configuration options for the native S3 file system. */
26+
final class NativeS3Options {
27+
28+
private NativeS3Options() {}
29+
30+
static final ConfigOption<String> REGION =
31+
ConfigOptions.key("s3.region").stringType().noDefaultValue();
32+
33+
static final ConfigOption<String> ENDPOINT =
34+
ConfigOptions.key("s3.endpoint").stringType().noDefaultValue();
35+
36+
static final ConfigOption<Boolean> PATH_STYLE_ACCESS =
37+
ConfigOptions.key("s3.path.style.access").booleanType().defaultValue(false);
38+
39+
static final ConfigOption<MemorySize> READ_CHUNK_SIZE =
40+
ConfigOptions.key("s3.read.chunk-size")
41+
.memoryType()
42+
.defaultValue(MemorySize.ofMebiBytes(8));
43+
44+
static final ConfigOption<MemorySize> MULTIPART_PART_SIZE =
45+
ConfigOptions.key("s3.upload.part.size")
46+
.memoryType()
47+
.defaultValue(MemorySize.ofMebiBytes(64));
48+
49+
static final ConfigOption<MemorySize> MULTIPART_THRESHOLD =
50+
ConfigOptions.key("s3.upload.multipart.threshold")
51+
.memoryType()
52+
.defaultValue(MemorySize.ofMebiBytes(64));
53+
}

0 commit comments

Comments
 (0)