Skip to content

Commit 45764da

Browse files
awol2005exawol2005ex
committed
[feature][plugin][s3writer] s3writer add fileType(text,orc,parquet) and sslEnabled property config
1 parent 50440b9 commit 45764da

File tree

9 files changed

+1343
-175
lines changed

9 files changed

+1343
-175
lines changed

‎plugin/writer/s3writer/pom.xml

+295
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,301 @@
5757
</exclusions>
5858
</dependency>
5959

60+
<dependency>
61+
<groupId>com.google.protobuf</groupId>
62+
<artifactId>protobuf-java</artifactId>
63+
<version>${protobuf.version}</version>
64+
</dependency>
65+
66+
<dependency>
67+
<groupId>org.apache.avro</groupId>
68+
<artifactId>avro</artifactId>
69+
<version>${avro.version}</version>
70+
<exclusions>
71+
<exclusion>
72+
<groupId>com.fasterxml.jackson.core</groupId>
73+
<artifactId>jackson-databind</artifactId>
74+
</exclusion>
75+
</exclusions>
76+
</dependency>
77+
78+
<dependency>
79+
<groupId>org.apache.hadoop</groupId>
80+
<artifactId>hadoop-common</artifactId>
81+
<version>${hadoop.version}</version>
82+
<exclusions>
83+
<exclusion>
84+
<groupId>com.fasterxml.jackson.core</groupId>
85+
<artifactId>jackson-databind</artifactId>
86+
</exclusion>
87+
<exclusion>
88+
<groupId>org.codehaus.jackson</groupId>
89+
<artifactId>jackson-core-asl</artifactId>
90+
</exclusion>
91+
<exclusion>
92+
<groupId>org.codehaus.jackson</groupId>
93+
<artifactId>jackson-mapper-asl</artifactId>
94+
</exclusion>
95+
<exclusion>
96+
<groupId>com.fasterxml.woodstox</groupId>
97+
<artifactId>woodstox-core</artifactId>
98+
</exclusion>
99+
<exclusion>
100+
<groupId>commons-codec</groupId>
101+
<artifactId>commons-codec</artifactId>
102+
</exclusion>
103+
<exclusion>
104+
<groupId>commons-net</groupId>
105+
<artifactId>commons-net</artifactId>
106+
</exclusion>
107+
<exclusion>
108+
<groupId>io.netty</groupId>
109+
<artifactId>netty</artifactId>
110+
</exclusion>
111+
<exclusion>
112+
<groupId>log4j</groupId>
113+
<artifactId>log4j</artifactId>
114+
</exclusion>
115+
<exclusion>
116+
<groupId>net.minidev</groupId>
117+
<artifactId>json-smart</artifactId>
118+
</exclusion>
119+
<exclusion>
120+
<groupId>org.codehaus.jettison</groupId>
121+
<artifactId>jettison</artifactId>
122+
</exclusion>
123+
<exclusion>
124+
<groupId>org.eclipse.jetty</groupId>
125+
<artifactId>jetty-server</artifactId>
126+
</exclusion>
127+
<exclusion>
128+
<groupId>org.xerial.snappy</groupId>
129+
<artifactId>snappy-java</artifactId>
130+
</exclusion>
131+
<exclusion>
132+
<groupId>org.apache.zookeeper</groupId>
133+
<artifactId>zookeeper</artifactId>
134+
</exclusion>
135+
<exclusion>
136+
<groupId>org.eclipse.jetty</groupId>
137+
<artifactId>jetty-util</artifactId>
138+
</exclusion>
139+
</exclusions>
140+
</dependency>
141+
142+
<dependency>
143+
<groupId>org.apache.hadoop</groupId>
144+
<artifactId>hadoop-aws</artifactId>
145+
<version>${hadoop.version}</version>
146+
<exclusions>
147+
<exclusion>
148+
<groupId>com.fasterxml.jackson.core</groupId>
149+
<artifactId>jackson-databind</artifactId>
150+
</exclusion>
151+
<exclusion>
152+
<groupId>org.codehaus.jackson</groupId>
153+
<artifactId>jackson-core-asl</artifactId>
154+
</exclusion>
155+
<exclusion>
156+
<groupId>org.codehaus.jackson</groupId>
157+
<artifactId>jackson-mapper-asl</artifactId>
158+
</exclusion>
159+
<exclusion>
160+
<groupId>com.fasterxml.woodstox</groupId>
161+
<artifactId>woodstox-core</artifactId>
162+
</exclusion>
163+
<exclusion>
164+
<groupId>commons-codec</groupId>
165+
<artifactId>commons-codec</artifactId>
166+
</exclusion>
167+
<exclusion>
168+
<groupId>commons-net</groupId>
169+
<artifactId>commons-net</artifactId>
170+
</exclusion>
171+
<exclusion>
172+
<groupId>io.netty</groupId>
173+
<artifactId>netty</artifactId>
174+
</exclusion>
175+
<exclusion>
176+
<groupId>log4j</groupId>
177+
<artifactId>log4j</artifactId>
178+
</exclusion>
179+
<exclusion>
180+
<groupId>net.minidev</groupId>
181+
<artifactId>json-smart</artifactId>
182+
</exclusion>
183+
<exclusion>
184+
<groupId>org.codehaus.jettison</groupId>
185+
<artifactId>jettison</artifactId>
186+
</exclusion>
187+
<exclusion>
188+
<groupId>org.eclipse.jetty</groupId>
189+
<artifactId>jetty-server</artifactId>
190+
</exclusion>
191+
<exclusion>
192+
<groupId>org.xerial.snappy</groupId>
193+
<artifactId>snappy-java</artifactId>
194+
</exclusion>
195+
<exclusion>
196+
<groupId>org.apache.zookeeper</groupId>
197+
<artifactId>zookeeper</artifactId>
198+
</exclusion>
199+
<exclusion>
200+
<groupId>org.eclipse.jetty</groupId>
201+
<artifactId>jetty-util</artifactId>
202+
</exclusion>
203+
</exclusions>
204+
</dependency>
205+
206+
<dependency>
207+
<groupId>org.apache.hadoop</groupId>
208+
<artifactId>hadoop-mapreduce-client-core</artifactId>
209+
<version>${hadoop.version}</version>
210+
<exclusions>
211+
<exclusion>
212+
<groupId>com.fasterxml.jackson.core</groupId>
213+
<artifactId>jackson-databind</artifactId>
214+
</exclusion>
215+
<exclusion>
216+
<groupId>commons-codec</groupId>
217+
<artifactId>commons-codec</artifactId>
218+
</exclusion>
219+
<exclusion>
220+
<groupId>io.netty</groupId>
221+
<artifactId>netty</artifactId>
222+
</exclusion>
223+
<exclusion>
224+
<groupId>org.eclipse.jetty</groupId>
225+
<artifactId>jetty-util</artifactId>
226+
</exclusion>
227+
</exclusions>
228+
</dependency>
229+
230+
<dependency>
231+
<groupId>org.apache.hive</groupId>
232+
<artifactId>hive-common</artifactId>
233+
<version>${hive.version}</version>
234+
<exclusions>
235+
<exclusion>
236+
<groupId>slf4j-api</groupId>
237+
<artifactId>org.slf4j</artifactId>
238+
</exclusion>
239+
<exclusion>
240+
<groupId>com.google.guava</groupId>
241+
<artifactId>guava</artifactId>
242+
</exclusion>
243+
<exclusion>
244+
<groupId>com.fasterxml.jackson.core</groupId>
245+
<artifactId>jackson-databind</artifactId>
246+
</exclusion>
247+
<exclusion>
248+
<groupId>commons-net</groupId>
249+
<artifactId>commons-net</artifactId>
250+
</exclusion>
251+
<exclusion>
252+
<groupId>log4j</groupId>
253+
<artifactId>log4j</artifactId>
254+
</exclusion>
255+
<exclusion>
256+
<groupId>org.apache.ant</groupId>
257+
<artifactId>ant</artifactId>
258+
</exclusion>
259+
<exclusion>
260+
<groupId>org.apache.hadoop</groupId>
261+
<artifactId>hadoop-yarn-server-common</artifactId>
262+
</exclusion>
263+
<exclusion>
264+
<groupId>org.apache.hadoop</groupId>
265+
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
266+
</exclusion>
267+
<exclusion>
268+
<groupId>org.eclipse.jetty</groupId>
269+
<artifactId>jetty-server</artifactId>
270+
</exclusion>
271+
<exclusion>
272+
<groupId>org.eclipse.jetty</groupId>
273+
<artifactId>jetty-io</artifactId>
274+
</exclusion>
275+
<exclusion>
276+
<groupId>org.eclipse.jetty</groupId>
277+
<artifactId>jetty-http</artifactId>
278+
</exclusion>
279+
<exclusion>
280+
<groupId>org.eclipse.jetty</groupId>
281+
<artifactId>jetty-client</artifactId>
282+
</exclusion>
283+
<exclusion>
284+
<groupId>org.apache.thrift</groupId>
285+
<artifactId>libthrift</artifactId>
286+
</exclusion>
287+
<exclusion>
288+
<groupId>org.apache.zookeeper</groupId>
289+
<artifactId>zookeeper</artifactId>
290+
</exclusion>
291+
<exclusion>
292+
<groupId>io.netty</groupId>
293+
<artifactId>netty</artifactId>
294+
</exclusion>
295+
</exclusions>
296+
</dependency>
297+
298+
<dependency>
299+
<groupId>org.apache.hive</groupId>
300+
<artifactId>hive-serde</artifactId>
301+
<version>3.1.3</version>
302+
<scope>compile</scope>
303+
</dependency>
304+
305+
<dependency>
306+
<groupId>org.apache.hive</groupId>
307+
<artifactId>hive-storage-api</artifactId>
308+
<version>2.8.1</version>
309+
</dependency>
310+
311+
<dependency>
312+
<groupId>org.apache.orc</groupId>
313+
<artifactId>orc-core</artifactId>
314+
<version>${orc.version}</version>
315+
</dependency>
316+
317+
<dependency>
318+
<groupId>org.apache.parquet</groupId>
319+
<artifactId>parquet-avro</artifactId>
320+
<version>${parquet.version}</version>
321+
<exclusions>
322+
<exclusion>
323+
<groupId>commons-codec</groupId>
324+
<artifactId>commons-codec</artifactId>
325+
</exclusion>
326+
<exclusion>
327+
<groupId>org.codehaus.jackson</groupId>
328+
<artifactId>jackson-core-asl</artifactId>
329+
</exclusion>
330+
<exclusion>
331+
<groupId>org.codehaus.jackson</groupId>
332+
<artifactId>jackson-mapper-asl</artifactId>
333+
</exclusion>
334+
</exclusions>
335+
</dependency>
336+
337+
<dependency>
338+
<groupId>org.apache.parquet</groupId>
339+
<artifactId>parquet-hadoop-bundle</artifactId>
340+
<version>${parquet.version}</version>
341+
</dependency>
342+
343+
<dependency>
344+
<groupId>org.xerial.snappy</groupId>
345+
<artifactId>snappy-java</artifactId>
346+
<version>${snappy.version}</version>
347+
</dependency>
348+
349+
<dependency>
350+
<groupId>com.fasterxml.woodstox</groupId>
351+
<artifactId>woodstox-core</artifactId>
352+
<version>${woodstox.version}</version>
353+
</dependency>
354+
60355
</dependencies>
61356

62357
<build>

‎plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Key.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
import com.wgzhao.addax.common.base.Key;
44

5-
public class S3Key extends Key
6-
{
5+
public class S3Key extends Key {
76
public static final String REGION = "region";
87

98
public static final String ENDPOINT = "endpoint";
@@ -22,4 +21,11 @@ public class S3Key extends Key
2221
public static final String DEFAULT_SUFFIX = "defaultSuffix";
2322

2423
public static final String PATH_STYLE_ACCESS_ENABLED = "pathStyleAccessEnabled";
24+
25+
public static final String FILE_TYPE = "fileType";
26+
27+
28+
public static final String SSLENABLED = "sslEnabled";
29+
30+
2531
}

‎plugin/writer/s3writer/src/main/java/com/wgzhao/addax/plugin/writer/s3writer/S3Util.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -11,27 +11,27 @@
1111

1212
import static com.wgzhao.addax.common.spi.ErrorCode.ILLEGAL_VALUE;
1313

14-
public class S3Util
15-
{
14+
public class S3Util {
1615
public static S3Client initS3Client(Configuration conf) {
1716
String regionStr = conf.getString(S3Key.REGION);
1817
Region region = Region.of(regionStr);
1918
String accessId = conf.getString(S3Key.ACCESS_ID);
2019
String accessKey = conf.getString(S3Key.ACCESS_KEY);
21-
String pathStyleAccessEnabled =conf.getString(S3Key.PATH_STYLE_ACCESS_ENABLED,"");
20+
String pathStyleAccessEnabled = conf.getString(S3Key.PATH_STYLE_ACCESS_ENABLED, "");
2221

23-
return initS3Client(conf.getString(S3Key.ENDPOINT), region, accessId, accessKey ,pathStyleAccessEnabled);
22+
23+
return initS3Client(conf.getString(S3Key.ENDPOINT), region, accessId, accessKey, pathStyleAccessEnabled);
2424

2525
}
2626

27-
public static S3Client initS3Client(String endpoint, Region region, String accessId, String accessKey ,String pathStyleAccessEnabled) {
27+
public static S3Client initS3Client(String endpoint, Region region, String accessId, String accessKey, String pathStyleAccessEnabled) {
2828
if (null == region) {
2929
region = Region.of("ap-northeast-1");
3030
}
3131
try {
3232
AwsBasicCredentials awsCreds = AwsBasicCredentials.create(accessId, accessKey);
3333
return S3Client.builder().serviceConfiguration(e -> {
34-
if("true".equals(pathStyleAccessEnabled)) {
34+
if ("true".equals(pathStyleAccessEnabled)) {
3535
e.pathStyleAccessEnabled(true);
3636
}
3737
})

0 commit comments

Comments
 (0)