Skip to content

Commit a151b65

Browse files
author
吴炳亨
authored
Merge pull request #272 from NigelWu95/dev
Dev
2 parents 2d32066 + 90b9d78 commit a151b65

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1489
-793
lines changed

‎docs/downloadfile.md

+6-6
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ queries=
1919
pre-down=
2020
add-prefix=
2121
rm-prefix=
22-
download-timeout=
23-
save-path=
22+
down-timeout=
23+
down-path=
2424
private=
2525
```
2626
|参数名|参数值及类型 | 含义|
2727
|-----|-------|-----|
28-
|process| 下载资源时设置为download | 表示资源下载操作|
28+
|process| 下载资源时设置为 download | 表示资源下载操作|
2929
|protocol| http/https| 使用 http 还是 https 访问资源进行下载（默认 http）|
3030
|domain| 域名字符串| 当数据源数据的资源为文件名列表时，需要设置进行访问的域名，当指定 url-index 时无需设置|
3131
|indexes|字符串| 设置输入行中 key 字段的下标（有默认值），参考[数据源 indexes 设置](datasource.md#1-公共参数)|
@@ -36,8 +36,8 @@ private=
3636
|pre-down| true/false|为 true 时表示预下载，即下载的内容不保存为文件，为 false 表示保存成本地文件，默认为 false|
3737
|add-prefix| 字符串| 表示为保存的文件名添加指定前缀|
3838
|rm-prefix| 字符串| 表示将得到的目标文件名去除存在的指定前缀后再作为保存的文件名|
39-
|download-timeout| 时间，单位秒|设置下载文件的超时时间，默认 1200s，下载大文件可根据需要调整|
40-
|save-path| 文件保存路径|设置下载文件的保存路径，为本地的目录名称|
39+
|down-timeout| 时间，单位秒|设置下载文件的超时时间，默认 1200s，下载大文件可根据需要调整|
40+
|down-path| 文件保存路径|设置下载文件的保存路径，为本地的目录名称，下载文件进行保存时必须设置|
4141
|private| 数据源私有类型|是否是对私有空间资源进行下载，选择对应的私有类型，参考[私有访问](#资源需要私有签名)|
4242

4343
### 关于 queries 参数
@@ -60,6 +60,6 @@ key 字段加上 domain 的方式访问源文件地址，key 下标用 indexes
6060

6161
### 命令行参数方式
6262
```
63-
-process=download -ak= -sk= -protocol= -domain= -host= -add-prefix= -save-path= ...
63+
-process=download -ak= -sk= -protocol= -domain= -host= -add-prefix= -down-path= ...
6464
```
6565

‎resources/log4j2.properties

+13-13
Original file line numberDiff line numberDiff line change
@@ -13,27 +13,27 @@ appender.console.layout.pattern = [%d{yyyy-MM-dd'T'HH:mm:ss.SSS}] %m%n
1313
rootLogger.level = info
1414
rootLogger.appenderRef.stdout.ref = STDOUT
1515

16-
#appender.procedure.type = File
17-
#appender.procedure.name = procedure
18-
#appender.procedure.fileName = procedure.log
19-
#appender.procedure.append = false
16+
appender.procedure.type = File
17+
appender.procedure.name = procedure
18+
appender.procedure.fileName = ${sys:procedure}.log
19+
appender.procedure.append = false
2020
appender.procedure.filter.threshold.type = ThresholdFilter
2121
appender.procedure.filter.threshold.level = warn
2222
appender.procedure.filter.threshold.onMatch = DENY
2323
appender.procedure.filter.threshold.onMisMatch = NEUTRAL
2424

25-
appender.procedure.type = RollingFile
26-
appender.procedure.name = procedure
27-
appender.procedure.fileName = ${sys:procedure}.log
28-
appender.procedure.filePattern = logs/.procedure-%i.log
29-
appender.procedure.policies.type = Policies
25+
#appender.procedure.type = RollingFile
26+
#appender.procedure.name = procedure
27+
#appender.procedure.fileName = ${sys:procedure}.log
28+
#appender.procedure.filePattern = logs/.procedure-%i.log
29+
#appender.procedure.policies.type = Policies
3030
#appender.procedure.policies.time.type = TimeBasedTriggeringPolicy
3131
#appender.procedure.policies.time.interval = 2
3232
#appender.procedure.policies.time.modulate = true
33-
appender.procedure.policies.size.type = SizeBasedTriggeringPolicy
34-
appender.procedure.policies.size.size = 20MB
35-
appender.procedure.strategy.type = DefaultRolloverStrategy
36-
appender.procedure.strategy.max = 1
33+
#appender.procedure.policies.size.type = SizeBasedTriggeringPolicy
34+
#appender.procedure.policies.size.size = 20MB
35+
#appender.procedure.strategy.type = DefaultRolloverStrategy
36+
#appender.procedure.strategy.max = 1
3737

3838
logger.procedure.name = procedure
3939
logger.procedure.level = info

‎src/main/java/com/qiniu/convert/IndentStringPair.java

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package com.qiniu.convert;
2+
3+
import com.qiniu.interfaces.KeyValuePair;
4+
import com.qiniu.util.FileUtils;
5+
6+
public class IndentStringPair implements KeyValuePair<String, String> {
7+
8+
private StringBuilder stringBuilder = new StringBuilder();
9+
private StringBuilder parentPath = new StringBuilder();
10+
private String separator;
11+
private int size;
12+
13+
public IndentStringPair(String separator) {
14+
this.separator = separator;
15+
}
16+
17+
@Override
18+
public void putKey(String key, String value) {
19+
value = value.replace("\n", "%0a").replace("\r", "%0d");
20+
int num = value.split(FileUtils.pathSeparator).length;
21+
if (num > 1) {
22+
if (value.endsWith(FileUtils.pathSeparator)) {
23+
parentPath.append(value, 0, value.substring(0, value.length() - 1)
24+
.lastIndexOf(FileUtils.pathSeparator));
25+
} else {
26+
parentPath.append(value, 0, value.lastIndexOf(FileUtils.pathSeparator));
27+
}
28+
StringBuilder stringBuilder = new StringBuilder();
29+
for (int j = 1; j < num; j++) stringBuilder.append("\t");
30+
stringBuilder.append(value.replace(parentPath, "").substring(1));
31+
stringBuilder.append(stringBuilder.toString()).append("\t");
32+
parentPath.delete(0, parentPath.length()).append(value).append("\t|");
33+
} else {
34+
stringBuilder.append(value).append("\t");
35+
parentPath.append(value).append("\t|");
36+
}
37+
stringBuilder.append(separator).append(value);
38+
size++;
39+
}
40+
41+
@Override
42+
public void put(String key, String value) {
43+
stringBuilder.append(separator).append(value);
44+
size++;
45+
}
46+
47+
@Override
48+
public void put(String key, Boolean value) {
49+
stringBuilder.append(separator).append(value);
50+
size++;
51+
}
52+
53+
@Override
54+
public void put(String key, Integer value) {
55+
stringBuilder.append(separator).append(value);
56+
size++;
57+
}
58+
59+
@Override
60+
public void put(String key, Long value) {
61+
stringBuilder.append(separator).append(value);
62+
size++;
63+
}
64+
65+
@Override
66+
public String getProtoEntity() {
67+
return parentPath.append(stringBuilder.deleteCharAt(stringBuilder.length() - 1)).substring(1);
68+
}
69+
70+
@Override
71+
public int size() {
72+
return size;
73+
}
74+
}

‎src/main/java/com/qiniu/convert/JsonObjectPair.java

+10
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,21 @@ public class JsonObjectPair implements KeyValuePair<String, JsonObject> {
77

88
private JsonObject jsonObject = new JsonObject();
99

10+
@Override
11+
public void putKey(String key, String value) {
12+
jsonObject.addProperty(key, value);
13+
}
14+
1015
@Override
1116
public void put(String key, String value) {
1217
jsonObject.addProperty(key, value);
1318
}
1419

20+
@Override
21+
public void put(String key, Boolean value) {
22+
jsonObject.addProperty(key, value);
23+
}
24+
1525
@Override
1626
public void put(String key, Integer value) {
1727
jsonObject.addProperty(key, value);

‎src/main/java/com/qiniu/convert/LineToMap.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public LineToMap(String parseType, String separator, String addKeyPrefix, String
1818
this.lineParser = line -> process(addKeyPrefix, rmKeyPrefix, ConvertingUtils.toPair(line, indexMap, new StringMapPair()));
1919
} else if ("csv".equals(parseType)) {
2020
this.lineParser = line -> process(addKeyPrefix, rmKeyPrefix, ConvertingUtils.toPair(line, ",", indexMap, new StringMapPair()));
21-
} else if ("tab".equals(parseType) || "self".equals(parseType)) {
21+
} else if ("tab".equals(parseType)) {
2222
this.lineParser = line -> process(addKeyPrefix, rmKeyPrefix, ConvertingUtils.toPair(line, separator, indexMap, new StringMapPair()));
2323
} else {
2424
throw new IOException("please check your format for line to map.");

‎src/main/java/com/qiniu/convert/MapToString.java

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ public MapToString(String format, String separator, List<String> fields) throws
1818
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(","));
1919
} else if ("tab".equals(format)) {
2020
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(separator));
21+
} else if ("yaml".equals(format)) {
22+
stringFormatter = line -> ConvertingUtils.toStringWithIndent(line, fields);
2123
} else {
2224
throw new IOException("please check your format for map to string.");
2325
}

‎src/main/java/com/qiniu/convert/StringBuilderPair.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,23 @@ public StringBuilderPair(String separator) {
1313
}
1414

1515
@Override
16-
public void put(String key, String value) {
16+
public void putKey(String key, String value) {
1717
stringBuilder.append(separator).append(value.replace("\n", "%0a").replace("\r", "%0d"));
1818
size++;
1919
}
2020

21+
@Override
22+
public void put(String key, String value) {
23+
stringBuilder.append(separator).append(value);
24+
size++;
25+
}
26+
27+
@Override
28+
public void put(String key, Boolean value) {
29+
stringBuilder.append(separator).append(value);
30+
size++;
31+
}
32+
2133
@Override
2234
public void put(String key, Integer value) {
2335
stringBuilder.append(separator).append(value);

‎src/main/java/com/qiniu/convert/StringMapPair.java

+10
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,21 @@ public class StringMapPair implements KeyValuePair<String, Map<String, String>>
99

1010
private Map<String, String> stringMap = new HashMap<>();
1111

12+
@Override
13+
public void putKey(String key, String value) {
14+
stringMap.put(key, value);
15+
}
16+
1217
@Override
1318
public void put(String key, String value) {
1419
stringMap.put(key, value);
1520
}
1621

22+
@Override
23+
public void put(String key, Boolean value) {
24+
stringMap.put(key, String.valueOf(value));
25+
}
26+
1727
@Override
1828
public void put(String key, Integer value) {
1929
stringMap.put(key, String.valueOf(value));

‎src/main/java/com/qiniu/datasource/AliOssContainer.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,11 @@
2020
import com.qiniu.util.ConvertingUtils;
2121
import com.qiniu.util.FileUtils;
2222

23-
import java.io.BufferedWriter;
2423
import java.io.IOException;
2524
import java.util.List;
2625
import java.util.Map;
2726

28-
public class AliOssContainer extends CloudStorageContainer<OSSObjectSummary, BufferedWriter, Map<String, String>> {
27+
public class AliOssContainer extends CloudStorageContainer<OSSObjectSummary, Map<String, String>> {
2928

3029
// private String accessKeyId;
3130
// private String accessKeySecret;
@@ -72,6 +71,8 @@ protected ITypeConvert<OSSObjectSummary, String> getNewStringConverter() {
7271
IStringFormat<OSSObjectSummary> stringFormatter;
7372
if ("json".equals(saveFormat)) {
7473
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new JsonObjectPair()).toString();
74+
} else if ("yaml".equals(saveFormat)) {
75+
stringFormatter = line -> ConvertingUtils.toStringWithIndent(line, fields);
7576
} else {
7677
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(saveSeparator));
7778
}
@@ -84,7 +85,7 @@ public String convertToV(OSSObjectSummary line) throws IOException {
8485
}
8586

8687
@Override
87-
protected IResultOutput<BufferedWriter> getNewResultSaver(String order) throws IOException {
88+
protected IResultOutput getNewResultSaver(String order) throws IOException {
8889
return order != null ? new FileSaveMapper(savePath, getSourceName(), order) : new FileSaveMapper(savePath);
8990
}
9091

‎src/main/java/com/qiniu/datasource/AwsS3Container.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,11 @@
1919
import com.qiniu.interfaces.IResultOutput;
2020
import com.qiniu.util.ConvertingUtils;
2121

22-
import java.io.BufferedWriter;
2322
import java.io.IOException;
2423
import java.util.List;
2524
import java.util.Map;
2625

27-
public class AwsS3Container extends CloudStorageContainer<S3ObjectSummary, BufferedWriter, Map<String, String>> {
26+
public class AwsS3Container extends CloudStorageContainer<S3ObjectSummary, Map<String, String>> {
2827

2928
// private String accessKeyId;
3029
// private String secretKey;
@@ -89,6 +88,8 @@ protected ITypeConvert<S3ObjectSummary, String> getNewStringConverter() {
8988
IStringFormat<S3ObjectSummary> stringFormatter;
9089
if ("json".equals(saveFormat)) {
9190
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new JsonObjectPair()).toString();
91+
} else if ("yaml".equals(saveFormat)) {
92+
stringFormatter = line -> ConvertingUtils.toStringWithIndent(line, fields);
9293
} else {
9394
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(saveSeparator));
9495
}
@@ -101,7 +102,7 @@ public String convertToV(S3ObjectSummary line) throws IOException {
101102
}
102103

103104
@Override
104-
protected IResultOutput<BufferedWriter> getNewResultSaver(String order) throws IOException {
105+
protected IResultOutput getNewResultSaver(String order) throws IOException {
105106
return order != null ? new FileSaveMapper(savePath, getSourceName(), order) : new FileSaveMapper(savePath);
106107
}
107108

‎src/main/java/com/qiniu/datasource/BaiduBosContainer.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,11 @@
1717
import com.qiniu.util.CloudApiUtils;
1818
import com.qiniu.util.ConvertingUtils;
1919

20-
import java.io.BufferedWriter;
2120
import java.io.IOException;
2221
import java.util.List;
2322
import java.util.Map;
2423

25-
public class BaiduBosContainer extends CloudStorageContainer<BosObjectSummary, BufferedWriter, Map<String, String>> {
24+
public class BaiduBosContainer extends CloudStorageContainer<BosObjectSummary, Map<String, String>> {
2625

2726
// private String accessKeyId;
2827
// private String accessKeySecret;
@@ -72,6 +71,8 @@ protected ITypeConvert<BosObjectSummary, String> getNewStringConverter() {
7271
IStringFormat<BosObjectSummary> stringFormatter;
7372
if ("json".equals(saveFormat)) {
7473
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new JsonObjectPair()).toString();
74+
} else if ("yaml".equals(saveFormat)) {
75+
stringFormatter = line -> ConvertingUtils.toStringWithIndent(line, fields);
7576
} else {
7677
stringFormatter = line -> ConvertingUtils.toPair(line, fields, new StringBuilderPair(saveSeparator));
7778
}
@@ -84,7 +85,7 @@ public String convertToV(BosObjectSummary line) throws IOException {
8485
}
8586

8687
@Override
87-
protected IResultOutput<BufferedWriter> getNewResultSaver(String order) throws IOException {
88+
protected IResultOutput getNewResultSaver(String order) throws IOException {
8889
return order != null ? new FileSaveMapper(savePath, getSourceName(), order) : new FileSaveMapper(savePath);
8990
}
9091

0 commit comments

Comments
 (0)