Skip to content

Commit 395509e

Browse files
author
吴炳亨
authored
Merge pull request #300 from NigelWu95/dev
Dev
2 parents bda9e62 + 0fe36f0 commit 395509e

25 files changed

+86
-53
lines changed

‎.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,10 @@ assembly/
2727
.DS_Store
2828

2929
logs*
30+
31+
### vscode
32+
.vscode
33+
.classpath
34+
.factorypath
35+
.project
36+
.settings/

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version=8.4.3
1+
version=8.4.4
22

33
package_no_test:
44
mvn clean package -Dmaven.test.skip=true

‎pom.xml

+6-2
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.qiniu</groupId>
88
<artifactId>qsuits</artifactId>
9-
<version>8.4.3</version>
9+
<version>8.4.4</version>
1010
<name>qsuits</name>
1111
<description>qiniu-suits is a efficient tools for qiniu api implemented by java8.</description>
1212
<url>https://github.com/NigelWu95/qiniu-suits-java</url>
@@ -75,7 +75,11 @@
7575
<artifactId>log4j-slf4j-impl</artifactId>
7676
<version>2.10.0</version>
7777
</dependency>
78-
78+
<dependency>
79+
<groupId>com.squareup.okhttp3</groupId>
80+
<artifactId>okhttp</artifactId>
81+
<version>3.14.4</version>
82+
</dependency>
7983
<dependency>
8084
<groupId>com.qcloud</groupId>
8185
<artifactId>cos_api</artifactId>

‎qsuits.iml

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@
1212
<orderEntry type="jdk" jdkName="1.8" jdkType="JavaSDK" />
1313
<orderEntry type="sourceFolder" forTests="false" />
1414
<orderEntry type="library" name="Maven: com.qiniu:qiniu-java-sdk:7.2.26" level="project" />
15-
<orderEntry type="library" scope="RUNTIME" name="Maven: com.squareup.okhttp3:okhttp:3.14.4" level="project" />
16-
<orderEntry type="library" scope="RUNTIME" name="Maven: com.squareup.okio:okio:1.17.2" level="project" />
1715
<orderEntry type="library" name="Maven: com.google.code.gson:gson:2.8.2" level="project" />
1816
<orderEntry type="library" name="Maven: commons-io:commons-io:1.3.2" level="project" />
1917
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.28" level="project" />
2018
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.13.2" level="project" />
2119
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-core:2.13.2" level="project" />
2220
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-slf4j-impl:2.10.0" level="project" />
21+
<orderEntry type="library" name="Maven: com.squareup.okhttp3:okhttp:3.14.4" level="project" />
22+
<orderEntry type="library" name="Maven: com.squareup.okio:okio:1.17.2" level="project" />
2323
<orderEntry type="library" name="Maven: com.qcloud:cos_api:5.5.1" level="project" />
2424
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.3" level="project" />
2525
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.6" level="project" />

‎src/main/java/com/qiniu/entry/EntryMain.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ public static void main(String[] args) throws Exception {
7373
boolean interactive = paramsMap.containsKey("interactive") && Boolean.parseBoolean(paramsMap.get("interactive"));
7474
CommonParams commonParams = single ? new CommonParams(paramsMap) : new CommonParams(entryParam);
7575
QSuitsEntry qSuitsEntry = new QSuitsEntry(entryParam, commonParams);
76-
ILineProcess<Map<String, String>> processor = single || interactive ? qSuitsEntry.whichNextProcessor(true) :
77-
qSuitsEntry.getProcessor();
76+
ILineProcess<Map<String, String>> processor = single || interactive ?
77+
qSuitsEntry.whichNextProcessor(true) : qSuitsEntry.getProcessor();
7878
if (processVerify && processor != null) {
7979
String process = processor.getProcessName();
8080
if (processor.getNextProcessor() != null) process = processor.getNextProcessor().getProcessName();
@@ -88,9 +88,8 @@ public static void main(String[] args) throws Exception {
8888
}
8989
Scanner scanner = new Scanner(System.in);
9090
String an = scanner.next();
91-
if (!an.equalsIgnoreCase("y") && !an.equalsIgnoreCase("yes")) {
92-
return;
93-
}
91+
scanner.close();
92+
if (!an.equalsIgnoreCase("y") && !an.equalsIgnoreCase("yes")) return;
9493
}
9594
}
9695
if (single) {

‎src/main/java/com/qiniu/entry/QSuitsEntry.java

+4
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,10 @@ public ILineProcess<Map<String, String>> whichNextProcessor(boolean single) thro
561561
processor.setRetryTimes(retryTimes);
562562
processor.setCheckType(entryParam.getValue("check", "").trim());
563563
if (privateProcessor != null) {
564+
processor.updateFields(new HashMap<String, String>(){{
565+
put("domain", null);
566+
put("urlIndex", "url");
567+
}});
564568
privateProcessor.setNextProcessor(processor);
565569
return privateProcessor;
566570
}

‎src/main/java/com/qiniu/interfaces/ILineProcess.java

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.io.IOException;
44
import java.util.List;
5+
import java.util.Map;
56

67
public interface ILineProcess<T> {
78

@@ -17,6 +18,8 @@ default void setBatchSize(int batchSize) throws IOException {}
1718

1819
default void setRetryTimes(int retryTimes) {}
1920

21+
default void updateFields(Map<String, String> fieldsMap) {}
22+
2023
default void setNextProcessor(ILineProcess<T> nextProcessor) {}
2124

2225
default void setCheckType(String checkType) {}

‎src/main/java/com/qiniu/process/Base.java

+15
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@
99
import com.qiniu.util.*;
1010

1111
import java.io.IOException;
12+
import java.lang.reflect.Field;
1213
import java.util.ArrayList;
1314
import java.util.List;
15+
import java.util.Map;
1416
import java.util.concurrent.atomic.AtomicInteger;
1517
import java.util.stream.Collectors;
1618

@@ -80,6 +82,19 @@ public void setRetryTimes(int retryTimes) {
8082
this.retryTimes = retryTimes < 1 ? 5 : retryTimes;
8183
}
8284

85+
public void updateFields(Map<String, String> fieldsMap) {
86+
Class cClass = this.getClass();
87+
fieldsMap.forEach((key, value) -> {
88+
try {
89+
Field field = cClass.getDeclaredField(key);
90+
field.setAccessible(true);
91+
field.set(this, value);
92+
} catch (Exception ignored) {
93+
ignored.printStackTrace();
94+
}
95+
});
96+
}
97+
8398
@SuppressWarnings("unchecked")
8499
public Base<T> clone() throws CloneNotSupportedException {
85100
Base<T> base = (Base<T>)super.clone();

‎src/main/java/com/qiniu/process/other/DownloadFile.java

+17-15
Original file line numberDiff line numberDiff line change
@@ -25,37 +25,38 @@ public class DownloadFile extends Base<Map<String, String>> {
2525
private Configuration configuration;
2626
private HttpDownloader downloader;
2727

28-
public DownloadFile(Configuration configuration, String protocol, String domain, String urlIndex, String host, int[] range,
29-
String suffixOrQuery, String addPrefix, String rmPrefix, String downPath, String savePath, int saveIndex)
30-
throws IOException {
28+
public DownloadFile(Configuration configuration, String protocol, String domain, String urlIndex, String host,
29+
int[] range, String suffixOrQuery, String addPrefix, String rmPrefix, String downPath, String savePath,
30+
int saveIndex) throws IOException {
3131
super("download", "", "", null, savePath, saveIndex);
3232
set(configuration, protocol, domain, urlIndex, host, range, suffixOrQuery, addPrefix, rmPrefix, downPath);
3333
downloader = new HttpDownloader(configuration);
3434
}
3535

36-
public DownloadFile(Configuration configuration, String protocol, String domain, String urlIndex, String host, int[] range,
37-
String suffixOrQuery, String addPrefix, String rmPrefix, String downPath) throws IOException {
36+
public DownloadFile(Configuration configuration, String protocol, String domain, String urlIndex, String host,
37+
int[] range, String suffixOrQuery, String addPrefix, String rmPrefix, String downPath) throws IOException {
3838
super("download", "", "", null);
3939
set(configuration, protocol, domain, urlIndex, host, range, suffixOrQuery, addPrefix, rmPrefix, downPath);
4040
downloader = new HttpDownloader(configuration);
4141
}
4242

4343
public DownloadFile(Configuration configuration, String protocol, String domain, String urlIndex, String host,
44-
int[] range, String suffixOrQuery, String addPrefix, String rmPrefix,
45-
String downPath, String savePath) throws IOException {
46-
this(configuration, protocol, domain, urlIndex, host, range, suffixOrQuery, addPrefix, rmPrefix,
47-
downPath, savePath, 0);
44+
int[] range, String suffixOrQuery, String addPrefix, String rmPrefix, String downPath, String savePath)
45+
throws IOException {
46+
this(configuration, protocol, domain, urlIndex, host, range, suffixOrQuery, addPrefix, rmPrefix, downPath,
47+
savePath, 0);
4848
}
4949

50-
private void set(Configuration configuration, String protocol, String domain, String urlIndex, String host, int[] range,
51-
String suffixOrQuery, String addPrefix, String rmPrefix, String downPath) throws IOException {
50+
private void set(Configuration configuration, String protocol, String domain, String urlIndex, String host,
51+
int[] range, String suffixOrQuery, String addPrefix, String rmPrefix, String downPath) throws IOException {
5252
this.configuration = configuration;
5353
if (domain == null || "".equals(domain)) {
5454
if (urlIndex == null || "".equals(urlIndex)) {
5555
throw new IOException("please set one of domain and url-index.");
5656
} else {
5757
this.urlIndex = urlIndex;
5858
}
59+
this.domain = null;
5960
} else {
6061
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
6162
RequestUtils.lookUpFirstIpFromHost(domain);
@@ -79,8 +80,10 @@ private void set(Configuration configuration, String protocol, String domain, St
7980
this.preDown = true;
8081
} else {
8182
File file = new File(downPath);
82-
if (file.exists() && !file.isDirectory()) {
83-
throw new IOException("please change down-path because it's existed file.");
83+
if (file.exists()) {
84+
if (!file.isDirectory()) throw new IOException("please change down-path because it's existed file.");
85+
} else if (!file.mkdirs()) {
86+
if (!file.mkdirs()) throw new IOException(String.join("", "directory: ", downPath, " can not be created."));
8487
}
8588
}
8689
}
@@ -108,8 +111,7 @@ protected String singleResult(Map<String, String> line) throws Exception {
108111
url = url + suffixOrQuery;
109112
} else {
110113
if (key == null || "".equals(key)) throw new IOException("key is not exists or empty in " + line);
111-
url = String.join("", protocol, "://", domain, "/",
112-
key.replace("\\?", "%3f"), suffixOrQuery);
114+
url = String.join("", protocol, "://", domain, "/", key.replace("\\?", "%3f"), suffixOrQuery);
113115
key = String.join("", addPrefix, FileUtils.rmPrefix(rmPrefix, key)); // 目标文件名
114116
}
115117
if (preDown) {

‎src/main/java/com/qiniu/process/other/ExportTS.java

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ private void set(Configuration configuration, String protocol, String domain, St
4343
} else {
4444
this.urlIndex = urlIndex;
4545
}
46+
this.domain = null;
4647
} else {
4748
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
4849
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/other/HttpDownloader.java

+2-5
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,8 @@ public File download(String url, String filepath, StringMap headers) throws IOEx
3131
File file = new File(filepath);
3232
if (file.exists()) {
3333
throw new IOException(String.join("", "file: ", filepath, " is already exists."));
34-
} else {
35-
boolean exists = FileUtils.mkDirAndFile(file);
36-
while (!exists) {
37-
exists = FileUtils.mkDirAndFile(file);
38-
}
34+
} else if (!FileUtils.mkDirAndFile(file)) {
35+
throw new IOException(String.join("", "file: ", filepath, " can not be created."));
3936
}
4037
byte[] buffer = new byte[4096];
4138
int byteRead;

‎src/main/java/com/qiniu/process/other/PublicUrl.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ private void set(String protocol, String domain, String urlIndex, String suffixO
4141
this.urlIndex = urlIndex;
4242
if ("".equals(this.suffixOrQuery)) throw new IOException("please set suffix or query if url-index used.");
4343
}
44+
this.domain = null;
4445
} else {
4546
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
4647
RequestUtils.lookUpFirstIpFromHost(domain);
@@ -64,7 +65,7 @@ public PublicUrl clone() throws CloneNotSupportedException {
6465

6566
@Override
6667
protected String resultInfo(Map<String, String> line) {
67-
return line.get("key");
68+
return domain == null ? line.get(urlIndex) : line.get("key");
6869
}
6970

7071
@Override
@@ -76,8 +77,7 @@ protected String singleResult(Map<String, String> line) throws Exception {
7677
} else {
7778
String key = line.get("key");
7879
if (key == null) throw new IOException("key is not exists or empty in " + line);
79-
url = String.join("", protocol, "://", domain, "/",
80-
key.replace("\\?", "%3f"), suffixOrQuery);
80+
url = String.join("", protocol, "://", domain, "/", key.replace("\\?", "%3f"), suffixOrQuery);
8181
if (nextProcessor == null) return url;
8282
}
8383
line.put("url", url);

‎src/main/java/com/qiniu/process/qiniu/AsyncFetch.java

+1
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ private void set(Configuration configuration, String protocol, String domain, St
7373
} else {
7474
this.urlIndex = urlIndex;
7575
}
76+
this.domain = null;
7677
} else {
7778
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
7879
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/CdnUrlProcess.java

+1
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ public void set(Configuration configuration, String protocol, String domain, Str
8080
} else {
8181
this.urlIndex = urlIndex;
8282
}
83+
this.domain = null;
8384
} else {
8485
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
8586
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/CdnUrlQuery.java

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ public void set(Configuration configuration, String protocol, String domain, Str
8181
} else {
8282
this.urlIndex = urlIndex;
8383
}
84+
this.domain = null;
8485
} else {
8586
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
8687
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/CensorManager.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,7 @@ private void jsonClear() {
141141

142142
public String censorString(String jobId) throws QiniuException {
143143
String queryUrl = String.join("", videoJobsUrl, jobId);
144-
String token = String.join(" ", "Qiniu",
145-
auth.signRequestV2(queryUrl, "GET", null, null));
144+
String token = String.join(" ", "Qiniu", auth.signRequestV2(queryUrl, "GET", null, null));
146145
headers.put("Authorization", token);
147146
Response response = client.get(queryUrl, headers);
148147
String result = response.bodyString();

‎src/main/java/com/qiniu/process/qiniu/FetchFile.java

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ private void set(Configuration configuration, String protocol, String domain, St
5252
} else {
5353
this.urlIndex = urlIndex;
5454
}
55+
this.domain = null;
5556
} else {
5657
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
5758
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/ImageCensor.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ private void set(Configuration configuration, String protocol, String domain, St
5252
} else {
5353
this.urlIndex = urlIndex;
5454
}
55+
this.domain = null;
5556
} else {
5657
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
5758
RequestUtils.lookUpFirstIpFromHost(domain);
@@ -84,8 +85,7 @@ protected String singleResult(Map<String, String> line) throws Exception {
8485
} else {
8586
String key = line.get("key");
8687
if (key == null) throw new IOException("key is not exists or empty in " + line);
87-
url = String.join("", protocol, "://", domain, "/",
88-
key.replace("\\?", "%3f"), suffixOrQuery);
88+
url = String.join("", protocol, "://", domain, "/", key.replace("\\?", "%3f"), suffixOrQuery);
8989
return String.join("\t", key, censorManager.doImageCensor(url, paramsJson));
9090
}
9191
}

‎src/main/java/com/qiniu/process/qiniu/PrivateUrl.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ private void set(String protocol, String domain, String urlIndex, String suffixO
4545
} else {
4646
this.urlIndex = urlIndex;
4747
}
48+
this.domain = null;
4849
} else {
4950
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
5051
RequestUtils.lookUpFirstIpFromHost(domain);
@@ -79,14 +80,12 @@ protected String singleResult(Map<String, String> line) throws Exception {
7980
String url;
8081
if (domain == null) {
8182
url = line.get(urlIndex);
82-
if (nextProcessor == null) return String.join("\t", auth
83-
.privateDownloadUrl(url + suffixOrQuery, expires));
83+
if (nextProcessor == null) return auth.privateDownloadUrl(url + suffixOrQuery, expires);
8484
} else {
8585
String key = line.get("key");
8686
if (key == null) throw new IOException("key is not exists or empty in " + line);
87-
url = String.join("", protocol, "://", domain, "/",
88-
key.replace("\\?", "%3f"), suffixOrQuery);
89-
if (nextProcessor == null) return String.join("\t", auth.privateDownloadUrl(url, expires));
87+
url = String.join("", protocol, "://", domain, "/", key.replace("\\?", "%3f"), suffixOrQuery);
88+
if (nextProcessor == null) return auth.privateDownloadUrl(url, expires);
9089
}
9190
url = auth.privateDownloadUrl(url, expires);
9291
line.put("url", auth.privateDownloadUrl(url, expires));

‎src/main/java/com/qiniu/process/qiniu/QueryAvinfo.java

+1
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ private void set(Configuration configuration, String protocol, String domain, St
4141
} else {
4242
this.urlIndex = urlIndex;
4343
}
44+
this.domain = null;
4445
} else {
4546
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
4647
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/QueryHash.java

+1
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ private void set(Configuration configuration, String protocol, String domain, St
4444
} else {
4545
this.urlIndex = urlIndex;
4646
}
47+
this.domain = null;
4748
} else {
4849
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
4950
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/SyncUpload.java

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ private void set(Configuration configuration, String protocol, String domain, St
6868
} else {
6969
this.urlIndex = urlIndex;
7070
}
71+
this.domain = null;
7172
} else {
7273
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
7374
RequestUtils.lookUpFirstIpFromHost(domain);

‎src/main/java/com/qiniu/process/qiniu/VideoCensor.java

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ private void set(Configuration configuration, String protocol, String domain, St
5656
} else {
5757
this.urlIndex = urlIndex;
5858
}
59+
this.domain = null;
5960
} else {
6061
this.protocol = protocol == null || !protocol.matches("(http|https)") ? "http" : protocol;
6162
RequestUtils.lookUpFirstIpFromHost(domain);

0 commit comments

Comments
 (0)