Skip to content

Commit 3f4e964

Browse files
author
吴炳亨
authored
Merge pull request #305 from NigelWu95/dev
fix log path set for huawei/tencent client generating, and add upload…
2 parents 402c669 + dad0c6e commit 3f4e964

17 files changed

+85
-33
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
version=8.4.5
1+
version=8.4.6
22

33
package_no_test:
44
mvn clean package -Dmaven.test.skip=true

‎docs/datamigration.md

+3
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,9 @@ rs-domain=rs-z0.qiniu.com
198198
#rs-domain=rs-z1.qiniu.com
199199
api-domain=api.qiniu.com
200200
#api-domain=api-z1.qiniu.com
201+
#上传域名，accup 为加速域名，srcup 为源站域名
202+
accup-domain=upload.qiniu.com
203+
srcup-domain=upload.qiniu.com
201204
#是否使用 https 对上述设置的域名进行请求
202205
config-https=true/false
203206
```

‎docs/datasource.md

+3
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,9 @@ rs-domain=rs-z0.qiniu.com
347347
#rs-domain=rs-z1.qiniu.com
348348
api-domain=api.qiniu.com
349349
#api-domain=api-z1.qiniu.com
350+
#上传域名，accup 为加速域名，srcup 为源站域名
351+
accup-domain=upload.qiniu.com
352+
srcup-domain=upload.qiniu.com
350353
#是否使用 https 对上述设置的域名进行请求
351354
config-https=true/false
352355
```

‎docs/uploadfile.md

+9
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,15 @@ directory-config 中的目录名可以采取相对 path 路径下的目录名。
7777
|key|上述配置文件中的 "/Users/wubingheng/Projects/Github/test" 等表示目录名或路径，不可重复，重复情况下后者会覆盖前者|
7878
|value| 表示数据源中某一行的内容，如 "qiniu_success_1.txt" 表示 "/Users/wubingheng/Projects/Github/test" 目录中可能存在该文件名|
7979

80+
### 上传域名
81+
```
82+
#上传域名，accup 为加速域名，srcup 为源站域名
83+
accup-domain=upload.qiniu.com
84+
srcup-domain=upload.qiniu.com
85+
#是否使用 https 对上述设置的域名进行请求
86+
config-https=true/false
87+
```
88+
8089
### 上传超时时间
8190
timeout 参数可以通过全局的 timeout 来设置，参考：[网络设置](../README.md#7-网络设置)
8291

‎pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.qiniu</groupId>
88
<artifactId>qsuits</artifactId>
9-
<version>8.4.5</version>
9+
<version>8.4.6</version>
1010
<name>qsuits</name>
1111
<description>qiniu-suits is a efficient tools for qiniu api implemented by java8.</description>
1212
<url>https://github.com/NigelWu95/qiniu-suits-java</url>

‎src/main/java/com/qiniu/datasource/QiniuLister.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -220,8 +220,8 @@ public List<FileInfo> currents() {
220220

221221
@Override
222222
public synchronized String currentEndKey() {
223-
if (hasNext()) return CloudApiUtils.decodeQiniuMarker(marker);
224-
if (truncateMarker != null && !"".equals(truncateMarker)) return CloudApiUtils.decodeQiniuMarker(truncateMarker);
223+
if (hasNext()) return CloudApiUtils.decodeQiniuQosMarker(marker);
224+
if (truncateMarker != null && !"".equals(truncateMarker)) return CloudApiUtils.decodeQiniuQosMarker(truncateMarker);
225225
if (last != null) return last.key;
226226
if (fileInfoList.size() > 0) last = fileInfoList.get(fileInfoList.size() - 1);
227227
if (last != null) return last.key;

‎src/main/java/com/qiniu/datasource/UpLister.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -215,9 +215,9 @@ public List<String> getDirectories() {
215215

216216
@Override
217217
public synchronized String currentEndKey() {
218-
if (hasNext()) return CloudApiUtils.decodeUpYunMarker(marker);
218+
if (hasNext()) return CloudApiUtils.decodeUpYunYosMarker(marker);
219219
if (truncateMarker != null && !"".equals(truncateMarker) && !"g2gCZAAEbmV4dGQAA2VvZg".equals(marker)) {
220-
return CloudApiUtils.decodeUpYunMarker(truncateMarker);
220+
return CloudApiUtils.decodeUpYunYosMarker(truncateMarker);
221221
}
222222
if (last != null) return last.key;
223223
if (fileItems.size() > 0) last = fileItems.get(fileItems.size() - 1);

‎src/main/java/com/qiniu/datasource/UpYosContainer.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ protected IResultOutput getNewResultSaver(String order) throws IOException {
9090

9191
@Override
9292
protected IStorageLister<FileItem> getLister(String prefix, String marker, String start, String end, int unitLen) throws SuitsException {
93-
if (marker == null || "".equals(marker)) marker = CloudApiUtils.getUpYunMarker(username, password, bucket, start);
93+
if (marker == null || "".equals(marker)) marker = CloudApiUtils.getUpYunYosMarker(username, password, bucket, start);
9494
return new UpLister(new UpYunClient(configuration, username, password), bucket, prefix, marker, end, unitLen);
9595
}
9696

‎src/main/java/com/qiniu/entry/QSuitsEntry.java

+12-4
Original file line numberDiff line numberDiff line change
@@ -140,19 +140,23 @@ private Configuration getDefaultQiniuConfig() throws IOException {
140140
}
141141

142142
private Configuration getDefaultQiniuConfig(String ak, String sk, String regionName, String bucket) throws IOException {
143-
com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
143+
com.qiniu.storage.Region region = CloudApiUtils.getQiniuQosRegion(regionName);
144144
String rsfDomain = entryParam.getValue("rsf-domain", null);
145145
String rsDomain = entryParam.getValue("rs-domain", null);
146146
String apiDomain = entryParam.getValue("api-domain", null);
147+
String accUpDomain = entryParam.getValue("accup-domain", null);
148+
String srcUpDomain = entryParam.getValue("srcup-domain", null);
147149
if (rsfDomain != null || rsDomain != null || apiDomain != null) {
148150
com.qiniu.storage.Region.Builder builder = new com.qiniu.storage.Region.Builder();
149151
if (rsfDomain != null) region = builder.rsfHost(rsfDomain).build();
150152
if (rsDomain != null) region = builder.rsHost(rsDomain).build();
151153
if (apiDomain != null) region = builder.apiHost(apiDomain).build();
154+
if (apiDomain != null) region = builder.accUpHost(accUpDomain).build();
155+
if (apiDomain != null) region = builder.srcUpHost(srcUpDomain).build();
152156
} else {
153157
region = (regionName == null || "".equals(regionName)) ?
154-
CloudApiUtils.getQiniuRegion(CloudApiUtils.getQiniuRegion(ak, sk, bucket))
155-
: CloudApiUtils.getQiniuRegion(regionName);
158+
CloudApiUtils.getQiniuQosRegion(CloudApiUtils.getQiniuQosRegion(ak, sk, bucket))
159+
: CloudApiUtils.getQiniuQosRegion(regionName);
156160
}
157161
Configuration configuration = new Configuration(region);
158162
if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
@@ -163,15 +167,19 @@ private Configuration getDefaultQiniuConfig(String ak, String sk, String regionN
163167
}
164168

165169
private Configuration getNewQiniuConfig() throws IOException {
166-
com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
170+
com.qiniu.storage.Region region = CloudApiUtils.getQiniuQosRegion(regionName);
167171
String rsfDomain = entryParam.getValue("rsf-domain", null);
168172
String rsDomain = entryParam.getValue("rs-domain", null);
169173
String apiDomain = entryParam.getValue("api-domain", null);
174+
String accUpDomain = entryParam.getValue("accup-domain", null);
175+
String srcUpDomain = entryParam.getValue("srcup-domain", null);
170176
if (rsfDomain != null || rsDomain != null || apiDomain != null) {
171177
com.qiniu.storage.Region.Builder builder = new com.qiniu.storage.Region.Builder();
172178
if (rsfDomain != null) region = builder.rsfHost(rsfDomain).build();
173179
if (rsDomain != null) region = builder.rsHost(rsDomain).build();
174180
if (apiDomain != null) region = builder.apiHost(apiDomain).build();
181+
if (apiDomain != null) region = builder.accUpHost(accUpDomain).build();
182+
if (apiDomain != null) region = builder.srcUpHost(srcUpDomain).build();
175183
}
176184
Configuration configuration = new Configuration(region);
177185
if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;

‎src/main/java/com/qiniu/process/huawei/PrivateUrl.java

+3
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.qiniu.interfaces.ILineProcess;
77
import com.qiniu.process.Base;
88
import com.qiniu.util.CloudApiUtils;
9+
import com.qiniu.util.LogUtils;
910

1011
import java.io.IOException;
1112
import java.util.HashMap;
@@ -38,6 +39,7 @@ public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, Str
3839
configuration.setEndPoint(endpoint);
3940
obsClient = new ObsClient(accessId, secretKey, configuration);
4041
CloudApiUtils.checkHuaWei(obsClient);
42+
LogUtils.getLogPath(LogUtils.QSUITS);
4143
}
4244

4345
public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, String endpoint, long expires,
@@ -58,6 +60,7 @@ public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, Str
5860
configuration.setEndPoint(endpoint);
5961
obsClient = new ObsClient(accessId, secretKey, configuration);
6062
CloudApiUtils.checkHuaWei(obsClient);
63+
LogUtils.getLogPath(LogUtils.QSUITS);
6164
}
6265

6366
public PrivateUrl(String accessKeyId, String accessKeySecret, String bucket, String endpoint, long expires,

‎src/main/java/com/qiniu/process/tencent/PrivateUrl.java

+3
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.qiniu.interfaces.ILineProcess;
1111
import com.qiniu.process.Base;
1212
import com.qiniu.util.CloudApiUtils;
13+
import com.qiniu.util.LogUtils;
1314

1415
import java.io.IOException;
1516
import java.net.URL;
@@ -43,6 +44,7 @@ public PrivateUrl(String secretId, String secretKey, String bucket, String regio
4344
else clientConfig.setHttpProtocol(HttpProtocol.http);
4445
cosClient = new COSClient(credentials, clientConfig);
4546
CloudApiUtils.checkTencent(cosClient);
47+
LogUtils.getLogPath(LogUtils.QSUITS);
4648
}
4749

4850
public PrivateUrl(String secretId, String secretKey, String bucket, String region, boolean useHttps, long expires,
@@ -62,6 +64,7 @@ public PrivateUrl(String secretId, String secretKey, String bucket, String regio
6264
else clientConfig.setHttpProtocol(HttpProtocol.http);
6365
cosClient = new COSClient(credentials, clientConfig);
6466
CloudApiUtils.checkTencent(cosClient);
67+
LogUtils.getLogPath(LogUtils.QSUITS);
6568
}
6669

6770
public PrivateUrl(String secretId, String secretKey, String bucket, String endpoint, boolean useHttps, long expires,

‎src/main/java/com/qiniu/sdk/UpYunClient.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public FileItem getFileInfo(String bucket, String key) throws IOException {
5555
conn.setUseCaches(false);
5656
String date = DatetimeUtils.getGMTDate();
5757
conn.setRequestProperty(UpYunConfig.DATE, date);
58-
conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunSign(UpYunConfig.METHOD_HEAD, date, uri,
58+
conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunYosSign(UpYunConfig.METHOD_HEAD, date, uri,
5959
userName, password, null));
6060
conn.connect();
6161
int code = conn.getResponseCode();
@@ -92,7 +92,7 @@ private String HttpGetAction(String uri, Map<String, String> headers) throws IOE
9292
conn.setUseCaches(false);
9393
String date = DatetimeUtils.getGMTDate();
9494
conn.setRequestProperty(UpYunConfig.DATE, date);
95-
conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunSign(UpYunConfig.METHOD_GET, date, uri,
95+
conn.setRequestProperty(UpYunConfig.AUTHORIZATION, CloudApiUtils.upYunYosSign(UpYunConfig.METHOD_GET, date, uri,
9696
userName, password, null));
9797
if (headers != null) {
9898
for (Map.Entry<String, String> entry : headers.entrySet()) {

‎src/main/java/com/qiniu/util/CloudApiUtils.java

+11-13
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,7 @@ public static int AliStatusCode(String error, int Default) {
129129
return aliStatus.getOrDefault(error, Default);
130130
}
131131

132-
public static int NetStatusCode(String error, int Default) {
133-
return aliStatus.getOrDefault(error, Default);
134-
}
135-
136-
public static String getQiniuMarker(FileInfo fileInfo) {
132+
public static String getQiniuQosMarker(FileInfo fileInfo) {
137133
return getQiniuMarker(fileInfo.key);
138134
}
139135

@@ -153,7 +149,7 @@ public static String getbaiduBosCosMarker(BosObjectSummary obsObject) {
153149
return obsObject.getKey();
154150
}
155151

156-
public static String getUpYunMarker(String bucket, FileItem fileItem) {
152+
public static String getUpYunYosMarker(String bucket, FileItem fileItem) {
157153
if (fileItem.key.contains("/")) {
158154
String convertedKey = fileItem.key.replace("/", "/~");
159155
int lastIndex = convertedKey.lastIndexOf("~");
@@ -186,19 +182,19 @@ public static String getTenCosMarker(String key) {
186182
return key;
187183
}
188184

189-
public static String getUpYunMarker(String username, String password, String bucket, String key) throws SuitsException {
185+
public static String getUpYunYosMarker(String username, String password, String bucket, String key) throws SuitsException {
190186
if (key == null || "".equals(key)) return null;
191187
UpYunClient upYunClient = new UpYunClient(new UpYunConfig(), username, password);
192188
try {
193-
return getUpYunMarker(bucket, upYunClient.getFileInfo(bucket, key));
189+
return getUpYunYosMarker(bucket, upYunClient.getFileInfo(bucket, key));
194190
} catch (Exception e) {
195191
throw new SuitsException(e, 848);
196192
} finally {
197193
upYunClient = null;
198194
}
199195
}
200196

201-
public static String decodeQiniuMarker(String marker) {
197+
public static String decodeQiniuQosMarker(String marker) {
202198
String decodedMarker = new String(Base64.decode(marker, Base64.URL_SAFE | Base64.NO_WRAP));
203199
JsonObject jsonObject = new JsonParser().parse(decodedMarker).getAsJsonObject();
204200
return jsonObject.get("k").getAsString();
@@ -216,7 +212,7 @@ public static String decodeHuaweiObsMarker(String marker) {
216212
return marker;
217213
}
218214

219-
public static String decodeUpYunMarker(String marker) {
215+
public static String decodeUpYunYosMarker(String marker) {
220216
String keyString = new String(decoder.decode(marker));
221217
int index = keyString.contains("/~") ? keyString.indexOf("/~") + 2 : keyString.indexOf("/") + 1;
222218
return keyString.substring(index).replace("(/~|/@~|/@#)", "/");
@@ -278,7 +274,7 @@ public static void checkBaidu(BosClient bosClient) {
278274
bosClient.listBuckets();
279275
}
280276

281-
public static Region getQiniuRegion(String regionName) throws IOException {
277+
public static Region getQiniuQosRegion(String regionName) throws IOException {
282278
if (regionName == null || "".equals(regionName)) return Region.autoRegion();
283279
switch (regionName) {
284280
case "z0":
@@ -305,7 +301,7 @@ public static Region getQiniuRegion(String regionName) throws IOException {
305301
}
306302
}
307303

308-
public static String getQiniuRegion(String accessKey, String secretKey, String bucket) throws SuitsException {
304+
public static String getQiniuQosRegion(String accessKey, String secretKey, String bucket) throws SuitsException {
309305
try {
310306
Auth auth = Auth.create(accessKey, secretKey);
311307
Configuration configuration = new Configuration();
@@ -347,6 +343,7 @@ public static String getAliOssRegion(String accessKeyId, String accessKeySecret,
347343
}
348344

349345
public static String getTenCosRegion(String secretId, String secretKey, String bucket) throws SuitsException {
346+
LogUtils.getLogPath(LogUtils.QSUITS);
350347
COSCredentials cred = new BasicCOSCredentials(secretId, secretKey);
351348
ClientConfig clientConfig = new ClientConfig();
352349
COSClient cosClient = new COSClient(cred, clientConfig);
@@ -396,6 +393,7 @@ public static String getS3Region(String s3AccessKeyId, String s3SecretKey, Strin
396393
}
397394

398395
public static String getHuaweiObsRegion(String accessKeyId, String secretKey, String bucket) throws SuitsException {
396+
LogUtils.getLogPath(LogUtils.QSUITS);
399397
ObsClient obsClient = new ObsClient(accessKeyId, secretKey, "https://obs.myhuaweicloud.com");
400398
try {
401399
return obsClient.getBucketLocation(bucket);
@@ -419,7 +417,7 @@ public static String getBaiduBosRegion(String accessKeyId, String secretKey, Str
419417

420418
private static final String lineSeparator = System.getProperty("line.separator");
421419

422-
public static String upYunSign(String method, String date, String path, String userName, String password, String md5)
420+
public static String upYunYosSign(String method, String date, String path, String userName, String password, String md5)
423421
throws IOException {
424422
StringBuilder sb = new StringBuilder();
425423
String sp = "&";

‎src/main/java/com/qiniu/util/LogUtils.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public static String getLogPath(String logName) {
3030
} while (logFile.exists());
3131
s = String.valueOf(serial);
3232
}
33-
System.setProperty(QSUITS, String.join(FileUtils.pathSeparator, LOG_PATH, QSUITS + s));
33+
System.setProperty(logName, String.join(FileUtils.pathSeparator, LOG_PATH, logName + s));
3434
System.setProperty(PROCEDURE, String.join(FileUtils.pathSeparator, LOG_PATH, PROCEDURE + s));
3535
return logPath.toString();
3636
}

‎src/test/java/com/qiniu/util/CloudApiUtilsTest.java

+26-4
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,19 @@
1111

1212
public class CloudApiUtilsTest {
1313

14+
@Test
15+
public void testGetQiniuRegion() throws IOException {
16+
PropertiesFile propertiesFile = new PropertiesFile("resources/.application.properties");
17+
String secretId = propertiesFile.getValue("ak");
18+
String secretKey = propertiesFile.getValue("sk");
19+
String bucket = propertiesFile.getValue("bucket");
20+
String region = CloudApiUtils.getQiniuQosRegion(secretId, secretKey, bucket);
21+
System.out.println(bucket + "\t" + region);
22+
}
23+
1424
@Test
1525
public void testGetTenCosRegion() throws IOException {
26+
LogUtils.getLogPath(LogUtils.QSUITS);
1627
PropertiesFile propertiesFile = new PropertiesFile("resources/.tencent.properties");
1728
String secretId = propertiesFile.getValue("ten-id");
1829
String secretKey = propertiesFile.getValue("ten-secret");
@@ -21,6 +32,17 @@ public void testGetTenCosRegion() throws IOException {
2132
System.out.println(bucket + "\t" + region);
2233
}
2334

35+
@Test
36+
public void testGetHuaweiObsRegion() throws IOException {
37+
LogUtils.getLogPath(LogUtils.QSUITS);
38+
PropertiesFile propertiesFile = new PropertiesFile("resources/.huawei.properties");
39+
String secretId = propertiesFile.getValue("hua-id");
40+
String secretKey = propertiesFile.getValue("hua-secret");
41+
String bucket = propertiesFile.getValue("bucket");
42+
String region = CloudApiUtils.getHuaweiObsRegion(secretId, secretKey, bucket);
43+
System.out.println(bucket + "\t" + region);
44+
}
45+
2446
@Test
2547
public void testGetS3Region() throws IOException {
2648
PropertiesFile propertiesFile = new PropertiesFile("resources/.s3.properties");
@@ -60,19 +82,19 @@ public void testGetUpYunMarker() {
6082
FileItem fileItem1 = new FileItem();
6183
fileItem1.key = name1;
6284
fileItem1.attribute = "folder";
63-
System.out.println(CloudApiUtils.getUpYunMarker(bucket, fileItem1));
85+
System.out.println(CloudApiUtils.getUpYunYosMarker(bucket, fileItem1));
6486
String name2 = "wordSplit/xml/20161220/FF8080815919A15101591AFE37C603F7/4028965B591534B501591BBEC0E8049A.txt";
6587
FileItem fileItem2 = new FileItem();
6688
fileItem2.key = name2;
67-
System.out.println(CloudApiUtils.getUpYunMarker(bucket, fileItem2));
89+
System.out.println(CloudApiUtils.getUpYunYosMarker(bucket, fileItem2));
6890
}
6991

7092
@Test
7193
public void testDecodeUpYunMarker() {
7294
String marker1 = "c3F1aXJyZWwvfndvcmRTcGxpdC9+eG1sL34yMDE2MTIyMC9AfkZGODA4MDgxNTkxOUExNTEwMTU5MTlEN0RDOEYwMDM2";
7395
String marker2 = "c3F1aXJyZWwvfndvcmRTcGxpdC9+eG1sL34yMDE2MTIyMC9+RkY4MDgwODE1OTE5QTE1MTAxNTkxQUZFMzdDNjAzRjcvQCM0MDI4OTY1QjU5MTUzNEI1MDE1OTFCQkVDMEU4MDQ5QS50eHQ=";
74-
System.out.println(CloudApiUtils.decodeUpYunMarker(marker1));
75-
System.out.println(CloudApiUtils.decodeUpYunMarker(marker2));
96+
System.out.println(CloudApiUtils.decodeUpYunYosMarker(marker1));
97+
System.out.println(CloudApiUtils.decodeUpYunYosMarker(marker2));
7698
}
7799

78100
@Test

‎src/test/java/com/qiniu/util/LogUtilsTest.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,10 @@ public class LogUtilsTest {
99
@Test
1010
public void testGetLogPath() {
1111
System.out.println(LogUtils.getLogPath(LogUtils.QSUITS));
12-
System.out.println(LogUtils.getLogPath(LogUtils.ERROR));
12+
System.out.println(System.getProperty("qsuits"));
13+
System.out.println(System.getProperty("procedure"));
1314
System.out.println(LogUtils.getLogPath(LogUtils.PROCEDURE));
15+
System.out.println(System.getProperty("qsuits"));
16+
System.out.println(System.getProperty("procedure"));
1417
}
1518
}

‎version.properties

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version=8.4.5
1+
version=8.4.6

0 commit comments

Comments
 (0)