Merge pull request #287 from NigelWu95/dev

optimize region get and add converting indexMap check for DefaultFile…

Author: 吴炳亨

docs/downloadfile.md

@@ -42,6 +42,9 @@ private=
 queries 参数用于设置 url 的后缀或 ?+参数部分，资源下载可能需要下载不同格式或尺寸的如图片文件，因此可以通过一些图片处理样式或参数来设置对处理之后的
 图片进行下载。当设置 private（私有签名）的情况下，该参数会使用在 privateurl 操作中（因为 privateurl 操作在前，当前操作在后）。  
 
+### 下载超时时间
+timeout 参数可以通过全局的 timeout 来设置，参考：[网络设置](../README.md#7-网络设置)  
+
 ### 关于 url-index
 当使用 file 源且 parse=tab/csv 时 [xx-]index(ex) 设置的下标必须为整数。url-index 表示输入行含 url 形式的源文件地址，未设置的情况下则使用 
 key 字段加上 domain 的方式访问源文件地址，key 下标用 indexes 参数设置。  

src/main/java/com/qiniu/datasource/DefaultFileContainer.java

@@ -16,6 +16,9 @@ public DefaultFileContainer(String path, Map<String, Map<String, String>> direct
                                 List<String> antiDirectories, boolean keepDir, Map<String, String> indexMap,
                                 List<String> fields, int unitLen, int threads) throws IOException {
         super(path, directoriesMap, antiDirectories, keepDir, indexMap, fields, unitLen, threads);
+        FileInfo test = new FileInfo();
+        test.filepath = "test";
+        ConvertingUtils.toPair(test, indexMap, new StringMapPair());
     }
 
     @Override

src/main/java/com/qiniu/entry/QSuitsEntry.java

@@ -136,11 +136,11 @@ private Configuration getDefaultQiniuConfig() throws IOException {
             qiniuAccessKey = entryParam.getValue("ak").trim();
             qiniuSecretKey = entryParam.getValue("sk").trim();
         }
-        return getDefaultQiniuConfig(qiniuAccessKey, qiniuSecretKey, regionName);
+        return getDefaultQiniuConfig(qiniuAccessKey, qiniuSecretKey, regionName, bucket);
     }
 
-    private Configuration getDefaultQiniuConfig(String ak, String sk, String regionName) throws IOException {
-        com.qiniu.storage.Region region = null;
+    private Configuration getDefaultQiniuConfig(String ak, String sk, String regionName, String bucket) throws IOException {
+        com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
         String rsfDomain = entryParam.getValue("rsf-domain", null);
         String rsDomain = entryParam.getValue("rs-domain", null);
         String apiDomain = entryParam.getValue("api-domain", null);
@@ -149,10 +149,11 @@ private Configuration getDefaultQiniuConfig(String ak, String sk, String regionN
             if (rsfDomain != null) region = builder.rsfHost(rsfDomain).build();
             if (rsDomain != null) region = builder.rsHost(rsDomain).build();
             if (apiDomain != null) region = builder.apiHost(apiDomain).build();
+        } else {
+            region = (regionName == null || "".equals(regionName)) ?
+                    CloudApiUtils.getQiniuRegion(CloudApiUtils.getQiniuRegion(ak, sk, bucket))
+                    : CloudApiUtils.getQiniuRegion(regionName);
         }
-        if (region == null) region = (regionName == null || "".equals(regionName)) ?
-                CloudApiUtils.getQiniuRegion(CloudApiUtils.getQiniuRegion(ak, sk, bucket))
-                : CloudApiUtils.getQiniuRegion(regionName);
         Configuration configuration = new Configuration(region);
         if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
         if (readTimeout> Constants.READ_TIMEOUT) configuration.readTimeout = readTimeout;
@@ -161,8 +162,8 @@ private Configuration getDefaultQiniuConfig(String ak, String sk, String regionN
         return configuration;
     }
 
-    private Configuration getNewQiniuConfig() {
-        com.qiniu.storage.Region region = null;
+    private Configuration getNewQiniuConfig() throws IOException {
+        com.qiniu.storage.Region region = CloudApiUtils.getQiniuRegion(regionName);
         String rsfDomain = entryParam.getValue("rsf-domain", null);
         String rsDomain = entryParam.getValue("rs-domain", null);
         String apiDomain = entryParam.getValue("api-domain", null);
@@ -172,7 +173,7 @@ private Configuration getNewQiniuConfig() {
             if (rsDomain != null) region = builder.rsHost(rsDomain).build();
             if (apiDomain != null) region = builder.apiHost(apiDomain).build();
         }
-        Configuration configuration = region == null ? new Configuration() : new Configuration(region);
+        Configuration configuration = new Configuration(region);
         if (connectTimeout > Constants.CONNECT_TIMEOUT) configuration.connectTimeout = connectTimeout;
         if (readTimeout> Constants.READ_TIMEOUT) configuration.readTimeout = readTimeout;
         if (requestTimeout > Constants.WRITE_TIMEOUT) configuration.writeTimeout = requestTimeout;
@@ -660,7 +661,7 @@ private ILineProcess<Map<String, String>> getAsyncFetch(Map<String, String> inde
         String ignore = entryParam.getValue("ignore-same-key", "false").trim();
         ParamsUtils.checked(ignore, "ignore-same-key", "(true|false)");
         String regionStr = entryParam.getValue("qiniu-region", regionName).trim();
-        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr);
+        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr, toBucket);
         AsyncFetch processor = single ? new AsyncFetch(ak, sk, configuration, toBucket, protocol, domain, urlIndex,
                 addPrefix, rmPrefix) : new AsyncFetch(ak, sk, configuration, toBucket, protocol, domain, urlIndex,
                 addPrefix, rmPrefix, savePath);
@@ -758,7 +759,7 @@ private ILineProcess<Map<String, String>> getMirrorFile(boolean single) throws I
         if (toBucket.equals(bucket) && "qiniu".equals(source))
             throw new IOException("the to-bucket can not be same as bucket if source is qiniu.");
         String regionStr = entryParam.getValue("qiniu-region", regionName).trim();
-        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr);
+        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr, toBucket);
         return single ? new MirrorFile(ak, sk, configuration, toBucket) : new MirrorFile(ak, sk, configuration, toBucket, savePath);
     }
 
@@ -1096,7 +1097,7 @@ private ILineProcess<Map<String, String>> getFetch(Map<String, String> indexMap,
         String addPrefix = entryParam.getValue("add-prefix", null);
         String rmPrefix = entryParam.getValue("rm-prefix", null);
         String regionStr = entryParam.getValue("qiniu-region", regionName).trim();
-        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr);
+        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr, toBucket);
         return single ? new FetchFile(ak, sk, configuration, toBucket, protocol, domain, urlIndex, addPrefix, rmPrefix)
                 : new FetchFile(ak, sk, configuration, toBucket, protocol, domain, urlIndex, addPrefix, rmPrefix, savePath);
     }
@@ -1131,7 +1132,7 @@ private ILineProcess<Map<String, String>> getSyncUpload(Map<String, String> inde
             }
         }
         String regionStr = entryParam.getValue("qiniu-region", regionName).trim();
-        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr);
+        Configuration configuration = getDefaultQiniuConfig(ak, sk, regionStr, toBucket);
         return single ? new SyncUpload(ak, sk, configuration, protocol, domain, urlIndex, host, addPrefix, rmPrefix,
                 toBucket, expires, policy, params) : new SyncUpload(ak, sk, configuration, protocol, domain, urlIndex,
                 host, addPrefix, rmPrefix, toBucket, expires, policy, params, savePath);

src/main/java/com/qiniu/util/CloudApiUtils.java

@@ -277,7 +277,7 @@ public static void checkBaidu(BosClient bosClient) {
     }
 
     public static Region getQiniuRegion(String regionName) throws IOException {
-        if (regionName == null) return Region.autoRegion();
+        if (regionName == null || "".equals(regionName)) return Region.autoRegion();
         switch (regionName) {
             case "z0":
             case "huadong":