update RestoreArchive process to use new api.

NigelWu95 · NigelWu95 · commit df5bce2b2d57 · 2020-07-13T19:11:16.000+08:00
diff --git a/docs/restorear.md b/docs/restorear.md
@@ -17,7 +17,6 @@ ak=
 sk=
 bucket=
 days=
-cond.[]=
 ```  
 |参数名|参数值及类型 | 含义|  
 |-----|-------|-----|  
@@ -26,7 +25,6 @@ cond.[]=
 |bucket| 字符串| 操作的资源原空间，当数据源为 qiniu 时无需再设置|  
 |indexes|字符串| 设置输入行中 key 字段的下标（有默认值），参考[数据源 indexes 设置](datasource.md#1-公共参数)|  
 |days| 整型数字| 设置解冻的有效期为 days（单位天数），范围 1-7 天|  
-|cond.[]| 字符串| 可以设置一些操作时的 condition，cond 当前支持设置 hash、mime、fsize、putTime 条件，只有条件匹配才会执行修改操作，如 cond.mime=text/plain|  
 
 运行参数：`-config=config.txt`
 
diff --git a/nigel/type_success_1.txt b/nigel/type_success_1.txt
@@ -0,0 +1,2 @@
+	{"code":200}
+7026814.ts	{"code":200}
diff --git a/nigel/type_success_2.txt b/nigel/type_success_2.txt
@@ -0,0 +1 @@
+7026814_转码后.ts	{"code":200}
diff --git a/nigel/type_success_3.txt b/nigel/type_success_3.txt
@@ -0,0 +1 @@
+UAA-4hndfVc5V6DJX0EvslAUBBI=/llmf1NC-2WRv3wJdHjFs0_szQ_vU	{"code":200}
diff --git a/nigel/type_success_4.txt b/nigel/type_success_4.txt
@@ -0,0 +1 @@
+upload_c1b9d660_e284_48c7_b180_2cf4d2debf13_00000135.tmp	{"code":200}
diff --git a/nigel/type_success_5.txt b/nigel/type_success_5.txt
@@ -0,0 +1 @@
+视频数据.mp4	{"code":200}
diff --git a/src/main/java/com/qiniu/entry/QSuitsEntry.java b/src/main/java/com/qiniu/entry/QSuitsEntry.java
@@ -607,19 +607,18 @@ private ILineProcess<Map<String, String>> getChangeType(boolean single) throws I
     private ILineProcess<Map<String, String>> getRestoreArchive(boolean single) throws IOException {
         String days = entryParam.getValue("days", "1").trim();
         ParamsUtils.checked(days, "days", "[1-7]");
-        StringBuilder condition = new StringBuilder();
-        for (Map.Entry<String, String> entry : entryParam.getParamsMap().entrySet()) {
-            if (entry.getKey().startsWith("cond.")) {
-                if (condition.length() > 0) {
-                    condition.append(entry.getKey().substring(5)).append("=").append(entry.getValue().trim()).append("&");
-                } else {
-                    condition.append(entry.getKey().substring(5)).append("=").append(entry.getValue().trim());
-                }
-            }
-        }
-        return single ? new RestoreArchive(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.parseInt(days),
-                condition.toString()) : new RestoreArchive(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket,
-                Integer.parseInt(days), condition.toString(), savePath);
+//        StringBuilder condition = new StringBuilder();
+//        for (Map.Entry<String, String> entry : entryParam.getParamsMap().entrySet()) {
+//            if (entry.getKey().startsWith("cond.")) {
+//                if (condition.length() > 0) {
+//                    condition.append(entry.getKey().substring(5)).append("=").append(entry.getValue().trim()).append("&");
+//                } else {
+//                    condition.append(entry.getKey().substring(5)).append("=").append(entry.getValue().trim());
+//                }
+//            }
+//        }
+        return single ? new RestoreArchive(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.parseInt(days))
+                : new RestoreArchive(qiniuAccessKey, qiniuSecretKey, getQiniuConfig(), bucket, Integer.parseInt(days), savePath);
     }
 
     private ILineProcess<Map<String, String>> getChangeLifecycle(boolean single) throws IOException {
diff --git a/src/main/java/com/qiniu/process/qiniu/RestoreArchive.java b/src/main/java/com/qiniu/process/qiniu/RestoreArchive.java
@@ -1,7 +1,5 @@
 package com.qiniu.process.qiniu;
 
-import com.google.gson.JsonArray;
-import com.google.gson.JsonObject;
 import com.qiniu.common.QiniuException;
 import com.qiniu.http.Client;
 import com.qiniu.http.Response;
@@ -10,51 +8,58 @@
 import com.qiniu.util.*;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 public class RestoreArchive extends Base<Map<String, String>> {
 
     private int days;
-    private String encodedCondition;
+    private ArrayList<String> ops;
+    private List<Map<String, String>> lines;
     private Configuration configuration;
     private Auth auth;
     private Client client;
     private static String requestUrl;
 
-    public RestoreArchive(String accessKey, String secretKey, Configuration configuration, String bucket, int days,
-                          String condition) throws IOException {
+    public RestoreArchive(String accessKey, String secretKey, Configuration configuration, String bucket, int days)
+            throws IOException {
         super("restorear", accessKey, secretKey, bucket);
         this.days = days;
-        if (condition != null && !condition.isEmpty()) encodedCondition = UrlSafeBase64.encodeToString(condition);
         this.configuration = configuration;
         this.auth = Auth.create(accessKey, secretKey);
         this.client = new Client(configuration.clone());
         CloudApiUtils.checkQiniu(accessKey, secretKey, configuration, bucket);
-        requestUrl = configuration.useHttpsDomains ? "https://rs.qbox.me/restoreAr" : "http://rs.qbox.me/restoreAr";
+        requestUrl = configuration.useHttpsDomains ? "https://rs.qbox.me/restoreAr/" : "http://rs.qbox.me/restoreAr/";
     }
 
     public RestoreArchive(String accessKey, String secretKey, Configuration configuration, String bucket, int days,
-                          String condition, String savePath, int saveIndex) throws IOException {
+                          String savePath, int saveIndex) throws IOException {
         super("restorear", accessKey, secretKey, bucket, savePath, saveIndex);
         this.days = days;
-        if (condition != null && !condition.isEmpty()) encodedCondition = UrlSafeBase64.encodeToString(condition);
+        this.batchSize = 1000;
+        this.ops = new ArrayList<>(1000);
+        this.lines = new ArrayList<>(1000);
         this.configuration = configuration;
         this.auth = Auth.create(accessKey, secretKey);
         this.client = new Client(configuration.clone());
         CloudApiUtils.checkQiniu(accessKey, secretKey, configuration, bucket);
-        requestUrl = configuration.useHttpsDomains ? "https://rs.qbox.me/restoreAr" : "http://rs.qbox.me/restoreAr";
+        requestUrl = configuration.useHttpsDomains ? "https://rs.qbox.me/restoreAr/" : "http://rs.qbox.me/restoreAr/";
     }
 
     public RestoreArchive(String accessKey, String secretKey, Configuration configuration, String bucket, int days,
-                          String condition, String savePath) throws IOException {
-        this(accessKey, secretKey, configuration, bucket, days, condition, savePath, 0);
+                          String savePath) throws IOException {
+        this(accessKey, secretKey, configuration, bucket, days, savePath, 0);
     }
 
     @Override
     public RestoreArchive clone() throws CloneNotSupportedException {
         RestoreArchive restoreArchive = (RestoreArchive)super.clone();
+        if (fileSaveMapper != null) {
+            restoreArchive.ops = new ArrayList<>(batchSize);
+            restoreArchive.lines = new ArrayList<>(batchSize);
+        }
         restoreArchive.auth = Auth.create(accessId, secretKey);
         restoreArchive.client = new Client(configuration.clone());
         return restoreArchive;
@@ -66,29 +71,38 @@ protected String resultInfo(Map<String, String> line) {
     }
 
     @Override
-    protected String batchResult(List<Map<String, String>> lineList) throws IOException {
-        JsonObject bodyJson = new JsonObject();
-        JsonArray entries = new JsonArray();
-        JsonObject entry = new JsonObject();
-        entry.addProperty("bucket", bucket);
-        for (Map<String, String> line : lineList) {
-            entry.addProperty("key", line.get("key"));
-            entry.addProperty("freeze_after_days", days);
-            entry.addProperty("cond", encodedCondition);
-            entries.add(entry);
+    protected List<Map<String, String>> putBatchOperations(List<Map<String, String>> processList) throws IOException {
+        ops.clear();
+        lines.clear();
+        String key;
+//        String encodedMetaValue;
+//        String path;
+        StringBuilder pathBuilder;
+        for (Map<String, String> map : processList) {
+            key = map.get("key");
+            if (key != null) {
+                lines.add(map);
+                pathBuilder = new StringBuilder("/restoreAr/")
+                        .append(UrlSafeBase64.encodeToString(String.join(":", bucket, key)))
+                        .append("/freezeAfterDays/").append(days);
+                ops.add(pathBuilder.toString());
+            } else {
+                fileSaveMapper.writeError("key is not exists or empty in " + map, false);
+            }
         }
-        bodyJson.add("entries", entries);
-        byte[] body = bodyJson.toString().getBytes();
-        bodyJson = null;
-        entries = null;
-        entry = null;
-        return HttpRespUtils.getResult(client.post(requestUrl, body,
-                auth.authorizationV2(requestUrl, "POST", body, Client.JsonMime), Client.JsonMime));
+        return lines;
+    }
+
+    @Override
+    protected String batchResult(List<Map<String, String>> lineList) throws IOException {
+        byte[] body = StringUtils.utf8Bytes(StringUtils.join(ops, "&op=", "op="));
+        return HttpRespUtils.getResult(client.post(CloudApiUtils.QINIU_RS_BATCH_URL, body,
+                auth.authorization(CloudApiUtils.QINIU_RS_BATCH_URL, body, Client.FormMime), Client.FormMime));
     }
 
     @Override
     protected List<Map<String, String>> parseBatchResult(List<Map<String, String>> processList, String result) throws Exception {
-        // 归档存储解冻操作本身不响应任何 body 内容
+        // 归档存储解冻操作可能不响应任何 body 内容
 //        if (result == null || "".equals(result)) throw new IOException("not valid refresh response.");
         if ("".equals(result)) {
             fileSaveMapper.writeSuccess(processList.stream().map(this::resultInfo).collect(Collectors.joining("\n")), false);
@@ -100,32 +114,25 @@ protected List<Map<String, String>> parseBatchResult(List<Map<String, String>> p
 
     @Override
     protected String singleResult(Map<String, String> line) throws IOException {
-        JsonObject bodyJson = new JsonObject();
-        JsonArray entries = new JsonArray();
-        JsonObject entry = new JsonObject();
-        entry.addProperty("bucket", bucket);
         String key = line.get("key");
-        entry.addProperty("key", key);
-        entry.addProperty("freeze_after_days", days);
-        entry.addProperty("cond", encodedCondition);
-        entries.add(entry);
-        bodyJson.add("entries", entries);
-        byte[] body = bodyJson.toString().getBytes();
-        Response response = client.post(requestUrl, body,
-                auth.authorizationV2(requestUrl, "POST", body, Client.JsonMime), Client.JsonMime);
+        if (key == null) throw new IOException("key is not exists or empty in " + line);
+        StringBuilder urlBuilder = new StringBuilder(requestUrl)
+                .append(UrlSafeBase64.encodeToString(String.join(":", bucket, key)))
+                .append("/freezeAfterDays/").append(days);
+        StringMap headers = auth.authorization(urlBuilder.toString(), null, Client.FormMime);
+        Response response = client.post(urlBuilder.toString(), null, headers, Client.FormMime);
         if (response.statusCode != 200) throw new QiniuException(response);
         response.close();
-        bodyJson = null;
-        entries = null;
-        entry = null;
-        body = null;
         return String.join("\t", key, "200");
     }
 
     @Override
     public void closeResource() {
         super.closeResource();
-        encodedCondition = null;
+        if (ops != null) ops.clear();
+        ops = null;
+        if (lines != null) lines.clear();
+        lines = null;
         configuration = null;
         auth = null;
         client = null;

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+ {"code":200}`
	`2`	`+7026814.ts {"code":200}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+UAA-4hndfVc5V6DJX0EvslAUBBI=/llmf1NC-2WRv3wJdHjFs0_szQ_vU {"code":200}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+upload_c1b9d660_e284_48c7_b180_2cf4d2debf13_00000135.tmp {"code":200}`