optimize converting utils.

Author: wubingheng

Diff for: src/main/java/com/qiniu/convert/Converter.java

@@ -21,7 +21,7 @@ public List<T> convertToVList(List<E> lineList) {
                 try {
                     mapList.add(convertToV(line));
                 } catch (Exception e) {
-                    errorList.add(JsonUtils.toJson(line) + "\tcvterr " + e.getMessage());
+                    errorList.add(JsonUtils.toJson(line) + "\tconvert error " + e.getMessage());
                 }
             }
         }

Diff for: src/main/java/com/qiniu/process/filtration/BaseFilter.java

@@ -1,5 +1,7 @@
 package com.qiniu.process.filtration;
 
+import com.qiniu.util.ConvertingUtils;
+
 import java.io.IOException;
 import java.time.LocalDateTime;
 import java.util.List;
@@ -111,7 +113,7 @@ public boolean filterKey(T item) {
     public boolean filterMimeType(T item) {
         try {
             if (item == null) return false;
-            String mType = valueFrom(item, "mime");
+            String mType = valueFrom(item, ConvertingUtils.defaultMimeField);
             boolean result = false;
             if (checkList(mimeType)) {
                 result = mimeType.stream().anyMatch(mType::contains);
@@ -127,7 +129,7 @@ public boolean filterMimeType(T item) {
     public boolean filterDatetime(T item) {
         try {
             if (item == null) return false;
-            LocalDateTime localDateTime = LocalDateTime.parse(valueFrom(item, "datetime"));
+            LocalDateTime localDateTime = LocalDateTime.parse(valueFrom(item, ConvertingUtils.defaultDatetimeField));
             return localDateTime.compareTo(datetimeMax) <= 0 && localDateTime.compareTo(datetimeMin) >= 0;
         } catch (Exception e) {
             return true;
@@ -137,7 +139,7 @@ public boolean filterDatetime(T item) {
     public boolean filterType(T item) {
         try {
             if (item == null) return false;
-            return valueFrom(item, "type").equals(type);
+            return valueFrom(item, ConvertingUtils.defaultTypeField).equals(type);
         } catch (NullPointerException e) {
             return true;
         }
@@ -146,7 +148,7 @@ public boolean filterType(T item) {
     public boolean filterStatus(T item) {
         try {
             if (item == null) return false;
-            return valueFrom(item, "status").equals(status);
+            return valueFrom(item, ConvertingUtils.defaultStatusField).equals(status);
         } catch (NullPointerException e) {
             return true;
         }

Diff for: src/main/java/com/qiniu/process/filtration/SeniorFilter.java

@@ -5,6 +5,7 @@
 import com.google.gson.JsonObject;
 import com.google.gson.reflect.TypeToken;
 import com.qiniu.config.JsonFile;
+import com.qiniu.util.ConvertingUtils;
 import com.qiniu.util.JsonUtils;
 
 import java.io.IOException;
@@ -59,7 +60,8 @@ public List<T> checkMimeType(List<T> lineList) {
                 if (line == null) continue;
                 key = valueFrom(line, "key");
                 if (key.contains(".")) {
-                    String finalKeyMimePair = key.substring(key.lastIndexOf(".") + 1) + ":" + valueFrom(line, "mime");
+                    String finalKeyMimePair = key.substring(key.lastIndexOf(".") + 1) + ":" +
+                            valueFrom(line, ConvertingUtils.defaultMimeField);
                     if (extMimeList.parallelStream().anyMatch(extMime ->
                             finalKeyMimePair.split("/")[0].equalsIgnoreCase(extMime))) {
                         continue;
@@ -83,7 +85,8 @@ public boolean checkMimeType(T line) {
         try {
             key = valueFrom(line, "key");
             if (key.contains(".")) {
-                String finalKeyMimePair = key.substring(key.lastIndexOf(".") + 1) + ":" + valueFrom(line, "mime");
+                String finalKeyMimePair = key.substring(key.lastIndexOf(".") + 1) + ":" +
+                        valueFrom(line, ConvertingUtils.defaultMimeField);
                 if (extMimeList.parallelStream().anyMatch(extMime ->
                         finalKeyMimePair.split("/")[0].equalsIgnoreCase(extMime))) {
                     return false;

Diff for: src/main/java/com/qiniu/util/ConvertingUtils.java

@@ -275,7 +275,7 @@ public static <T> T toPair(JsonObject json, Map<String, String> indexMap, KeyVal
         for (String index : indexMap.keySet()) {
             jsonElement = json.get(index);
             if (jsonElement == null || jsonElement instanceof JsonNull) {
-                throw new IOException("the index: " + index + " can't be found in " + json);
+                if (!allFieldsSet.contains(index)) throw new IOException("the index: " + index + " can't be found in " + json);
             } else {
                 pair.put(indexMap.get(index), JsonUtils.toString(jsonElement));
             }
@@ -314,7 +314,7 @@ public static <T> T toPair(Map<String, String> line, List<String> fields, KeyVal
                 else if (intFields.contains(field)) pair.put(field, Integer.valueOf(value));
                 else pair.put(field, value);
             } else {
-                throw new IOException("the field: " + field + " can't be found in " + line);
+                if (!allFieldsSet.contains(field)) throw new IOException("the field: " + field + " can't be found in " + line);
             }
         }
         if (pair.size() == 0) throw new IOException("empty result string.");