Merge pull request #281 from NigelWu95/dev

fix TextFileRandomReader and ProcessUtils.needBucketProcesses and rel…

Author: 吴炳亨

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.qiniu</groupId>
     <artifactId>qsuits</artifactId>
-    <version>8.3.4</version>
+    <version>8.3.5</version>
     <name>qsuits</name>
     <description>qiniu-suits is a efficient tools for qiniu api implemented by java8.</description>
     <url>https://github.com/NigelWu95/qiniu-suits-java</url>

src/main/java/com/qiniu/datasource/TextContainer.java

@@ -146,7 +146,7 @@ public void export(ITextReader reader, IResultOutput saver, ILineProcess<T> proc
                     if (retry == 0) throw e;
                 }
             }
-            if (srcList != null) statistics.addAndGet(srcList.size());
+            statistics.addAndGet(srcList.size());
             convertedList = converter.convertToVList(srcList);
             if (converter.errorSize() > 0) saver.writeError(converter.errorLines(), false);
             if (stringConverter != null) {

src/main/java/com/qiniu/datasource/TextFileContainer.java

@@ -66,7 +66,7 @@ protected ITextReader generateReader(String name) throws IOException {
     private List<ITextReader> splitSingleFile(File file) throws IOException {
         int lineSize = FileUtils.predictLineSize(file);
         long linesNumber = file.length() / lineSize;
-        if (linesNumber < threads) {
+        if (linesNumber < threads * 2) {
             return new ArrayList<ITextReader>(){{ add(new TextFileReader(file, null, unitLen)); }};
         }
         long avgLines = linesNumber / threads;

src/main/java/com/qiniu/datasource/TextFileRandomReader.java

@@ -28,7 +28,7 @@ public TextFileRandomReader(String name, RandomAccessFile accessFile, String end
         this.end = "".equals(end) ? null : end;
         if (originalLine != null) {
             line = new String(originalLine.getBytes(StandardCharsets.ISO_8859_1));
-            if (this.end != null && this.end.compareTo(line) >= 0) lineList.add(line);
+            lineList.add(line);
         }
     }
 
@@ -43,7 +43,9 @@ public List<String> readLines() throws IOException {
         List<String> srcList = new ArrayList<>(limit);
         srcList.addAll(lineList);
         lineList.clear();
-        if (end != null) {
+        if (line.equals(this.end)) {
+            line = null;
+        } else if (end != null) {
             while (true) {
                 if (srcList.size() >= limit) {
                     line = null;

src/main/java/com/qiniu/util/ProcessUtils.java

@@ -57,7 +57,6 @@ public final class ProcessUtils {
         add("qupload");
         add("mime");
         add("metadata");
-        add("fetch");
         add("syncupload");
     }};
     public static Set<String> canBatchProcesses = new HashSet<String>(){{
@@ -100,6 +99,7 @@ public final class ProcessUtils {
     }};
     public static Set<String> needQiniuAuthProcesses = new HashSet<String>(){{
         addAll(qiniuProcessesWithBucket);
+        add("fetch");
         add("asyncfetch");
         add("privateurl");
         add("imagecensor");

version.properties

@@ -1 +1 @@
-version=8.3.4
+version=8.3.5