diff --git a/pom.xml b/pom.xml index 1621ff82..54732aa2 100755 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.qiniu qsuits - 8.3.4 + 8.3.5 qsuits qiniu-suits is a efficient tools for qiniu api implemented by java8. https://github.com/NigelWu95/qiniu-suits-java diff --git a/src/main/java/com/qiniu/datasource/TextContainer.java b/src/main/java/com/qiniu/datasource/TextContainer.java index b7bc5b82..ef5e28b6 100755 --- a/src/main/java/com/qiniu/datasource/TextContainer.java +++ b/src/main/java/com/qiniu/datasource/TextContainer.java @@ -146,7 +146,7 @@ public void export(ITextReader reader, IResultOutput saver, ILineProcess proc if (retry == 0) throw e; } } - if (srcList != null) statistics.addAndGet(srcList.size()); + statistics.addAndGet(srcList.size()); convertedList = converter.convertToVList(srcList); if (converter.errorSize() > 0) saver.writeError(converter.errorLines(), false); if (stringConverter != null) { diff --git a/src/main/java/com/qiniu/datasource/TextFileContainer.java b/src/main/java/com/qiniu/datasource/TextFileContainer.java index 624535d3..8bf52a58 100755 --- a/src/main/java/com/qiniu/datasource/TextFileContainer.java +++ b/src/main/java/com/qiniu/datasource/TextFileContainer.java @@ -66,7 +66,7 @@ protected ITextReader generateReader(String name) throws IOException { private List splitSingleFile(File file) throws IOException { int lineSize = FileUtils.predictLineSize(file); long linesNumber = file.length() / lineSize; - if (linesNumber < threads) { + if (linesNumber < threads * 2) { return new ArrayList(){{ add(new TextFileReader(file, null, unitLen)); }}; } long avgLines = linesNumber / threads; diff --git a/src/main/java/com/qiniu/datasource/TextFileRandomReader.java b/src/main/java/com/qiniu/datasource/TextFileRandomReader.java index 883bba7c..1d8a1ec4 100755 --- a/src/main/java/com/qiniu/datasource/TextFileRandomReader.java +++ b/src/main/java/com/qiniu/datasource/TextFileRandomReader.java @@ -28,7 +28,7 @@ public TextFileRandomReader(String name, RandomAccessFile accessFile, String end this.end = "".equals(end) ? null : end; if (originalLine != null) { line = new String(originalLine.getBytes(StandardCharsets.ISO_8859_1)); - if (this.end != null && this.end.compareTo(line) >= 0) lineList.add(line); + lineList.add(line); } } @@ -43,7 +43,9 @@ public List readLines() throws IOException { List srcList = new ArrayList<>(limit); srcList.addAll(lineList); lineList.clear(); - if (end != null) { + if (line.equals(this.end)) { + line = null; + } else if (end != null) { while (true) { if (srcList.size() >= limit) { line = null; diff --git a/src/main/java/com/qiniu/util/ProcessUtils.java b/src/main/java/com/qiniu/util/ProcessUtils.java index ef9d5bf5..12db1111 100755 --- a/src/main/java/com/qiniu/util/ProcessUtils.java +++ b/src/main/java/com/qiniu/util/ProcessUtils.java @@ -57,7 +57,6 @@ public final class ProcessUtils { add("qupload"); add("mime"); add("metadata"); - add("fetch"); add("syncupload"); }}; public static Set canBatchProcesses = new HashSet(){{ @@ -100,6 +99,7 @@ public final class ProcessUtils { }}; public static Set needQiniuAuthProcesses = new HashSet(){{ addAll(qiniuProcessesWithBucket); + add("fetch"); add("asyncfetch"); add("privateurl"); add("imagecensor"); diff --git a/version.properties b/version.properties index 6c4f9d3a..76e5e955 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -version=8.3.4 \ No newline at end of file +version=8.3.5 \ No newline at end of file