Skip to content

Commit 9928ac5

Browse files
svobol13svobol13
authored
Merge pull request #243 from svobol13/master
Fixed leaky buffer when streaming.
2 parents 04bde57 + ba73426 commit 9928ac5

File tree

6 files changed

+137
-22
lines changed

6 files changed

+137
-22
lines changed

‎src/main/java/com/jsoniter/IterImpl.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
5959
case '[': // If open symbol, increase level
6060
level++;
6161
break;
62-
case ']': // If close symbol, increase level
62+
case ']': // If close symbol, decrease level
6363
level--;
6464

6565
// If we have returned to the original level, we're done
@@ -85,7 +85,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
8585
case '{': // If open symbol, increase level
8686
level++;
8787
break;
88-
case '}': // If close symbol, increase level
88+
case '}': // If close symbol, decrease level
8989
level--;
9090

9191
// If we have returned to the original level, we're done

‎src/main/java/com/jsoniter/IterImplForStreaming.java

+23-18
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ final static void skipArray(JsonIterator iter) throws IOException {
7171
case '[': // If open symbol, increase level
7272
level++;
7373
break;
74-
case ']': // If close symbol, increase level
74+
case ']': // If close symbol, decrease level
7575
level--;
7676

7777
// If we have returned to the original level, we're done
@@ -101,7 +101,7 @@ final static void skipObject(JsonIterator iter) throws IOException {
101101
case '{': // If open symbol, increase level
102102
level++;
103103
break;
104-
case '}': // If close symbol, increase level
104+
case '}': // If close symbol, decrease level
105105
level--;
106106

107107
// If we have returned to the original level, we're done
@@ -147,7 +147,8 @@ final static void skipString(JsonIterator iter) throws IOException {
147147
throw iter.reportError("skipString", "incomplete string");
148148
}
149149
if (escaped) {
150-
iter.head = 1; // skip the first char as last char is \
150+
// TODO add unit test to prove/verify bug
151+
iter.head += 1; // skip the first char as last char is \
151152
}
152153
} else {
153154
iter.head = end;
@@ -274,19 +275,19 @@ public final static boolean loadMore(JsonIterator iter) throws IOException {
274275
}
275276

276277
private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOException {
277-
int n;
278-
int offset;
279-
if (iter.skipStartedAt == 0 || iter.skipStartedAt < iter.tail / 2) {
280-
byte[] newBuf = new byte[iter.buf.length * 2];
281-
offset = iter.tail - iter.skipStartedAt;
282-
System.arraycopy(iter.buf, iter.skipStartedAt, newBuf, 0, offset);
283-
iter.buf = newBuf;
284-
n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
285-
} else {
286-
offset = iter.tail - iter.skipStartedAt;
287-
System.arraycopy(iter.buf, iter.skipStartedAt, iter.buf, 0, offset);
288-
n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
289-
}
278+
int offset = iter.tail - iter.skipStartedAt;
279+
byte[] srcBuffer = iter.buf;
280+
// Check there is no unused buffer capacity
281+
if ((getUnusedBufferByteCount(iter)) == 0) {
282+
// If auto expand buffer enabled, then create larger buffer
283+
if (iter.autoExpandBufferStep > 0) {
284+
iter.buf = new byte[iter.buf.length + iter.autoExpandBufferStep];
285+
} else {
286+
throw iter.reportError("loadMore", String.format("buffer is full and autoexpansion is disabled. tail: [%s] skipStartedAt: [%s]", iter.tail, iter.skipStartedAt));
287+
}
288+
}
289+
System.arraycopy(srcBuffer, iter.skipStartedAt, iter.buf, 0, offset);
290+
int n = iter.in.read(iter.buf, offset, iter.buf.length - offset);
290291
iter.skipStartedAt = 0;
291292
if (n < 1) {
292293
if (n == -1) {
@@ -301,6 +302,11 @@ private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOExce
301302
return true;
302303
}
303304

305+
private static int getUnusedBufferByteCount(JsonIterator iter) {
306+
// Get bytes from 0 to skipStart + from tail till end
307+
return iter.buf.length - iter.tail + iter.skipStartedAt;
308+
}
309+
304310
final static byte readByte(JsonIterator iter) throws IOException {
305311
if (iter.head == iter.tail) {
306312
if (!loadMore(iter)) {
@@ -643,8 +649,7 @@ static final int readInt(final JsonIterator iter, final byte c) throws IOExcepti
643649

644650
static void assertNotLeadingZero(JsonIterator iter) throws IOException {
645651
try {
646-
byte nextByte = IterImpl.readByte(iter);
647-
iter.unreadByte();
652+
byte nextByte = iter.buf[iter.head];
648653
int ind2 = IterImplNumber.intDigits[nextByte];
649654
if (ind2 == IterImplNumber.INVALID_CHAR_FOR_NUMBER) {
650655
return;

‎src/main/java/com/jsoniter/JsonIterator.java

+13-1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ public class JsonIterator implements Closeable {
2121
final static ValueType[] valueTypes = new ValueType[256];
2222
InputStream in;
2323
byte[] buf;
24+
// Whenever buf is not large enough new one is created with size of
25+
// buf.length + autoExpandBufferStep. Set to < 1 to disable auto expanding.
26+
int autoExpandBufferStep;
2427
int head;
2528
int tail;
2629
int skipStartedAt = -1; // skip should keep bytes starting at this pos
@@ -60,13 +63,22 @@ private JsonIterator(InputStream in, byte[] buf, int head, int tail) {
6063
this.tail = tail;
6164
}
6265

66+
private JsonIterator(InputStream in, byte[] buf, int autoExpandBufferStep) {
67+
this(in, buf, 0, 0);
68+
this.autoExpandBufferStep = autoExpandBufferStep;
69+
}
70+
6371
public JsonIterator() {
6472
this(null, new byte[0], 0, 0);
6573
}
6674

6775
public static JsonIterator parse(InputStream in, int bufSize) {
76+
return parse(in, bufSize, bufSize);
77+
}
78+
79+
public static JsonIterator parse(InputStream in, int bufSize, int autoExpandBufferStep) {
6880
enableStreamingSupport();
69-
return new JsonIterator(in, new byte[bufSize], 0, 0);
81+
return new JsonIterator(in, new byte[bufSize], autoExpandBufferStep);
7082
}
7183

7284
public static JsonIterator parse(byte[] buf) {

‎src/test/java/com/jsoniter/IterImplForStreamingTest.java

+80-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
package com.jsoniter;
22

3+
import com.jsoniter.any.Any;
4+
import com.jsoniter.spi.JsonException;
5+
import java.io.ByteArrayInputStream;
6+
import java.io.IOException;
7+
import java.io.InputStream;
38
import junit.framework.TestCase;
9+
import org.junit.experimental.categories.Category;
10+
import sun.reflect.generics.reflectiveObjects.NotImplementedException;
411

512
public class IterImplForStreamingTest extends TestCase {
613

@@ -11,4 +18,76 @@ public void testReadMaxDouble() throws Exception {
1118
String number = new String(numberChars.chars, 0, numberChars.charsLength);
1219
assertEquals(maxDouble, number);
1320
}
14-
}
21+
22+
@Category(StreamingCategory.class)
23+
public void testLoadMore() throws IOException {
24+
final String originalContent = "1234567890";
25+
final byte[] src = ("{\"a\":\"" + originalContent + "\"}").getBytes();
26+
27+
int initialBufferSize;
28+
Any parsedString;
29+
// Case #1: Data fits into initial buffer, autoresizing on
30+
// Input must definitely fit into such large buffer
31+
initialBufferSize = src.length * 2;
32+
JsonIterator jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 512);
33+
jsonIterator.readObject();
34+
parsedString = jsonIterator.readAny();
35+
assertEquals(originalContent, parsedString.toString());
36+
// Check buffer was not expanded
37+
assertEquals(initialBufferSize, jsonIterator.buf.length);
38+
39+
// Case #2: Data does not fit into initial buffer, autoresizing off
40+
initialBufferSize = originalContent.length() / 2;
41+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 0);
42+
jsonIterator.readObject();
43+
try {
44+
jsonIterator.readAny();
45+
fail("Expect to fail because buffer is too small.");
46+
} catch (JsonException e) {
47+
if (!e.getMessage().startsWith("loadMore")) {
48+
throw e;
49+
}
50+
}
51+
// Check buffer was not expanded
52+
assertEquals(initialBufferSize, jsonIterator.buf.length);
53+
54+
// Case #3: Data does fit into initial buffer, autoresizing on
55+
initialBufferSize = originalContent.length() / 2;
56+
int autoExpandBufferStep = initialBufferSize * 3;
57+
jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, autoExpandBufferStep);
58+
jsonIterator.readObject();
59+
parsedString = jsonIterator.readAny();
60+
assertEquals(originalContent, parsedString.toString());
61+
// Check buffer was expanded exactly once
62+
assertEquals(initialBufferSize + autoExpandBufferStep, jsonIterator.buf.length);
63+
64+
// Case #4: Data does not fit (but largest string does) into initial buffer, autoresizing on
65+
initialBufferSize = originalContent.length() + 2;
66+
jsonIterator = JsonIterator.parse(new ByteArrayInputStream(src), initialBufferSize, 0);
67+
jsonIterator.readObject();
68+
parsedString = jsonIterator.readAny();
69+
assertEquals(originalContent, parsedString.toString());
70+
// Check buffer was expanded exactly once
71+
assertEquals(initialBufferSize, jsonIterator.buf.length);
72+
}
73+
74+
private static InputStream getSluggishInputStream(final byte[] src) {
75+
return new InputStream() {
76+
int position = 0;
77+
78+
@Override
79+
public int read() throws IOException {
80+
throw new NotImplementedException();
81+
}
82+
83+
@Override
84+
public int read(byte[] b, int off, int len) throws IOException {
85+
if (position < src.length) {
86+
b[off] = src[position++];
87+
return 1;
88+
}
89+
return -1;
90+
}
91+
};
92+
}
93+
}

‎src/test/java/com/jsoniter/any/TestLong.java

+18
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,28 @@
11
package com.jsoniter.any;
22

3+
import com.jsoniter.spi.JsonException;
34
import junit.framework.TestCase;
45

56
public class TestLong extends TestCase {
67
public void test_to_string_should_trim() {
78
Any any = Any.lazyLong(" 1000".getBytes(), 0, " 1000".length());
89
assertEquals("1000", any.toString());
910
}
11+
12+
public void test_should_fail_with_leading_zero() {
13+
byte[] bytes = "01".getBytes();
14+
Any any = Any.lazyLong(bytes, 0, bytes.length);
15+
try {
16+
any.toLong();
17+
fail("This should fail.");
18+
} catch (JsonException e) {
19+
20+
}
21+
}
22+
23+
public void test_should_work_with_zero() {
24+
byte[] bytes = "0".getBytes();
25+
Any any = Any.lazyLong(bytes, 0, bytes.length);
26+
assertEquals(0L, any.toLong());
27+
}
1028
}

‎src/test/java/com/jsoniter/suite/AllTestCases.java

+1
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
TestGson.class,
5454
com.jsoniter.output.TestGson.class,
5555
TestStreamBuffer.class,
56+
IterImplForStreamingTest.class,
5657
TestCollection.class,
5758
TestList.class,
5859
TestAnnotationJsonObject.class,

0 commit comments

Comments
 (0)