Refactor StringDecoder

Simplify and optimize the processing of the input stream.

The existing implementation was using bufferUntil and creating a List
for every line along with an EndFrameBuffer inserted for the
bufferUntil predicate. So the larger the input buffer and the more
lines it contained, the greater the overhead.

The new implementation avoids bufferUntil for all lines and
instead uses concatMapIterable to aggregate the lines from a buffer
into a single list. So the larger the input buffer and the more
lines it contains, the better the throughput. The only buffering
used then is for partial chunks and those are accumulated in a list.

See gh-25915
This commit is contained in:
Rossen Stoyanchev
2020-10-22 21:28:06 +01:00
parent f7ec92c647
commit db9e0b0ccb
2 changed files with 73 additions and 132 deletions

View File

@@ -139,20 +139,28 @@ class StringDecoderTests extends AbstractDecoderTests<StringDecoder> {
@Test
void maxInMemoryLimit() {
Flux<DataBuffer> input = Flux.just(
stringBuffer("abc\n"), stringBuffer("defg\n"), stringBuffer("hijkl\n"));
stringBuffer("abc\n"), stringBuffer("defg\n"),
stringBuffer("hi"), stringBuffer("jkl"), stringBuffer("mnop"));
this.decoder.setMaxInMemorySize(5);
testDecode(input, String.class, step ->
step.expectNext("abc", "defg").verifyError(DataBufferLimitException.class));
}
@Test // gh-24312
void maxInMemoryLimitReleaseUnprocessedLinesFromCurrentBuffer() {
@Test
void maxInMemoryLimitDoesNotApplyToParsedItemsThatDontRequireBuffering() {
Flux<DataBuffer> input = Flux.just(
stringBuffer("TOO MUCH DATA\nanother line\n\nand another\n"));
this.decoder.setMaxInMemorySize(5);
testDecode(input, String.class, step -> step.verifyError(DataBufferLimitException.class));
testDecode(input, String.class, step -> step
.expectNext("TOO MUCH DATA")
.expectNext("another line")
.expectNext("")
.expectNext("and another")
.expectComplete()
.verify());
}
@Test // gh-24339