Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Limits on input stream in codecs

Browse Source 
- Add maxInMemorySize property to Decoder and HttpMessageReader
  implementations that aggregate input to trigger
  DataBufferLimitException when reached.

- For codecs that call DataBufferUtils#join, there is now an overloaded
  variant with a maxInMemorySize extra argument. Internally, a custom
  LimitedDataBufferList is used to count and enforce the limit.

- Jackson2Tokenizer and XmlEventDecoder support those limits per
  streamed JSON object.

See gh-23884
This commit is contained in:
Rossen Stoyanchev
2019-10-28 14:26:26 +00:00
parent ce0b012f43
commit 89d053d7f4
16 changed files with 672 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
spring-web/src/test/java/org/springframework/http/codec/json/Jackson2TokenizerTests.java
View File

@@ -20,7 +20,6 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.function.Consumer;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.TreeNode;
@@ -36,6 +35,7 @@ import reactor.test.StepVerifier;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.io.buffer.AbstractLeakCheckingTests;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
@@ -181,11 +181,68 @@ public class Jackson2TokenizerTests extends AbstractLeakCheckingTests {
testTokenize(asList("[1", ",2,", "3]"), asList("1", "2", "3"), true);
}
private void testTokenize(List<String> input, List<String> output, boolean tokenize) {
StepVerifier.FirstStep<String> builder = StepVerifier.create(decode(input, tokenize, -1));
output.forEach(expected -> builder.assertNext(actual -> {
try {
JSONAssert.assertEquals(expected, actual, true);
}
catch (JSONException ex) {
throw new RuntimeException(ex);
}
}));
builder.verifyComplete();
}
@Test
public void testLimit() {
List<String> source = asList("[",
"{", "\"id\":1,\"name\":\"Dan\"", "},",
"{", "\"id\":2,\"name\":\"Ron\"", "},",
"{", "\"id\":3,\"name\":\"Bartholomew\"", "}",
"]");
String expected = String.join("", source);
int maxInMemorySize = expected.length();
StepVerifier.create(decode(source, false, maxInMemorySize))
.expectNext(expected)
.verifyComplete();
StepVerifier.create(decode(source, false, maxInMemorySize - 1))
.expectError(DataBufferLimitException.class);
}
@Test
public void testLimitTokenized() {
List<String> source = asList("[",
"{", "\"id\":1, \"name\":\"Dan\"", "},",
"{", "\"id\":2, \"name\":\"Ron\"", "},",
"{", "\"id\":3, \"name\":\"Bartholomew\"", "}",
"]");
String expected = "{\"id\":3,\"name\":\"Bartholomew\"}";
int maxInMemorySize = expected.length();
StepVerifier.create(decode(source, true, maxInMemorySize))
.expectNext("{\"id\":1,\"name\":\"Dan\"}")
.expectNext("{\"id\":2,\"name\":\"Ron\"}")
.expectNext(expected)
.verifyComplete();
StepVerifier.create(decode(source, true, maxInMemorySize - 1))
.expectNext("{\"id\":1,\"name\":\"Dan\"}")
.expectNext("{\"id\":2,\"name\":\"Ron\"}")
.verifyError(DataBufferLimitException.class);
}
@Test
public void errorInStream() {
DataBuffer buffer = stringBuffer("{\"id\":1,\"name\":");
Flux<DataBuffer> source = Flux.just(buffer).concatWith(Flux.error(new RuntimeException()));
Flux<TokenBuffer> result = Jackson2Tokenizer.tokenize(source, this.jsonFactory, this.objectMapper, true);
Flux<TokenBuffer> result = Jackson2Tokenizer.tokenize(source, this.jsonFactory, this.objectMapper, true, -1);
StepVerifier.create(result)
.expectError(RuntimeException.class)
@@ -195,7 +252,7 @@ public class Jackson2TokenizerTests extends AbstractLeakCheckingTests {
@Test // SPR-16521
public void jsonEOFExceptionIsWrappedAsDecodingError() {
Flux<DataBuffer> source = Flux.just(stringBuffer("{\"status\": \"noClosingQuote}"));
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(source, this.jsonFactory, this.objectMapper, false);
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(source, this.jsonFactory, this.objectMapper, false, -1);
StepVerifier.create(tokens)
.expectError(DecodingException.class)
@@ -203,12 +260,13 @@ public class Jackson2TokenizerTests extends AbstractLeakCheckingTests {
}
private void testTokenize(List<String> source, List<String> expected, boolean tokenizeArrayElements) {
private Flux<String> decode(List<String> source, boolean tokenize, int maxInMemorySize) {
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(
Flux.fromIterable(source).map(this::stringBuffer),
this.jsonFactory, this.objectMapper, tokenizeArrayElements);
this.jsonFactory, this.objectMapper, tokenize, maxInMemorySize);
Flux<String> result = tokens
return tokens
.map(tokenBuffer -> {
try {
TreeNode root = this.objectMapper.readTree(tokenBuffer.asParser());
@@ -218,10 +276,6 @@ public class Jackson2TokenizerTests extends AbstractLeakCheckingTests {
throw new UncheckedIOException(ex);
}
});
StepVerifier.FirstStep<String> builder = StepVerifier.create(result);
expected.forEach(s -> builder.assertNext(new JSONAssertConsumer(s)));
builder.verifyComplete();
}
private DataBuffer stringBuffer(String value) {
@@ -231,24 +285,4 @@ public class Jackson2TokenizerTests extends AbstractLeakCheckingTests {
return buffer;
}
private static class JSONAssertConsumer implements Consumer<String> {
private final String expected;
JSONAssertConsumer(String expected) {
this.expected = expected;
}
@Override
public void accept(String s) {
try {
JSONAssert.assertEquals(this.expected, s, true);
}
catch (JSONException ex) {
throw new RuntimeException(ex);
}
}
}
}

48
spring-web/src/test/java/org/springframework/http/codec/xml/XmlEventDecoderTests.java
View File

@@ -28,6 +28,7 @@ import reactor.test.StepVerifier;
import org.springframework.core.io.buffer.AbstractLeakCheckingTests;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import static org.assertj.core.api.Assertions.assertThat;
@@ -44,11 +45,12 @@ public class XmlEventDecoderTests extends AbstractLeakCheckingTests {
private XmlEventDecoder decoder = new XmlEventDecoder();
@Test
public void toXMLEventsAalto() {
Flux<XMLEvent> events =
this.decoder.decode(stringBuffer(XML), null, null, Collections.emptyMap());
this.decoder.decode(stringBufferMono(XML), null, null, Collections.emptyMap());
StepVerifier.create(events)
.consumeNextWith(e -> assertThat(e.isStartDocument()).isTrue())
@@ -69,7 +71,7 @@ public class XmlEventDecoderTests extends AbstractLeakCheckingTests {
decoder.useAalto = false;
Flux<XMLEvent> events =
this.decoder.decode(stringBuffer(XML), null, null, Collections.emptyMap());
this.decoder.decode(stringBufferMono(XML), null, null, Collections.emptyMap());
StepVerifier.create(events)
.consumeNextWith(e -> assertThat(e.isStartDocument()).isTrue())
@@ -86,10 +88,32 @@ public class XmlEventDecoderTests extends AbstractLeakCheckingTests {
.verify();
}
@Test
public void toXMLEventsWithLimit() {
this.decoder.setMaxInMemorySize(6);
Flux<String> source = Flux.just(
"<pojo>", "<foo>", "foofoo", "</foo>", "<bar>", "barbarbar", "</bar>", "</pojo>");
Flux<XMLEvent> events = this.decoder.decode(
source.map(this::stringBuffer), null, null, Collections.emptyMap());
StepVerifier.create(events)
.consumeNextWith(e -> assertThat(e.isStartDocument()).isTrue())
.consumeNextWith(e -> assertStartElement(e, "pojo"))
.consumeNextWith(e -> assertStartElement(e, "foo"))
.consumeNextWith(e -> assertCharacters(e, "foofoo"))
.consumeNextWith(e -> assertEndElement(e, "foo"))
.consumeNextWith(e -> assertStartElement(e, "bar"))
.expectError(DataBufferLimitException.class)
.verify();
}
@Test
public void decodeErrorAalto() {
Flux<DataBuffer> source = Flux.concat(
stringBuffer("<pojo>"),
stringBufferMono("<pojo>"),
Flux.error(new RuntimeException()));
Flux<XMLEvent> events =
@@ -107,7 +131,7 @@ public class XmlEventDecoderTests extends AbstractLeakCheckingTests {
decoder.useAalto = false;
Flux<DataBuffer> source = Flux.concat(
stringBuffer("<pojo>"),
stringBufferMono("<pojo>"),
Flux.error(new RuntimeException()));
Flux<XMLEvent> events =
@@ -133,13 +157,15 @@ public class XmlEventDecoderTests extends AbstractLeakCheckingTests {
assertThat(event.asCharacters().getData()).isEqualTo(expectedData);
}
private Mono<DataBuffer> stringBuffer(String value) {
return Mono.defer(() -> {
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
DataBuffer buffer = this.bufferFactory.allocateBuffer(bytes.length);
buffer.write(bytes);
return Mono.just(buffer);
});
private DataBuffer stringBuffer(String value) {
byte[] bytes = value.getBytes(StandardCharsets.UTF_8);
DataBuffer buffer = this.bufferFactory.allocateBuffer(bytes.length);
buffer.write(bytes);
return buffer;
}
private Mono<DataBuffer> stringBufferMono(String value) {
return Mono.defer(() -> Mono.just(stringBuffer(value)));
}
}