Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Limits on input stream in codecs

Browse Source 
- Add maxInMemorySize property to Decoder and HttpMessageReader
  implementations that aggregate input to trigger
  DataBufferLimitException when reached.

- For codecs that call DataBufferUtils#join, there is now an overloaded
  variant with a maxInMemorySize extra argument. Internally, a custom
  LimitedDataBufferList is used to count and enforce the limit.

- Jackson2Tokenizer and XmlEventDecoder support those limits per
  streamed JSON object.

See gh-23884
This commit is contained in:
Rossen Stoyanchev
2019-10-28 14:26:26 +00:00
parent ce0b012f43
commit 89d053d7f4
16 changed files with 672 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
spring-web/src/main/java/org/springframework/http/codec/FormHttpMessageReader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,6 +30,7 @@ import reactor.core.publisher.Mono;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.http.MediaType;
@@ -62,6 +63,8 @@ public class FormHttpMessageReader extends LoggingCodecSupport
private Charset defaultCharset = DEFAULT_CHARSET;
private int maxInMemorySize = 256 * 1024;
/**
* Set the default character set to use for reading form data when the
@@ -80,6 +83,26 @@ public class FormHttpMessageReader extends LoggingCodecSupport
return this.defaultCharset;
}
/**
* Set the max number of bytes for input form data. As form data is buffered
* before it is parsed, this helps to limit the amount of buffering. Once
* the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default this is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canRead(ResolvableType elementType, @Nullable MediaType mediaType) {
@@ -105,7 +128,7 @@ public class FormHttpMessageReader extends LoggingCodecSupport
MediaType contentType = message.getHeaders().getContentType();
Charset charset = getMediaTypeCharset(contentType);
return DataBufferUtils.join(message.getBody())
return DataBufferUtils.join(message.getBody(), this.maxInMemorySize)
.map(buffer -> {
CharBuffer charBuffer = charset.decode(buffer.asByteBuffer());
String body = charBuffer.toString();

30
spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java
View File

@@ -37,6 +37,7 @@ import org.springframework.core.codec.CodecException;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.http.codec.HttpMessageDecoder;
@@ -59,6 +60,9 @@ import org.springframework.util.MimeType;
*/
public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport implements HttpMessageDecoder<Object> {
private int maxInMemorySize = 256 * 1024;
/**
* Constructor with a Jackson {@link ObjectMapper} to use.
*/
@@ -67,6 +71,28 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
}
/**
* Set the max number of bytes that can be buffered by this decoder. This
* is either the size of the entire input when decoding as a whole, or the
* size of one top-level JSON object within a JSON stream. When the limit
* is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default this is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {
JavaType javaType = getObjectMapper().getTypeFactory().constructType(elementType.getType());
@@ -81,7 +107,7 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
ObjectMapper mapper = getObjectMapper();
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(
Flux.from(input), mapper.getFactory(), mapper, true);
Flux.from(input), mapper.getFactory(), mapper, true, getMaxInMemorySize());
ObjectReader reader = getObjectReader(elementType, hints);
@@ -103,7 +129,7 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
public Mono<Object> decodeToMono(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
return DataBufferUtils.join(input)
return DataBufferUtils.join(input, this.maxInMemorySize)
.map(dataBuffer -> decode(dataBuffer, elementType, mimeType, hints));
}

46
spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java
View File

@@ -35,6 +35,7 @@ import reactor.core.publisher.Flux;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
/**
@@ -61,30 +62,39 @@ final class Jackson2Tokenizer {
private int arrayDepth;
private final int maxInMemorySize;
private int byteCount;
// TODO: change to ByteBufferFeeder when supported by Jackson
// See https://github.com/FasterXML/jackson-core/issues/478
private final ByteArrayFeeder inputFeeder;
private Jackson2Tokenizer(
JsonParser parser, DeserializationContext deserializationContext, boolean tokenizeArrayElements) {
private Jackson2Tokenizer(JsonParser parser, DeserializationContext deserializationContext,
boolean tokenizeArrayElements, int maxInMemorySize) {
this.parser = parser;
this.deserializationContext = deserializationContext;
this.tokenizeArrayElements = tokenizeArrayElements;
this.tokenBuffer = new TokenBuffer(parser, deserializationContext);
this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder();
this.maxInMemorySize = maxInMemorySize;
}
private List<TokenBuffer> tokenize(DataBuffer dataBuffer) {
byte[] bytes = new byte[dataBuffer.readableByteCount()];
int bufferSize = dataBuffer.readableByteCount();
byte[] bytes = new byte[bufferSize];
dataBuffer.read(bytes);
DataBufferUtils.release(dataBuffer);
try {
this.inputFeeder.feedInput(bytes, 0, bytes.length);
return parseTokenBufferFlux();
List<TokenBuffer> result = parseTokenBufferFlux();
assertInMemorySize(bufferSize, result);
return result;
}
catch (JsonProcessingException ex) {
throw new DecodingException("JSON decoding error: " + ex.getOriginalMessage(), ex);
@@ -174,18 +184,40 @@ final class Jackson2Tokenizer {
(token == JsonToken.END_ARRAY && this.arrayDepth == 0));
}
private void assertInMemorySize(int currentBufferSize, List<TokenBuffer> result) {
if (this.maxInMemorySize >= 0) {
if (!result.isEmpty()) {
this.byteCount = 0;
}
else if (currentBufferSize > Integer.MAX_VALUE - this.byteCount) {
raiseLimitException();
}
else {
this.byteCount += currentBufferSize;
if (this.byteCount > this.maxInMemorySize) {
raiseLimitException();
}
}
}
}
private void raiseLimitException() {
throw new DataBufferLimitException(
"Exceeded limit on max bytes per JSON object: " + this.maxInMemorySize);
}
/**
* Tokenize the given {@code Flux<DataBuffer>} into {@code Flux<TokenBuffer>}.
* @param dataBuffers the source data buffers
* @param jsonFactory the factory to use
* @param objectMapper the current mapper instance
* @param tokenizeArrayElements if {@code true} and the "top level" JSON object is
* @param tokenizeArrays if {@code true} and the "top level" JSON object is
* an array, each element is returned individually immediately after it is received
* @return the resulting token buffers
*/
public static Flux<TokenBuffer> tokenize(Flux<DataBuffer> dataBuffers, JsonFactory jsonFactory,
ObjectMapper objectMapper, boolean tokenizeArrayElements) {
ObjectMapper objectMapper, boolean tokenizeArrays, int maxInMemorySize) {
try {
JsonParser parser = jsonFactory.createNonBlockingByteArrayParser();
@@ -194,7 +226,7 @@ final class Jackson2Tokenizer {
context = ((DefaultDeserializationContext) context).createInstance(
objectMapper.getDeserializationConfig(), parser, objectMapper.getInjectableValues());
}
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrayElements);
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrays, maxInMemorySize);
return dataBuffers.concatMapIterable(tokenizer::tokenize).concatWith(tokenizer.endOfInput());
}
catch (IOException ex) {

7
spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufDecoder.java
View File

@@ -36,6 +36,7 @@ import org.springframework.core.ResolvableType;
import org.springframework.core.codec.Decoder;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
@@ -127,7 +128,7 @@ public class ProtobufDecoder extends ProtobufCodecSupport implements Decoder<Mes
public Mono<Message> decodeToMono(Publisher<DataBuffer> inputStream, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
return DataBufferUtils.join(inputStream)
return DataBufferUtils.join(inputStream, this.maxMessageSize)
.map(dataBuffer -> decode(dataBuffer, elementType, mimeType, hints));
}
@@ -205,8 +206,8 @@ public class ProtobufDecoder extends ProtobufCodecSupport implements Decoder<Mes
return messages;
}
if (this.messageBytesToRead > this.maxMessageSize) {
throw new DecodingException(
"The number of bytes to read from the incoming stream " +
throw new DataBufferLimitException(
"The number of bytes to read for message " +
"(" + this.messageBytesToRead + ") exceeds " +
"the configured limit (" + this.maxMessageSize + ")");
}

27
spring-web/src/main/java/org/springframework/http/codec/xml/Jaxb2XmlDecoder.java
View File

@@ -49,6 +49,7 @@ import org.springframework.core.codec.CodecException;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.lang.Nullable;
@@ -87,6 +88,8 @@ public class Jaxb2XmlDecoder extends AbstractDecoder<Object> {
private Function<Unmarshaller, Unmarshaller> unmarshallerProcessor = Function.identity();
private int maxInMemorySize = 256 * 1024;
public Jaxb2XmlDecoder() {
super(MimeTypeUtils.APPLICATION_XML, MimeTypeUtils.TEXT_XML);
@@ -119,6 +122,28 @@ public class Jaxb2XmlDecoder extends AbstractDecoder<Object> {
return this.unmarshallerProcessor;
}
/**
* Set the max number of bytes that can be buffered by this decoder.
* This is either the size of the entire input when decoding as a whole, or when
* using async parsing with Aalto XML, it is the size of one top-level XML tree.
* When the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default this is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
this.xmlEventDecoder.setMaxInMemorySize(byteCount);
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {
@@ -153,7 +178,7 @@ public class Jaxb2XmlDecoder extends AbstractDecoder<Object> {
public Mono<Object> decodeToMono(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
return DataBufferUtils.join(input)
return DataBufferUtils.join(input, this.maxInMemorySize)
.map(dataBuffer -> decode(dataBuffer, elementType, mimeType, hints));
}

78
spring-web/src/main/java/org/springframework/http/codec/xml/XmlEventDecoder.java
View File

@@ -40,6 +40,7 @@ import reactor.core.publisher.Flux;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.AbstractDecoder;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.ClassUtils;
@@ -89,26 +90,50 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
boolean useAalto = aaltoPresent;
private int maxInMemorySize = 256 * 1024;
public XmlEventDecoder() {
super(MimeTypeUtils.APPLICATION_XML, MimeTypeUtils.TEXT_XML);
}
/**
* Set the max number of bytes that can be buffered by this decoder. This
* is either the size the entire input when decoding as a whole, or when
* using async parsing via Aalto XML, it is size one top-level XML tree.
* When the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default this is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
@SuppressWarnings({"rawtypes", "unchecked", "cast"}) // XMLEventReader is Iterator<Object> on JDK 9
public Flux<XMLEvent> decode(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
if (this.useAalto) {
AaltoDataBufferToXmlEvent mapper = new AaltoDataBufferToXmlEvent();
AaltoDataBufferToXmlEvent mapper = new AaltoDataBufferToXmlEvent(this.maxInMemorySize);
return Flux.from(input)
.flatMapIterable(mapper)
.doFinally(signalType -> mapper.endOfInput());
}
else {
return DataBufferUtils.join(input).
flatMapIterable(buffer -> {
return DataBufferUtils.join(input, this.maxInMemorySize)
.flatMapIterable(buffer -> {
try {
InputStream is = buffer.asInputStream();
Iterator eventReader = inputFactory.createXMLEventReader(is);
@@ -140,10 +165,22 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
private final XMLEventAllocator eventAllocator = EventAllocatorImpl.getDefaultInstance();
private final int maxInMemorySize;
private int byteCount;
private int elementDepth;
public AaltoDataBufferToXmlEvent(int maxInMemorySize) {
this.maxInMemorySize = maxInMemorySize;
}
@Override
public List<? extends XMLEvent> apply(DataBuffer dataBuffer) {
try {
increaseByteCount(dataBuffer);
this.streamReader.getInputFeeder().feedInput(dataBuffer.asByteBuffer());
List<XMLEvent> events = new ArrayList<>();
while (true) {
@@ -157,8 +194,12 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
if (event.isEndDocument()) {
break;
}
checkDepthAndResetByteCount(event);
}
}
if (this.maxInMemorySize > 0 && this.byteCount > this.maxInMemorySize) {
raiseLimitException();
}
return events;
}
catch (XMLStreamException ex) {
@@ -169,9 +210,40 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
}
}
private void increaseByteCount(DataBuffer dataBuffer) {
if (this.maxInMemorySize > 0) {
if (dataBuffer.readableByteCount() > Integer.MAX_VALUE - this.byteCount) {
raiseLimitException();
}
else {
this.byteCount += dataBuffer.readableByteCount();
}
}
}
private void checkDepthAndResetByteCount(XMLEvent event) {
if (this.maxInMemorySize > 0) {
if (event.isStartElement()) {
this.byteCount = this.elementDepth == 1 ? 0 : this.byteCount;
this.elementDepth++;
}
else if (event.isEndElement()) {
this.elementDepth--;
this.byteCount = this.elementDepth == 1 ? 0 : this.byteCount;
}
}
}
private void raiseLimitException() {
throw new DataBufferLimitException(
"Exceeded limit on max bytes per XML top-level node: " + this.maxInMemorySize);
}
public void endOfInput() {
this.streamReader.getInputFeeder().endOfInput();
}
}
}