Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Limits on input stream in codecs

Browse Source 
- Add maxInMemorySize property to Decoder and HttpMessageReader
  implementations that aggregate input to trigger
  DataBufferLimitException when reached.

- For codecs that call DataBufferUtils#join, there is now an overloaded
  variant with a maxInMemorySize extra argument. Internally, a custom
  LimitedDataBufferList is used to count and enforce the limit.

- Jackson2Tokenizer and XmlEventDecoder support those limits per
  streamed JSON object.

- Configurable limits for multipart requests with Synchronoss NIO.

- Centralized maxInMemorySize exposed via CodecConfigurer along with
  ability to plug in an instance of MultipartHttpMessageWrite.

Closes gh-23884
This commit is contained in:
Rossen Stoyanchev
2019-10-28 14:26:26 +00:00
parent cf1b7620c7
commit 871464811c
25 changed files with 1169 additions and 191 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
spring-web/src/main/java/org/springframework/http/codec/CodecConfigurer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -143,6 +143,21 @@ public interface CodecConfigurer {
*/
void jaxb2Encoder(Encoder<?> encoder);
/**
* Configure a limit on the number of bytes that can be buffered whenever
* the input stream needs to be aggregated. This can be a result of
* decoding to a single {@code DataBuffer},
* {@link java.nio.ByteBuffer ByteBuffer}, {@code byte[]},
* {@link org.springframework.core.io.Resource Resource}, {@code String}, etc.
* It can also occur when splitting the input stream, e.g. delimited text,
* in which case the limit applies to data buffered between delimiters.
* <p>By default this is not set, in which case individual codec defaults
* apply. In 5.1 most codecs are not limited except {@code FormHttpMessageReader}
* which is limited to 256K. In 5.2 all codecs are limited to 256K by default.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @sine 5.1.11
*/
void maxInMemorySize(int byteCount);
/**
* Whether to log form data at DEBUG level, and headers at TRACE level.
* Both may contain sensitive information.

27
spring-web/src/main/java/org/springframework/http/codec/FormHttpMessageReader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,6 +30,7 @@ import reactor.core.publisher.Mono;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.http.MediaType;
@@ -62,6 +63,8 @@ public class FormHttpMessageReader extends LoggingCodecSupport
private Charset defaultCharset = DEFAULT_CHARSET;
private int maxInMemorySize = 256 * 1024;
/**
* Set the default character set to use for reading form data when the
@@ -80,6 +83,26 @@ public class FormHttpMessageReader extends LoggingCodecSupport
return this.defaultCharset;
}
/**
* Set the max number of bytes for input form data. As form data is buffered
* before it is parsed, this helps to limit the amount of buffering. Once
* the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default this is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canRead(ResolvableType elementType, @Nullable MediaType mediaType) {
@@ -105,7 +128,7 @@ public class FormHttpMessageReader extends LoggingCodecSupport
MediaType contentType = message.getHeaders().getContentType();
Charset charset = getMediaTypeCharset(contentType);
return DataBufferUtils.join(message.getBody())
return DataBufferUtils.join(message.getBody(), getMaxInMemorySize())
.map(buffer -> {
CharBuffer charBuffer = charset.decode(buffer.asByteBuffer());
String body = charBuffer.toString();

16
spring-web/src/main/java/org/springframework/http/codec/ServerCodecConfigurer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -76,6 +76,20 @@ public interface ServerCodecConfigurer extends CodecConfigurer {
*/
interface ServerDefaultCodecs extends DefaultCodecs {
/**
* Configure the {@code HttpMessageReader} to use for multipart requests.
* <p>By default, if
* <a href="https://github.com/synchronoss/nio-multipart">Synchronoss NIO Multipart</a>
* is present, this is set to
* {@link org.springframework.http.codec.multipart.MultipartHttpMessageReader
* MultipartHttpMessageReader} created with an instance of
* {@link org.springframework.http.codec.multipart.SynchronossPartHttpMessageReader
* SynchronossPartHttpMessageReader}.
* @param reader the message reader to use for multipart requests.
* @since 5.1.11
*/
void multipartReader(HttpMessageReader<?> reader);
/**
* Configure the {@code Encoder} to use for Server-Sent Events.
* <p>By default if this is not set, and Jackson is available, the

31
spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Decoder.java
View File

@@ -38,6 +38,7 @@ import org.springframework.core.codec.CodecException;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.http.codec.HttpMessageDecoder;
import org.springframework.http.server.reactive.ServerHttpRequest;
@@ -57,6 +58,9 @@ import org.springframework.util.MimeType;
*/
public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport implements HttpMessageDecoder<Object> {
private int maxInMemorySize = -1;
/**
* Until https://github.com/FasterXML/jackson-core/issues/476 is resolved,
* we need to ensure buffer recycling is off.
@@ -74,6 +78,29 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
}
/**
* Set the max number of bytes that can be buffered by this decoder. This
* is either the size of the entire input when decoding as a whole, or the
* size of one top-level JSON object within a JSON stream. When the limit
* is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default in 5.1 this is set to -1, unlimited. In 5.2 the default
* value for this limit is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {
JavaType javaType = getObjectMapper().getTypeFactory().constructType(elementType.getType());
@@ -87,7 +114,7 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(
Flux.from(input), this.jsonFactory, getObjectMapper(), true);
Flux.from(input), this.jsonFactory, getObjectMapper(), true, getMaxInMemorySize());
return decodeInternal(tokens, elementType, mimeType, hints);
}
@@ -96,7 +123,7 @@ public abstract class AbstractJackson2Decoder extends Jackson2CodecSupport imple
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
Flux<TokenBuffer> tokens = Jackson2Tokenizer.tokenize(
Flux.from(input), this.jsonFactory, getObjectMapper(), false);
Flux.from(input), this.jsonFactory, getObjectMapper(), false, getMaxInMemorySize());
return decodeInternal(tokens, elementType, mimeType, hints).singleOrEmpty();
}

51
spring-web/src/main/java/org/springframework/http/codec/json/Jackson2Tokenizer.java
View File

@@ -34,6 +34,7 @@ import reactor.core.publisher.Flux;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
/**
@@ -60,30 +61,39 @@ final class Jackson2Tokenizer {
private int arrayDepth;
private final int maxInMemorySize;
private int byteCount;
// TODO: change to ByteBufferFeeder when supported by Jackson
// See https://github.com/FasterXML/jackson-core/issues/478
private final ByteArrayFeeder inputFeeder;
private Jackson2Tokenizer(
JsonParser parser, DeserializationContext deserializationContext, boolean tokenizeArrayElements) {
private Jackson2Tokenizer(JsonParser parser, DeserializationContext deserializationContext,
boolean tokenizeArrayElements, int maxInMemorySize) {
this.parser = parser;
this.deserializationContext = deserializationContext;
this.tokenizeArrayElements = tokenizeArrayElements;
this.tokenBuffer = new TokenBuffer(parser, deserializationContext);
this.inputFeeder = (ByteArrayFeeder) this.parser.getNonBlockingInputFeeder();
this.maxInMemorySize = maxInMemorySize;
}
private Flux<TokenBuffer> tokenize(DataBuffer dataBuffer) {
int bufferSize = dataBuffer.readableByteCount();
byte[] bytes = new byte[dataBuffer.readableByteCount()];
dataBuffer.read(bytes);
DataBufferUtils.release(dataBuffer);
try {
this.inputFeeder.feedInput(bytes, 0, bytes.length);
return parseTokenBufferFlux();
List<TokenBuffer> result = parseTokenBufferFlux();
assertInMemorySize(bufferSize, result);
return Flux.fromIterable(result);
}
catch (JsonProcessingException ex) {
return Flux.error(new DecodingException("JSON decoding error: " + ex.getOriginalMessage(), ex));
@@ -96,7 +106,8 @@ final class Jackson2Tokenizer {
private Flux<TokenBuffer> endOfInput() {
this.inputFeeder.endOfInput();
try {
return parseTokenBufferFlux();
List<TokenBuffer> result = parseTokenBufferFlux();
return Flux.fromIterable(result);
}
catch (JsonProcessingException ex) {
return Flux.error(new DecodingException("JSON decoding error: " + ex.getOriginalMessage(), ex));
@@ -106,7 +117,7 @@ final class Jackson2Tokenizer {
}
}
private Flux<TokenBuffer> parseTokenBufferFlux() throws IOException {
private List<TokenBuffer> parseTokenBufferFlux() throws IOException {
List<TokenBuffer> result = new ArrayList<>();
while (true) {
@@ -124,7 +135,7 @@ final class Jackson2Tokenizer {
processTokenArray(token, result);
}
}
return Flux.fromIterable(result);
return result;
}
private void updateDepth(JsonToken token) {
@@ -171,18 +182,40 @@ final class Jackson2Tokenizer {
(token == JsonToken.END_ARRAY && this.arrayDepth == 0));
}
private void assertInMemorySize(int currentBufferSize, List<TokenBuffer> result) {
if (this.maxInMemorySize >= 0) {
if (!result.isEmpty()) {
this.byteCount = 0;
}
else if (currentBufferSize > Integer.MAX_VALUE - this.byteCount) {
raiseLimitException();
}
else {
this.byteCount += currentBufferSize;
if (this.byteCount > this.maxInMemorySize) {
raiseLimitException();
}
}
}
}
private void raiseLimitException() {
throw new DataBufferLimitException(
"Exceeded limit on max bytes per JSON object: " + this.maxInMemorySize);
}
/**
* Tokenize the given {@code Flux<DataBuffer>} into {@code Flux<TokenBuffer>}.
* @param dataBuffers the source data buffers
* @param jsonFactory the factory to use
* @param objectMapper the current mapper instance
* @param tokenizeArrayElements if {@code true} and the "top level" JSON object is
* @param tokenizeArrays if {@code true} and the "top level" JSON object is
* an array, each element is returned individually immediately after it is received
* @return the resulting token buffers
*/
public static Flux<TokenBuffer> tokenize(Flux<DataBuffer> dataBuffers, JsonFactory jsonFactory,
ObjectMapper objectMapper, boolean tokenizeArrayElements) {
ObjectMapper objectMapper, boolean tokenizeArrays, int maxInMemorySize) {
try {
JsonParser parser = jsonFactory.createNonBlockingByteArrayParser();
@@ -191,7 +224,7 @@ final class Jackson2Tokenizer {
context = ((DefaultDeserializationContext) context).createInstance(
objectMapper.getDeserializationConfig(), parser, objectMapper.getInjectableValues());
}
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrayElements);
Jackson2Tokenizer tokenizer = new Jackson2Tokenizer(parser, context, tokenizeArrays, maxInMemorySize);
return dataBuffers.flatMap(tokenizer::tokenize, Flux::error, tokenizer::endOfInput);
}
catch (IOException ex) {

8
spring-web/src/main/java/org/springframework/http/codec/multipart/MultipartHttpMessageReader.java
View File

@@ -65,6 +65,14 @@ public class MultipartHttpMessageReader extends LoggingCodecSupport
}
/**
* Return the configured parts reader.
* @since 5.1.11
*/
public HttpMessageReader<Part> getPartReader() {
return this.partReader;
}
@Override
public List<MediaType> getReadableMediaTypes() {
return Collections.singletonList(MediaType.MULTIPART_FORM_DATA);

287
spring-web/src/main/java/org/springframework/http/codec/multipart/SynchronossPartHttpMessageReader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,14 +40,18 @@ import org.synchronoss.cloud.nio.multipart.NioMultipartParser;
import org.synchronoss.cloud.nio.multipart.NioMultipartParserListener;
import org.synchronoss.cloud.nio.multipart.PartBodyStreamStorageFactory;
import org.synchronoss.cloud.nio.stream.storage.StreamStorage;
import reactor.core.publisher.BaseSubscriber;
import reactor.core.publisher.Flux;
import reactor.core.publisher.FluxSink;
import reactor.core.publisher.Mono;
import reactor.core.publisher.SignalType;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.core.log.LogFormatUtils;
@@ -69,15 +73,83 @@ import org.springframework.util.Assert;
* @author Sebastien Deleuze
* @author Rossen Stoyanchev
* @author Arjen Poutsma
* @author Brian Clozel
* @since 5.0
* @see <a href="https://github.com/synchronoss/nio-multipart">Synchronoss NIO Multipart</a>
* @see MultipartHttpMessageReader
*/
public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implements HttpMessageReader<Part> {
private final DataBufferFactory bufferFactory = new DefaultDataBufferFactory();
// Static DataBufferFactory to copy from FileInputStream or wrap bytes[].
private static final DataBufferFactory bufferFactory = new DefaultDataBufferFactory();
private final PartBodyStreamStorageFactory streamStorageFactory = new DefaultPartBodyStreamStorageFactory();
private int maxInMemorySize = -1;
private long maxDiskUsagePerPart = -1;
private long maxParts = -1;
/**
* Configure the maximum amount of memory that is allowed to use per part.
* When the limit is exceeded:
* <ul>
* <li>file parts are written to a temporary file.
* <li>non-file parts are rejected with {@link DataBufferLimitException}.
* </ul>
* <p>By default in 5.1 this is set to -1 in which case this limit is
* not enforced and all parts may be written to disk and are limited only
* by the {@link #setMaxDiskUsagePerPart(long) maxDiskUsagePerPart} property.
* In 5.2 this default value for this limit is set to 256K.
* @param byteCount the in-memory limit in bytes, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Get the {@link #setMaxInMemorySize configured} maximum in-memory size.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
/**
* Configure the maximum amount of disk space allowed for file parts.
* <p>By default this is set to -1.
* @param maxDiskUsagePerPart the disk limit in bytes, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxDiskUsagePerPart(long maxDiskUsagePerPart) {
this.maxDiskUsagePerPart = maxDiskUsagePerPart;
}
/**
* Get the {@link #setMaxDiskUsagePerPart configured} maximum disk usage.
* @since 5.1.11
*/
public long getMaxDiskUsagePerPart() {
return this.maxDiskUsagePerPart;
}
/**
* Specify the maximum number of parts allowed in a given multipart request.
* @since 5.1.11
*/
public void setMaxParts(long maxParts) {
this.maxParts = maxParts;
}
/**
* Return the {@link #setMaxParts configured} limit on the number of parts.
* @since 5.1.11
*/
public long getMaxParts() {
return this.maxParts;
}
@Override
@@ -94,7 +166,7 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
@Override
public Flux<Part> read(ResolvableType elementType, ReactiveHttpInputMessage message, Map<String, Object> hints) {
return Flux.create(new SynchronossPartGenerator(message, this.bufferFactory, this.streamStorageFactory))
return Flux.create(new SynchronossPartGenerator(message))
.doOnNext(part -> {
if (!Hints.isLoggingSuppressed(hints)) {
LogFormatUtils.traceDebug(logger, traceOn -> Hints.getLogPrefix(hints) + "Parsed " +
@@ -107,33 +179,36 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
@Override
public Mono<Part> readMono(ResolvableType elementType, ReactiveHttpInputMessage message, Map<String, Object> hints) {
return Mono.error(new UnsupportedOperationException("Cannot read multipart request body into single Part"));
public Mono<Part> readMono(
ResolvableType elementType, ReactiveHttpInputMessage message, Map<String, Object> hints) {
return Mono.error(new UnsupportedOperationException(
"Cannot read multipart request body into single Part"));
}
/**
* Consume and feed input to the Synchronoss parser, then listen for parser
* output events and adapt to {@code Flux<Sink<Part>>}.
* Subscribe to the input stream and feed the Synchronoss parser. Then listen
* for parser output, creating parts, and pushing them into the FluxSink.
*/
private static class SynchronossPartGenerator implements Consumer<FluxSink<Part>> {
private class SynchronossPartGenerator extends BaseSubscriber<DataBuffer> implements Consumer<FluxSink<Part>> {
private final ReactiveHttpInputMessage inputMessage;
private final DataBufferFactory bufferFactory;
private final LimitedPartBodyStreamStorageFactory storageFactory = new LimitedPartBodyStreamStorageFactory();
private final PartBodyStreamStorageFactory streamStorageFactory;
private NioMultipartParserListener listener;
SynchronossPartGenerator(ReactiveHttpInputMessage inputMessage, DataBufferFactory bufferFactory,
PartBodyStreamStorageFactory streamStorageFactory) {
private NioMultipartParser parser;
public SynchronossPartGenerator(ReactiveHttpInputMessage inputMessage) {
this.inputMessage = inputMessage;
this.bufferFactory = bufferFactory;
this.streamStorageFactory = streamStorageFactory;
}
@Override
public void accept(FluxSink<Part> emitter) {
public void accept(FluxSink<Part> sink) {
HttpHeaders headers = this.inputMessage.getHeaders();
MediaType mediaType = headers.getContentType();
Assert.state(mediaType != null, "No content type set");
@@ -142,40 +217,57 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
Charset charset = Optional.ofNullable(mediaType.getCharset()).orElse(StandardCharsets.UTF_8);
MultipartContext context = new MultipartContext(mediaType.toString(), length, charset.name());
NioMultipartParserListener listener = new FluxSinkAdapterListener(emitter, this.bufferFactory, context);
NioMultipartParser parser = Multipart
.multipart(context)
.usePartBodyStreamStorageFactory(this.streamStorageFactory)
.forNIO(listener);
this.listener = new FluxSinkAdapterListener(sink, context, this.storageFactory);
this.inputMessage.getBody().subscribe(buffer -> {
byte[] resultBytes = new byte[buffer.readableByteCount()];
buffer.read(resultBytes);
try {
parser.write(resultBytes);
}
catch (IOException ex) {
listener.onError("Exception thrown providing input to the parser", ex);
}
finally {
DataBufferUtils.release(buffer);
}
}, ex -> {
try {
listener.onError("Request body input error", ex);
parser.close();
}
catch (IOException ex2) {
listener.onError("Exception thrown while closing the parser", ex2);
}
}, () -> {
try {
parser.close();
}
catch (IOException ex) {
listener.onError("Exception thrown while closing the parser", ex);
}
});
this.parser = Multipart
.multipart(context)
.usePartBodyStreamStorageFactory(this.storageFactory)
.forNIO(this.listener);
this.inputMessage.getBody().subscribe(this);
}
@Override
protected void hookOnNext(DataBuffer buffer) {
int size = buffer.readableByteCount();
this.storageFactory.increaseByteCount(size);
byte[] resultBytes = new byte[size];
buffer.read(resultBytes);
try {
this.parser.write(resultBytes);
}
catch (IOException ex) {
cancel();
int index = this.storageFactory.getCurrentPartIndex();
this.listener.onError("Parser error for part [" + index + "]", ex);
}
finally {
DataBufferUtils.release(buffer);
}
}
@Override
protected void hookOnError(Throwable ex) {
try {
this.parser.close();
}
catch (IOException ex2) {
// ignore
}
finally {
int index = this.storageFactory.getCurrentPartIndex();
this.listener.onError("Failure while parsing part[" + index + "]", ex);
}
}
@Override
protected void hookFinally(SignalType type) {
try {
this.parser.close();
}
catch (IOException ex) {
this.listener.onError("Error while closing parser", ex);
}
}
private int getContentLength(HttpHeaders headers) {
@@ -186,6 +278,54 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
}
private class LimitedPartBodyStreamStorageFactory implements PartBodyStreamStorageFactory {
private final PartBodyStreamStorageFactory storageFactory = maxInMemorySize > 0 ?
new DefaultPartBodyStreamStorageFactory(maxInMemorySize) :
new DefaultPartBodyStreamStorageFactory();
private int index = 1;
private boolean isFilePart;
private long partSize;
public int getCurrentPartIndex() {
return this.index;
}
@Override
public StreamStorage newStreamStorageForPartBody(Map<String, List<String>> headers, int index) {
this.index = index;
this.isFilePart = (MultipartUtils.getFileName(headers) != null);
this.partSize = 0;
if (maxParts > 0 && index > maxParts) {
throw new DecodingException("Too many parts (" + index + " allowed)");
}
return this.storageFactory.newStreamStorageForPartBody(headers, index);
}
public void increaseByteCount(long byteCount) {
this.partSize += byteCount;
if (maxInMemorySize > 0 && !this.isFilePart && this.partSize >= maxInMemorySize) {
throw new DataBufferLimitException("Part[" + this.index + "] " +
"exceeded the in-memory limit of " + maxInMemorySize + " bytes");
}
if (maxDiskUsagePerPart > 0 && this.isFilePart && this.partSize > maxDiskUsagePerPart) {
throw new DecodingException("Part[" + this.index + "] " +
"exceeded the disk usage limit of " + maxDiskUsagePerPart + " bytes");
}
}
public void partFinished() {
this.index++;
this.isFilePart = false;
this.partSize = 0;
}
}
/**
* Listen for parser output and adapt to {@code Flux<Sink<Part>>}.
*/
@@ -193,43 +333,48 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
private final FluxSink<Part> sink;
private final DataBufferFactory bufferFactory;
private final MultipartContext context;
private final LimitedPartBodyStreamStorageFactory storageFactory;
private final AtomicInteger terminated = new AtomicInteger(0);
FluxSinkAdapterListener(FluxSink<Part> sink, DataBufferFactory factory, MultipartContext context) {
FluxSinkAdapterListener(
FluxSink<Part> sink, MultipartContext context, LimitedPartBodyStreamStorageFactory factory) {
this.sink = sink;
this.bufferFactory = factory;
this.context = context;
this.storageFactory = factory;
}
@Override
public void onPartFinished(StreamStorage storage, Map<String, List<String>> headers) {
HttpHeaders httpHeaders = new HttpHeaders();
httpHeaders.putAll(headers);
this.storageFactory.partFinished();
this.sink.next(createPart(storage, httpHeaders));
}
private Part createPart(StreamStorage storage, HttpHeaders httpHeaders) {
String filename = MultipartUtils.getFileName(httpHeaders);
if (filename != null) {
return new SynchronossFilePart(httpHeaders, filename, storage, this.bufferFactory);
return new SynchronossFilePart(httpHeaders, filename, storage);
}
else if (MultipartUtils.isFormField(httpHeaders, this.context)) {
String value = MultipartUtils.readFormParameterValue(storage, httpHeaders);
return new SynchronossFormFieldPart(httpHeaders, this.bufferFactory, value);
return new SynchronossFormFieldPart(httpHeaders, value);
}
else {
return new SynchronossPart(httpHeaders, storage, this.bufferFactory);
return new SynchronossPart(httpHeaders, storage);
}
}
@Override
public void onError(String message, Throwable cause) {
if (this.terminated.getAndIncrement() == 0) {
this.sink.error(new RuntimeException(message, cause));
this.sink.error(new DecodingException(message, cause));
}
}
@@ -256,14 +401,10 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
private final HttpHeaders headers;
private final DataBufferFactory bufferFactory;
AbstractSynchronossPart(HttpHeaders headers, DataBufferFactory bufferFactory) {
AbstractSynchronossPart(HttpHeaders headers) {
Assert.notNull(headers, "HttpHeaders is required");
Assert.notNull(bufferFactory, "DataBufferFactory is required");
this.name = MultipartUtils.getFieldName(headers);
this.headers = headers;
this.bufferFactory = bufferFactory;
}
@Override
@@ -276,10 +417,6 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
return this.headers;
}
DataBufferFactory getBufferFactory() {
return this.bufferFactory;
}
@Override
public String toString() {
return "Part '" + this.name + "', headers=" + this.headers;
@@ -291,15 +428,15 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
private final StreamStorage storage;
SynchronossPart(HttpHeaders headers, StreamStorage storage, DataBufferFactory factory) {
super(headers, factory);
SynchronossPart(HttpHeaders headers, StreamStorage storage) {
super(headers);
Assert.notNull(storage, "StreamStorage is required");
this.storage = storage;
}
@Override
public Flux<DataBuffer> content() {
return DataBufferUtils.readInputStream(getStorage()::getInputStream, getBufferFactory(), 4096);
return DataBufferUtils.readInputStream(getStorage()::getInputStream, bufferFactory, 4096);
}
protected StreamStorage getStorage() {
@@ -315,8 +452,8 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
private final String filename;
SynchronossFilePart(HttpHeaders headers, String filename, StreamStorage storage, DataBufferFactory factory) {
super(headers, storage, factory);
SynchronossFilePart(HttpHeaders headers, String filename, StreamStorage storage) {
super(headers, storage);
this.filename = filename;
}
@@ -375,8 +512,8 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
private final String content;
SynchronossFormFieldPart(HttpHeaders headers, DataBufferFactory bufferFactory, String content) {
super(headers, bufferFactory);
SynchronossFormFieldPart(HttpHeaders headers, String content) {
super(headers);
this.content = content;
}
@@ -388,9 +525,7 @@ public class SynchronossPartHttpMessageReader extends LoggingCodecSupport implem
@Override
public Flux<DataBuffer> content() {
byte[] bytes = this.content.getBytes(getCharset());
DataBuffer buffer = getBufferFactory().allocateBuffer(bytes.length);
buffer.write(bytes);
return Flux.just(buffer);
return Flux.just(bufferFactory.wrap(bytes));
}
private Charset getCharset() {

23
spring-web/src/main/java/org/springframework/http/codec/protobuf/ProtobufDecoder.java
View File

@@ -36,6 +36,7 @@ import org.springframework.core.ResolvableType;
import org.springframework.core.codec.Decoder;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
@@ -101,10 +102,24 @@ public class ProtobufDecoder extends ProtobufCodecSupport implements Decoder<Mes
}
/**
* The max size allowed per message.
* <p>By default in 5.1 this is set to 64K. In 5.2 the default for this limit
* is set to 256K.
* @param maxMessageSize the max size per message, or -1 for unlimited
*/
public void setMaxMessageSize(int maxMessageSize) {
this.maxMessageSize = maxMessageSize;
}
/**
* Return the {@link #setMaxMessageSize configured} message size limit.
* @since 5.1.11
*/
public int getMaxMessageSize() {
return this.maxMessageSize;
}
@Override
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {
@@ -127,7 +142,7 @@ public class ProtobufDecoder extends ProtobufCodecSupport implements Decoder<Mes
public Mono<Message> decodeToMono(Publisher<DataBuffer> inputStream, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
return DataBufferUtils.join(inputStream).map(dataBuffer -> {
return DataBufferUtils.join(inputStream, getMaxMessageSize()).map(dataBuffer -> {
try {
Message.Builder builder = getMessageBuilder(elementType.toClass());
ByteBuffer buffer = dataBuffer.asByteBuffer();
@@ -198,9 +213,9 @@ public class ProtobufDecoder extends ProtobufCodecSupport implements Decoder<Mes
if (!readMessageSize(input)) {
return messages;
}
if (this.messageBytesToRead > this.maxMessageSize) {
throw new DecodingException(
"The number of bytes to read from the incoming stream " +
if (this.maxMessageSize > 0 && this.messageBytesToRead > this.maxMessageSize) {
throw new DataBufferLimitException(
"The number of bytes to read for message " +
"(" + this.messageBytesToRead + ") exceeds " +
"the configured limit (" + this.maxMessageSize + ")");
}

62
spring-web/src/main/java/org/springframework/http/codec/support/BaseDefaultCodecs.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.springframework.core.codec.AbstractDataBufferDecoder;
import org.springframework.core.codec.ByteArrayDecoder;
import org.springframework.core.codec.ByteArrayEncoder;
import org.springframework.core.codec.ByteBufferDecoder;
@@ -38,6 +39,7 @@ import org.springframework.http.codec.FormHttpMessageReader;
import org.springframework.http.codec.HttpMessageReader;
import org.springframework.http.codec.HttpMessageWriter;
import org.springframework.http.codec.ResourceHttpMessageWriter;
import org.springframework.http.codec.json.AbstractJackson2Decoder;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.http.codec.json.Jackson2JsonEncoder;
import org.springframework.http.codec.json.Jackson2SmileDecoder;
@@ -95,6 +97,9 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
@Nullable
private Encoder<?> jaxb2Encoder;
@Nullable
private Integer maxInMemorySize;
private boolean enableLoggingRequestDetails = false;
private boolean registerDefaults = true;
@@ -130,6 +135,16 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
this.jaxb2Encoder = encoder;
}
@Override
public void maxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
@Nullable
protected Integer maxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public void enableLoggingRequestDetails(boolean enable) {
this.enableLoggingRequestDetails = enable;
@@ -155,17 +170,20 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
return Collections.emptyList();
}
List<HttpMessageReader<?>> readers = new ArrayList<>();
readers.add(new DecoderHttpMessageReader<>(new ByteArrayDecoder()));
readers.add(new DecoderHttpMessageReader<>(new ByteBufferDecoder()));
readers.add(new DecoderHttpMessageReader<>(new DataBufferDecoder()));
readers.add(new DecoderHttpMessageReader<>(new ResourceDecoder()));
readers.add(new DecoderHttpMessageReader<>(StringDecoder.textPlainOnly()));
readers.add(new DecoderHttpMessageReader<>(init(new ByteArrayDecoder())));
readers.add(new DecoderHttpMessageReader<>(init(new ByteBufferDecoder())));
readers.add(new DecoderHttpMessageReader<>(init(new DataBufferDecoder())));
readers.add(new DecoderHttpMessageReader<>(init(new ResourceDecoder())));
readers.add(new DecoderHttpMessageReader<>(init(StringDecoder.textPlainOnly())));
if (protobufPresent) {
Decoder<?> decoder = this.protobufDecoder != null ? this.protobufDecoder : new ProtobufDecoder();
Decoder<?> decoder = this.protobufDecoder != null ? this.protobufDecoder : init(new ProtobufDecoder());
readers.add(new DecoderHttpMessageReader<>(decoder));
}
FormHttpMessageReader formReader = new FormHttpMessageReader();
if (this.maxInMemorySize != null) {
formReader.setMaxInMemorySize(this.maxInMemorySize);
}
formReader.setEnableLoggingRequestDetails(this.enableLoggingRequestDetails);
readers.add(formReader);
@@ -174,6 +192,28 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
return readers;
}
private <T extends Decoder<?>> T init(T decoder) {
if (this.maxInMemorySize != null) {
if (decoder instanceof AbstractDataBufferDecoder) {
((AbstractDataBufferDecoder<?>) decoder).setMaxInMemorySize(this.maxInMemorySize);
}
if (decoder instanceof ProtobufDecoder) {
((ProtobufDecoder) decoder).setMaxMessageSize(this.maxInMemorySize);
}
if (jackson2Present) {
if (decoder instanceof AbstractJackson2Decoder) {
((AbstractJackson2Decoder) decoder).setMaxInMemorySize(this.maxInMemorySize);
}
}
if (jaxb2Present) {
if (decoder instanceof Jaxb2XmlDecoder) {
((Jaxb2XmlDecoder) decoder).setMaxInMemorySize(this.maxInMemorySize);
}
}
}
return decoder;
}
/**
* Hook for client or server specific typed readers.
*/
@@ -189,13 +229,13 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
}
List<HttpMessageReader<?>> readers = new ArrayList<>();
if (jackson2Present) {
readers.add(new DecoderHttpMessageReader<>(getJackson2JsonDecoder()));
readers.add(new DecoderHttpMessageReader<>(init(getJackson2JsonDecoder())));
}
if (jackson2SmilePresent) {
readers.add(new DecoderHttpMessageReader<>(new Jackson2SmileDecoder()));
readers.add(new DecoderHttpMessageReader<>(init(new Jackson2SmileDecoder())));
}
if (jaxb2Present) {
Decoder<?> decoder = this.jaxb2Decoder != null ? this.jaxb2Decoder : new Jaxb2XmlDecoder();
Decoder<?> decoder = this.jaxb2Decoder != null ? this.jaxb2Decoder : init(new Jaxb2XmlDecoder());
readers.add(new DecoderHttpMessageReader<>(decoder));
}
extendObjectReaders(readers);
@@ -216,7 +256,7 @@ class BaseDefaultCodecs implements CodecConfigurer.DefaultCodecs {
return Collections.emptyList();
}
List<HttpMessageReader<?>> result = new ArrayList<>();
result.add(new DecoderHttpMessageReader<>(StringDecoder.allMimeTypes()));
result.add(new DecoderHttpMessageReader<>(init(StringDecoder.allMimeTypes())));
return result;
}

18
spring-web/src/main/java/org/springframework/http/codec/support/ServerDefaultCodecsImpl.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -39,10 +39,18 @@ class ServerDefaultCodecsImpl extends BaseDefaultCodecs implements ServerCodecCo
DefaultServerCodecConfigurer.class.getClassLoader());
@Nullable
private HttpMessageReader<?> multipartReader;
@Nullable
private Encoder<?> sseEncoder;
@Override
public void multipartReader(HttpMessageReader<?> reader) {
this.multipartReader = reader;
}
@Override
public void serverSentEventEncoder(Encoder<?> encoder) {
this.sseEncoder = encoder;
@@ -51,10 +59,18 @@ class ServerDefaultCodecsImpl extends BaseDefaultCodecs implements ServerCodecCo
@Override
protected void extendTypedReaders(List<HttpMessageReader<?>> typedReaders) {
if (this.multipartReader != null) {
typedReaders.add(this.multipartReader);
return;
}
if (synchronossMultipartPresent) {
boolean enable = isEnableLoggingRequestDetails();
SynchronossPartHttpMessageReader partReader = new SynchronossPartHttpMessageReader();
Integer size = maxInMemorySize();
if (size != null) {
partReader.setMaxInMemorySize(size);
}
partReader.setEnableLoggingRequestDetails(enable);
typedReaders.add(partReader);

26
spring-web/src/main/java/org/springframework/http/codec/xml/Jaxb2XmlDecoder.java
View File

@@ -43,6 +43,7 @@ import org.springframework.core.codec.CodecException;
import org.springframework.core.codec.DecodingException;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
@@ -78,6 +79,8 @@ public class Jaxb2XmlDecoder extends AbstractDecoder<Object> {
private Function<Unmarshaller, Unmarshaller> unmarshallerProcessor = Function.identity();
private int maxInMemorySize = -1;
public Jaxb2XmlDecoder() {
super(MimeTypeUtils.APPLICATION_XML, MimeTypeUtils.TEXT_XML);
@@ -110,6 +113,29 @@ public class Jaxb2XmlDecoder extends AbstractDecoder<Object> {
return this.unmarshallerProcessor;
}
/**
* Set the max number of bytes that can be buffered by this decoder.
* This is either the size of the entire input when decoding as a whole, or when
* using async parsing with Aalto XML, it is the size of one top-level XML tree.
* When the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default in 5.1 this is set to -1, unlimited. In 5.2 the default
* value for this limit is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
this.xmlEventDecoder.setMaxInMemorySize(byteCount);
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
public boolean canDecode(ResolvableType elementType, @Nullable MimeType mimeType) {

77
spring-web/src/main/java/org/springframework/http/codec/xml/XmlEventDecoder.java
View File

@@ -40,6 +40,7 @@ import reactor.core.publisher.Flux;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.AbstractDecoder;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferLimitException;
import org.springframework.core.io.buffer.DataBufferUtils;
import org.springframework.lang.Nullable;
import org.springframework.util.ClassUtils;
@@ -88,26 +89,51 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
boolean useAalto = aaltoPresent;
private int maxInMemorySize = -1;
public XmlEventDecoder() {
super(MimeTypeUtils.APPLICATION_XML, MimeTypeUtils.TEXT_XML);
}
/**
* Set the max number of bytes that can be buffered by this decoder. This
* is either the size the entire input when decoding as a whole, or when
* using async parsing via Aalto XML, it is size one top-level XML tree.
* When the limit is exceeded, {@link DataBufferLimitException} is raised.
* <p>By default in 5.1 this is set to -1, unlimited. In 5.2 the default
* value for this limit is set to 256K.
* @param byteCount the max number of bytes to buffer, or -1 for unlimited
* @since 5.1.11
*/
public void setMaxInMemorySize(int byteCount) {
this.maxInMemorySize = byteCount;
}
/**
* Return the {@link #setMaxInMemorySize configured} byte count limit.
* @since 5.1.11
*/
public int getMaxInMemorySize() {
return this.maxInMemorySize;
}
@Override
@SuppressWarnings({"rawtypes", "unchecked"}) // on JDK 9 where XMLEventReader is Iterator<Object>
public Flux<XMLEvent> decode(Publisher<DataBuffer> input, ResolvableType elementType,
@Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
if (this.useAalto) {
AaltoDataBufferToXmlEvent mapper = new AaltoDataBufferToXmlEvent();
AaltoDataBufferToXmlEvent mapper = new AaltoDataBufferToXmlEvent(this.maxInMemorySize);
return Flux.from(input)
.flatMapIterable(mapper)
.doFinally(signalType -> mapper.endOfInput());
}
else {
return DataBufferUtils.join(input).
flatMapIterable(buffer -> {
return DataBufferUtils.join(input, getMaxInMemorySize())
.flatMapIterable(buffer -> {
try {
InputStream is = buffer.asInputStream();
Iterator eventReader = inputFactory.createXMLEventReader(is);
@@ -139,10 +165,22 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
private final XMLEventAllocator eventAllocator = EventAllocatorImpl.getDefaultInstance();
private final int maxInMemorySize;
private int byteCount;
private int elementDepth;
public AaltoDataBufferToXmlEvent(int maxInMemorySize) {
this.maxInMemorySize = maxInMemorySize;
}
@Override
public List<? extends XMLEvent> apply(DataBuffer dataBuffer) {
try {
increaseByteCount(dataBuffer);
this.streamReader.getInputFeeder().feedInput(dataBuffer.asByteBuffer());
List<XMLEvent> events = new ArrayList<>();
while (true) {
@@ -156,8 +194,12 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
if (event.isEndDocument()) {
break;
}
checkDepthAndResetByteCount(event);
}
}
if (this.maxInMemorySize > 0 && this.byteCount > this.maxInMemorySize) {
raiseLimitException();
}
return events;
}
catch (XMLStreamException ex) {
@@ -168,6 +210,35 @@ public class XmlEventDecoder extends AbstractDecoder<XMLEvent> {
}
}
private void increaseByteCount(DataBuffer dataBuffer) {
if (this.maxInMemorySize > 0) {
if (dataBuffer.readableByteCount() > Integer.MAX_VALUE - this.byteCount) {
raiseLimitException();
}
else {
this.byteCount += dataBuffer.readableByteCount();
}
}
}
private void checkDepthAndResetByteCount(XMLEvent event) {
if (this.maxInMemorySize > 0) {
if (event.isStartElement()) {
this.byteCount = this.elementDepth == 1 ? 0 : this.byteCount;
this.elementDepth++;
}
else if (event.isEndElement()) {
this.elementDepth--;
this.byteCount = this.elementDepth == 1 ? 0 : this.byteCount;
}
}
}
private void raiseLimitException() {
throw new DataBufferLimitException(
"Exceeded limit on max bytes per XML top-level node: " + this.maxInMemorySize);
}
public void endOfInput() {
this.streamReader.getInputFeeder().endOfInput();
}