Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Buffer leak fixes

Browse Source 
Address issues where buffers are allocated (and cached somehow) at or
before subscription, and before explicit demand.

The commit adds tests proving the leaks and fixes. The common thread
for all tests is a "zero demand" subscriber that subscribes  but does
not request, and then cancels without consuming anything.

Closes gh-22107
This commit is contained in:
Rossen Stoyanchev
2019-03-26 21:11:19 -04:00
parent 65b46079a2
commit c54355784e
16 changed files with 504 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
spring-web/src/main/java/org/springframework/http/codec/EncoderHttpMessageWriter.java
View File

@@ -29,6 +29,7 @@ import org.springframework.core.codec.AbstractEncoder;
import org.springframework.core.codec.Encoder;
import org.springframework.core.codec.Hints;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.PooledDataBuffer;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpLogging;
import org.springframework.http.MediaType;
@@ -124,7 +125,8 @@ public class EncoderHttpMessageWriter<T> implements HttpMessageWriter<T> {
}))
.flatMap(buffer -> {
headers.setContentLength(buffer.readableByteCount());
return message.writeWith(Mono.just(buffer));
return message.writeWith(Mono.fromCallable(() -> buffer)
.doOnDiscard(PooledDataBuffer.class, PooledDataBuffer::release));
});
}

4
spring-web/src/main/java/org/springframework/http/codec/FormHttpMessageWriter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -134,7 +134,7 @@ public class FormHttpMessageWriter extends LoggingCodecSupport
logFormData(form, hints);
String value = serializeForm(form, charset);
ByteBuffer byteBuffer = charset.encode(value);
DataBuffer buffer = message.bufferFactory().wrap(byteBuffer);
DataBuffer buffer = message.bufferFactory().wrap(byteBuffer); // wrapping only, no allocation
message.getHeaders().setContentLength(byteBuffer.remaining());
return message.writeWith(Mono.just(buffer));
});

33
spring-web/src/main/java/org/springframework/http/codec/ServerSentEventHttpMessageWriter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -143,30 +143,32 @@ public class ServerSentEventHttpMessageWriter implements HttpMessageWriter<Objec
sb.append("data:");
}
return Flux.concat(encodeText(sb, mediaType, factory),
Flux<DataBuffer> flux = Flux.concat(
encodeText(sb, mediaType, factory),
encodeData(data, valueType, mediaType, factory, hints),
encodeText("\n", mediaType, factory))
.doOnDiscard(PooledDataBuffer.class, DataBufferUtils::release);
encodeText("\n", mediaType, factory));
return flux.doOnDiscard(PooledDataBuffer.class, DataBufferUtils::release);
});
}
private void writeField(String fieldName, Object fieldValue, StringBuilder stringBuilder) {
stringBuilder.append(fieldName);
stringBuilder.append(':');
stringBuilder.append(fieldValue.toString());
stringBuilder.append("\n");
private void writeField(String fieldName, Object fieldValue, StringBuilder sb) {
sb.append(fieldName);
sb.append(':');
sb.append(fieldValue.toString());
sb.append("\n");
}
@SuppressWarnings("unchecked")
private <T> Flux<DataBuffer> encodeData(@Nullable T data, ResolvableType valueType,
private <T> Flux<DataBuffer> encodeData(@Nullable T dataValue, ResolvableType valueType,
MediaType mediaType, DataBufferFactory factory, Map<String, Object> hints) {
if (data == null) {
if (dataValue == null) {
return Flux.empty();
}
if (data instanceof String) {
String text = (String) data;
if (dataValue instanceof String) {
String text = (String) dataValue;
return Flux.from(encodeText(StringUtils.replace(text, "\n", "\ndata:") + "\n", mediaType, factory));
}
@@ -175,15 +177,14 @@ public class ServerSentEventHttpMessageWriter implements HttpMessageWriter<Objec
}
return ((Encoder<T>) this.encoder)
.encode(Mono.just(data), factory, valueType, mediaType, hints)
.encode(Mono.just(dataValue), factory, valueType, mediaType, hints)
.concatWith(encodeText("\n", mediaType, factory));
}
private Mono<DataBuffer> encodeText(CharSequence text, MediaType mediaType, DataBufferFactory bufferFactory) {
Assert.notNull(mediaType.getCharset(), "Expected MediaType with charset");
byte[] bytes = text.toString().getBytes(mediaType.getCharset());
return Mono.defer(() ->
Mono.just(bufferFactory.allocateBuffer(bytes.length).write(bytes)));
return Mono.fromCallable(() -> bufferFactory.wrap(bytes)); // wrapping, not allocating
}
@Override

19
spring-web/src/main/java/org/springframework/http/codec/json/AbstractJackson2Encoder.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -126,12 +126,10 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
.filter(mediaType -> mediaType.isCompatibleWith(mimeType))
.findFirst()
.map(mediaType -> {
byte[] separator =
STREAM_SEPARATORS.getOrDefault(mediaType, NEWLINE_SEPARATOR);
byte[] separator = STREAM_SEPARATORS.getOrDefault(mediaType, NEWLINE_SEPARATOR);
return Flux.from(inputStream).map(value -> {
DataBuffer buffer =
encodeValue(value, mimeType, bufferFactory, elementType, hints,
encoding);
DataBuffer buffer = encodeValue(
value, mimeType, bufferFactory, elementType, hints, encoding);
if (separator != null) {
buffer.write(separator);
}
@@ -139,11 +137,9 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
});
})
.orElseGet(() -> {
ResolvableType listType =
ResolvableType.forClassWithGenerics(List.class, elementType);
ResolvableType listType = ResolvableType.forClassWithGenerics(List.class, elementType);
return Flux.from(inputStream).collectList().map(list ->
encodeValue(list, mimeType, bufferFactory, listType, hints,
encoding)).flux();
encodeValue(list, mimeType, bufferFactory, listType, hints, encoding)).flux();
});
}
}
@@ -174,8 +170,7 @@ public abstract class AbstractJackson2Encoder extends Jackson2CodecSupport imple
OutputStream outputStream = buffer.asOutputStream();
try {
JsonGenerator generator =
getObjectMapper().getFactory().createGenerator(outputStream, encoding);
JsonGenerator generator = getObjectMapper().getFactory().createGenerator(outputStream, encoding);
writer.writeValue(generator, value);
release = false;
}

138
spring-web/src/main/java/org/springframework/http/codec/multipart/MultipartHttpMessageWriter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ import org.springframework.core.codec.Hints;
import org.springframework.core.io.Resource;
import org.springframework.core.io.buffer.DataBuffer;
import org.springframework.core.io.buffer.DataBufferFactory;
import org.springframework.core.io.buffer.DefaultDataBufferFactory;
import org.springframework.core.io.buffer.PooledDataBuffer;
import org.springframework.core.log.LogFormatUtils;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
@@ -99,8 +99,6 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
private final List<MediaType> supportedMediaTypes;
private final DataBufferFactory bufferFactory = new DefaultDataBufferFactory();
/**
* Constructor with a default list of part writers (String and Resource).
@@ -187,17 +185,17 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
ResolvableType elementType, @Nullable MediaType mediaType, ReactiveHttpOutputMessage outputMessage,
Map<String, Object> hints) {
return Mono.from(inputStream).flatMap(map -> {
if (this.formWriter == null || isMultipart(map, mediaType)) {
return writeMultipart(map, outputMessage, hints);
}
else {
@SuppressWarnings("unchecked")
MultiValueMap<String, String> formData = (MultiValueMap<String, String>) map;
return this.formWriter.write(Mono.just(formData), elementType, mediaType, outputMessage, hints);
}
});
return Mono.from(inputStream)
.flatMap(map -> {
if (this.formWriter == null || isMultipart(map, mediaType)) {
return writeMultipart(map, outputMessage, hints);
}
else {
@SuppressWarnings("unchecked")
Mono<MultiValueMap<String, String>> input = Mono.just((MultiValueMap<String, String>) map);
return this.formWriter.write(input, elementType, mediaType, outputMessage, hints);
}
});
}
private boolean isMultipart(MultiValueMap<String, ?> map, @Nullable MediaType contentType) {
@@ -230,9 +228,12 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
LogFormatUtils.formatValue(map, !traceOn) :
"parts " + map.keySet() + " (content masked)"));
DataBufferFactory bufferFactory = outputMessage.bufferFactory();
Flux<DataBuffer> body = Flux.fromIterable(map.entrySet())
.concatMap(entry -> encodePartValues(boundary, entry.getKey(), entry.getValue()))
.concatWith(Mono.just(generateLastLine(boundary)));
.concatMap(entry -> encodePartValues(boundary, entry.getKey(), entry.getValue(), bufferFactory))
.concatWith(generateLastLine(boundary, bufferFactory))
.doOnDiscard(PooledDataBuffer.class, PooledDataBuffer::release);
return outputMessage.writeWith(body);
}
@@ -245,14 +246,16 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
return MimeTypeUtils.generateMultipartBoundary();
}
private Flux<DataBuffer> encodePartValues(byte[] boundary, String name, List<?> values) {
private Flux<DataBuffer> encodePartValues(
byte[] boundary, String name, List<?> values, DataBufferFactory bufferFactory) {
return Flux.concat(values.stream().map(v ->
encodePart(boundary, name, v)).collect(Collectors.toList()));
encodePart(boundary, name, v, bufferFactory)).collect(Collectors.toList()));
}
@SuppressWarnings("unchecked")
private <T> Flux<DataBuffer> encodePart(byte[] boundary, String name, T value) {
MultipartHttpOutputMessage outputMessage = new MultipartHttpOutputMessage(this.bufferFactory, getCharset());
private <T> Flux<DataBuffer> encodePart(byte[] boundary, String name, T value, DataBufferFactory bufferFactory) {
MultipartHttpOutputMessage outputMessage = new MultipartHttpOutputMessage(bufferFactory, getCharset());
HttpHeaders outputHeaders = outputMessage.getHeaders();
T body;
@@ -314,37 +317,46 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
Flux<DataBuffer> partContent = partContentReady.thenMany(Flux.defer(outputMessage::getBody));
return Flux.concat(Mono.just(generateBoundaryLine(boundary)), partContent, Mono.just(generateNewLine()));
return Flux.concat(
generateBoundaryLine(boundary, bufferFactory),
partContent,
generateNewLine(bufferFactory));
}
private DataBuffer generateBoundaryLine(byte[] boundary) {
DataBuffer buffer = this.bufferFactory.allocateBuffer(boundary.length + 4);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write(boundary);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
private Mono<DataBuffer> generateBoundaryLine(byte[] boundary, DataBufferFactory bufferFactory) {
return Mono.fromCallable(() -> {
DataBuffer buffer = bufferFactory.allocateBuffer(boundary.length + 4);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write(boundary);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
});
}
private DataBuffer generateNewLine() {
DataBuffer buffer = this.bufferFactory.allocateBuffer(2);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
private Mono<DataBuffer> generateNewLine(DataBufferFactory bufferFactory) {
return Mono.fromCallable(() -> {
DataBuffer buffer = bufferFactory.allocateBuffer(2);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
});
}
private DataBuffer generateLastLine(byte[] boundary) {
DataBuffer buffer = this.bufferFactory.allocateBuffer(boundary.length + 6);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write(boundary);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
private Mono<DataBuffer> generateLastLine(byte[] boundary, DataBufferFactory bufferFactory) {
return Mono.fromCallable(() -> {
DataBuffer buffer = bufferFactory.allocateBuffer(boundary.length + 6);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write(boundary);
buffer.write((byte)'-');
buffer.write((byte)'-');
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
});
}
@@ -391,29 +403,31 @@ public class MultipartHttpMessageWriter extends LoggingCodecSupport
if (this.body != null) {
return Mono.error(new IllegalStateException("Multiple calls to writeWith() not supported"));
}
this.body = Flux.just(generateHeaders()).concatWith(body);
this.body = generateHeaders().concatWith(body);
// We don't actually want to write (just save the body Flux)
return Mono.empty();
}
private DataBuffer generateHeaders() {
DataBuffer buffer = this.bufferFactory.allocateBuffer();
for (Map.Entry<String, List<String>> entry : this.headers.entrySet()) {
byte[] headerName = entry.getKey().getBytes(this.charset);
for (String headerValueString : entry.getValue()) {
byte[] headerValue = headerValueString.getBytes(this.charset);
buffer.write(headerName);
buffer.write((byte)':');
buffer.write((byte)' ');
buffer.write(headerValue);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
private Mono<DataBuffer> generateHeaders() {
return Mono.fromCallable(() -> {
DataBuffer buffer = this.bufferFactory.allocateBuffer();
for (Map.Entry<String, List<String>> entry : this.headers.entrySet()) {
byte[] headerName = entry.getKey().getBytes(this.charset);
for (String headerValueString : entry.getValue()) {
byte[] headerValue = headerValueString.getBytes(this.charset);
buffer.write(headerName);
buffer.write((byte)':');
buffer.write((byte)' ');
buffer.write(headerValue);
buffer.write((byte)'\r');
buffer.write((byte)'\n');
}
}
}
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
buffer.write((byte)'\r');
buffer.write((byte)'\n');
return buffer;
});
}
@Override

50
spring-web/src/main/java/org/springframework/http/codec/xml/Jaxb2XmlEncoder.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2018 the original author or authors.
* Copyright 2002-2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import org.springframework.core.ResolvableType;
import org.springframework.core.codec.AbstractSingleValueEncoder;
@@ -97,7 +98,7 @@ public class Jaxb2XmlEncoder extends AbstractSingleValueEncoder<Object> {
}
@Override
protected Flux<DataBuffer> encode(Object value, DataBufferFactory dataBufferFactory,
protected Flux<DataBuffer> encode(Object value, DataBufferFactory bufferFactory,
ResolvableType type, @Nullable MimeType mimeType, @Nullable Map<String, Object> hints) {
if (!Hints.isLoggingSuppressed(hints)) {
@@ -107,29 +108,30 @@ public class Jaxb2XmlEncoder extends AbstractSingleValueEncoder<Object> {
});
}
boolean release = true;
DataBuffer buffer = dataBufferFactory.allocateBuffer(1024);
OutputStream outputStream = buffer.asOutputStream();
Class<?> clazz = ClassUtils.getUserClass(value);
try {
Marshaller marshaller = initMarshaller(clazz);
marshaller.marshal(value, outputStream);
release = false;
return Flux.just(buffer);
}
catch (MarshalException ex) {
return Flux.error(new EncodingException(
"Could not marshal " + value.getClass() + " to XML", ex));
}
catch (JAXBException ex) {
return Flux.error(new CodecException("Invalid JAXB configuration", ex));
}
finally {
if (release) {
DataBufferUtils.release(buffer);
return Flux.defer(() -> {
boolean release = true;
DataBuffer buffer = bufferFactory.allocateBuffer(1024);
OutputStream outputStream = buffer.asOutputStream();
Class<?> clazz = ClassUtils.getUserClass(value);
try {
Marshaller marshaller = initMarshaller(clazz);
marshaller.marshal(value, outputStream);
release = false;
return Mono.fromCallable(() -> buffer); // Rely on doOnDiscard in base class
}
}
catch (MarshalException ex) {
return Flux.error(new EncodingException(
"Could not marshal " + value.getClass() + " to XML", ex));
}
catch (JAXBException ex) {
return Flux.error(new CodecException("Invalid JAXB configuration", ex));
}
finally {
if (release) {
DataBufferUtils.release(buffer);
}
}
});
}
private Marshaller initMarshaller(Class<?> clazz) throws JAXBException {