Spring/spring-framework
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SPR-7909 - Sending large payloads with RestTemplate results in an OutOfMemoryError

Browse Source
This commit is contained in:
Arjen Poutsma
2011-01-25 16:08:15 +00:00
parent 33674933ea
commit d0d6a07870
12 changed files with 388 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
org.springframework.web/src/main/java/org/springframework/http/client/AbstractBufferingClientHttpRequest.java Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.http.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import org.springframework.http.HttpHeaders;
/**
* Abstract base for {@link ClientHttpRequest} that buffers output in a byte array before sending it over the wire.
*
* @author Arjen Poutsma
* @since 3.0.6
*/
abstract class AbstractBufferingClientHttpRequest extends AbstractClientHttpRequest {
private ByteArrayOutputStream bufferedOutput = new ByteArrayOutputStream();
@Override
protected OutputStream getBodyInternal(HttpHeaders headers) throws IOException {
return this.bufferedOutput;
}
@Override
protected ClientHttpResponse executeInternal(HttpHeaders headers) throws IOException {
byte[] bytes = this.bufferedOutput.toByteArray();
if (headers.getContentLength() == -1) {
headers.setContentLength(bytes.length);
}
ClientHttpResponse result = executeInternal(headers, bytes);
this.bufferedOutput = null;
return result;
}
/**
* Abstract template method that writes the given headers and content to the HTTP request.
* @param headers the HTTP headers
* @param bufferedOutput the body content
* @return the response object for the executed request
*/
protected abstract ClientHttpResponse executeInternal(HttpHeaders headers, byte[] bufferedOutput)
throws IOException;
}

27
org.springframework.web/src/main/java/org/springframework/http/client/AbstractClientHttpRequest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
package org.springframework.http.client;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
@@ -29,27 +28,32 @@ import org.springframework.util.Assert;
* @author Arjen Poutsma
* @since 3.0
*/
public abstract class AbstractClientHttpRequest implements ClientHttpRequest {
abstract class AbstractClientHttpRequest implements ClientHttpRequest {
private boolean executed = false;
private final HttpHeaders headers = new HttpHeaders();
private final ByteArrayOutputStream bufferedOutput = new ByteArrayOutputStream();
public final HttpHeaders getHeaders() {
return executed ? HttpHeaders.readOnlyHttpHeaders(headers) : this.headers;
}
public final OutputStream getBody() throws IOException {
checkExecuted();
return this.bufferedOutput;
return getBodyInternal(this.headers);
}
/**
* Abstract template method that returns the body.
*
* @param headers the HTTP headers
* @return the body output stream
*/
protected abstract OutputStream getBodyInternal(HttpHeaders headers) throws IOException;
public final ClientHttpResponse execute() throws IOException {
checkExecuted();
ClientHttpResponse result = executeInternal(this.headers, this.bufferedOutput.toByteArray());
ClientHttpResponse result = executeInternal(this.headers);
this.executed = true;
return result;
}
@@ -58,14 +62,13 @@ public abstract class AbstractClientHttpRequest implements ClientHttpRequest {
Assert.state(!this.executed, "ClientHttpRequest already executed");
}
/**
* Abstract template method that writes the given headers and content to the HTTP request.
*
* @param headers the HTTP headers
* @param bufferedOutput the body content
* @return the response object for the executed request
*/
protected abstract ClientHttpResponse executeInternal(HttpHeaders headers, byte[] bufferedOutput)
throws IOException;
protected abstract ClientHttpResponse executeInternal(HttpHeaders headers) throws IOException;
}

9
org.springframework.web/src/main/java/org/springframework/http/client/SimpleClientHttpRequest.java → org.springframework.web/src/main/java/org/springframework/http/client/BufferingSimpleClientHttpRequest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,19 +28,18 @@ import org.springframework.http.HttpMethod;
import org.springframework.util.FileCopyUtils;
/**
* {@link ClientHttpRequest} implementation that uses standard J2SE facilities to execute requests.
* {@link ClientHttpRequest} implementation that uses standard J2SE facilities to execute buffered requests.
* Created via the {@link SimpleClientHttpRequestFactory}.
*
* @author Arjen Poutsma
* @since 3.0
* @see SimpleClientHttpRequestFactory#createRequest(java.net.URI, HttpMethod)
*/
final class SimpleClientHttpRequest extends AbstractClientHttpRequest {
final class BufferingSimpleClientHttpRequest extends AbstractBufferingClientHttpRequest {
private final HttpURLConnection connection;
SimpleClientHttpRequest(HttpURLConnection connection) {
BufferingSimpleClientHttpRequest(HttpURLConnection connection) {
this.connection = connection;
}

4
org.springframework.web/src/main/java/org/springframework/http/client/CommonsClientHttpRequest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -41,7 +41,7 @@ import org.springframework.http.HttpMethod;
* @since 3.0
* @see CommonsClientHttpRequestFactory#createRequest(java.net.URI, HttpMethod)
*/
final class CommonsClientHttpRequest extends AbstractClientHttpRequest {
final class CommonsClientHttpRequest extends AbstractBufferingClientHttpRequest {
private final HttpClient httpClient;

58
org.springframework.web/src/main/java/org/springframework/http/client/SimpleClientHttpRequestFactory.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,8 +36,14 @@ import org.springframework.util.Assert;
*/
public class SimpleClientHttpRequestFactory implements ClientHttpRequestFactory {
private static final int DEFAULT_CHUNK_SIZE = 4096;
private Proxy proxy;
private boolean bufferRequestBody = true;
private int chunkSize = DEFAULT_CHUNK_SIZE;
/**
* Sets the {@link Proxy} to use for this request factory.
*/
@@ -45,16 +51,48 @@ public class SimpleClientHttpRequestFactory implements ClientHttpRequestFactory
this.proxy = proxy;
}
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
HttpURLConnection connection = openConnection(uri.toURL(), proxy);
prepareConnection(connection, httpMethod.name());
return new SimpleClientHttpRequest(connection);
/**
* Indicates whether this request factory should buffer the {@linkplain ClientHttpRequest#getBody() request body}
* internally.
* <p>Default is {@code true}. When sending large amounts of data via POST or PUT, it is recommended to change this
* property to {@code false}, so as not to run out of memory. This will result in a {@link ClientHttpRequest}
* that either streams directly to the underlying {@link HttpURLConnection} (if the
* {@link org.springframework.http.HttpHeaders#getContentLength() Content-Length} is known in advance), or that will
* use "Chunked transfer encoding" (if the {@code Content-Length} is not known in advance).
*
* @see #setChunkSize(int)
* @see HttpURLConnection#setFixedLengthStreamingMode(int)
*/
public void setBufferRequestBody(boolean bufferRequestBody) {
this.bufferRequestBody = bufferRequestBody;
}
/**
* Opens and returns a connection to the given URL.
* <p>The default implementation uses the given {@linkplain #setProxy(java.net.Proxy) proxy} - if any - to open a
* connection.
* Sets the number of bytes to write in each chunk when not buffering request bodies locally.
* <p>Note that this parameter is only used when {@link #setBufferRequestBody(boolean) bufferRequestBody} is set
* to {@code false}, and the {@link org.springframework.http.HttpHeaders#getContentLength() Content-Length}
* is not known in advance.
*
* @see #setBufferRequestBody(boolean)
*/
public void setChunkSize(int chunkSize) {
this.chunkSize = chunkSize;
}
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
HttpURLConnection connection = openConnection(uri.toURL(), proxy);
prepareConnection(connection, httpMethod.name());
if (bufferRequestBody) {
return new BufferingSimpleClientHttpRequest(connection);
}
else {
return new StreamingSimpleClientHttpRequest(connection, chunkSize);
}
}
/**
* Opens and returns a connection to the given URL. <p>The default implementation uses the given {@linkplain
* #setProxy(java.net.Proxy) proxy} - if any - to open a connection.
*
* @param url the URL to open a connection to
* @param proxy the proxy to use, may be {@code null}
@@ -68,8 +106,8 @@ public class SimpleClientHttpRequestFactory implements ClientHttpRequestFactory
}
/**
* Template method for preparing the given {@link HttpURLConnection}.
* <p>The default implementation prepares the connection for input and output, and sets the HTTP method.
* Template method for preparing the given {@link HttpURLConnection}. <p>The default implementation prepares the
* connection for input and output, and sets the HTTP method.
*
* @param connection the connection to prepare
* @param httpMethod the HTTP request method ({@code GET}, {@code POST}, etc.)

5
org.springframework.web/src/main/java/org/springframework/http/client/SimpleClientHttpResponse.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -26,7 +26,8 @@ import org.springframework.util.StringUtils;
/**
* {@link ClientHttpResponse} implementation that uses standard J2SE facilities.
* Obtained via the {@link SimpleClientHttpRequest#execute()}.
* Obtained via {@link BufferingSimpleClientHttpRequest#execute()} and
* {@link StreamingSimpleClientHttpRequest#execute()}.
*
* @author Arjen Poutsma
* @since 3.0

112
org.springframework.web/src/main/java/org/springframework/http/client/StreamingSimpleClientHttpRequest.java Normal file
View File

@@ -0,0 +1,112 @@
/*
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.http.client;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
/**
* {@link ClientHttpRequest} implementation that uses standard J2SE facilities to execute streaming requests.
* Created via the {@link SimpleClientHttpRequestFactory}.
*
* @author Arjen Poutsma
* @since 3.0
* @see SimpleClientHttpRequestFactory#createRequest(java.net.URI, HttpMethod)
*/
public class StreamingSimpleClientHttpRequest extends AbstractClientHttpRequest {
private final HttpURLConnection connection;
private final int chunkSize;
private OutputStream body;
StreamingSimpleClientHttpRequest(HttpURLConnection connection, int chunkSize) {
this.connection = connection;
this.chunkSize = chunkSize;
}
public HttpMethod getMethod() {
return HttpMethod.valueOf(this.connection.getRequestMethod());
}
public URI getURI() {
try {
return this.connection.getURL().toURI();
}
catch (URISyntaxException ex) {
throw new IllegalStateException("Could not get HttpURLConnection URI: " + ex.getMessage(), ex);
}
}
@Override
protected OutputStream getBodyInternal(HttpHeaders headers) throws IOException {
if (body == null) {
int contentLength = (int) headers.getContentLength();
if (contentLength >= 0) {
this.connection.setFixedLengthStreamingMode(contentLength);
}
else {
this.connection.setChunkedStreamingMode(chunkSize);
}
for (Map.Entry<String, List<String>> entry : headers.entrySet()) {
String headerName = entry.getKey();
for (String headerValue : entry.getValue()) {
this.connection.addRequestProperty(headerName, headerValue);
}
}
this.connection.connect();
this.body = this.connection.getOutputStream();
}
return new NonClosingOutputStream(body);
}
@Override
protected ClientHttpResponse executeInternal(HttpHeaders headers) throws IOException {
try {
if (body != null) {
body.close();
}
}
catch (IOException ex) {
// ignore
}
return new SimpleClientHttpResponse(this.connection);
}
private static class NonClosingOutputStream extends FilterOutputStream {
private NonClosingOutputStream(OutputStream out) {
super(out);
}
@Override
public void close() throws IOException {
}
}
}

26
org.springframework.web/src/main/java/org/springframework/http/converter/FormHttpMessageConverter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2011 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,7 +19,6 @@ package org.springframework.http.converter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
@@ -94,19 +93,25 @@ public class FormHttpMessageConverter implements HttpMessageConverter<MultiValue
this.partConverters.add(new ResourceHttpMessageConverter());
}
/** Add a message body converter. Such a converters is used to convert objects to MIME parts. */
/**
* Add a message body converter. Such a converters is used to convert objects to MIME parts.
*/
public final void addPartConverter(HttpMessageConverter<?> partConverter) {
Assert.notNull(partConverter, "'partConverter' must not be NULL");
this.partConverters.add(partConverter);
}
/** Set the message body converters to use. These converters are used to convert objects to MIME parts. */
/**
* Set the message body converters to use. These converters are used to convert objects to MIME parts.
*/
public final void setPartConverters(List<HttpMessageConverter<?>> partConverters) {
Assert.notEmpty(partConverters, "'partConverters' must not be empty");
this.partConverters = partConverters;
}
/** Sets the character set used for writing form data. */
/**
* Sets the character set used for writing form data.
*/
public void setCharset(Charset charset) {
this.charset = charset;
}
@@ -196,7 +201,8 @@ public class FormHttpMessageConverter implements HttpMessageConverter<MultiValue
if (contentType != null) {
outputMessage.getHeaders().setContentType(contentType);
charset = contentType.getCharSet() != null ? contentType.getCharSet() : this.charset;
} else {
}
else {
outputMessage.getHeaders().setContentType(MediaType.APPLICATION_FORM_URLENCODED);
charset = this.charset;
}
@@ -218,7 +224,9 @@ public class FormHttpMessageConverter implements HttpMessageConverter<MultiValue
builder.append('&');
}
}
FileCopyUtils.copy(builder.toString(), new OutputStreamWriter(outputMessage.getBody(), charset));
byte[] bytes = builder.toString().getBytes(charset.name());
outputMessage.getHeaders().setContentLength(bytes.length);
FileCopyUtils.copy(bytes, outputMessage.getBody());
}
private void writeMultipart(MultiValueMap<String, Object> parts, HttpOutputMessage outputMessage)
@@ -328,7 +336,9 @@ public class FormHttpMessageConverter implements HttpMessageConverter<MultiValue
}
}
/** Implementation of {@link org.springframework.http.HttpOutputMessage} used for writing multipart data. */
/**
* Implementation of {@link org.springframework.http.HttpOutputMessage} used for writing multipart data.
*/
private class MultipartHttpOutputMessage implements HttpOutputMessage {
private final HttpHeaders headers = new HttpHeaders();