INT-3410 TCP NIO Deadlock with Bound TE

JIRA: https://jira.spring.io/browse/INT-3410

When running a fixed thread pool with a bound queue,
and CALLER_RUNS execution rejection policy, it
was possible to deadlock the IO selector thread.

Add a `CompositeExecutor` to use different threads for
IO to those used for message assembly.

Add a `CallerBlocksPolicy` to block the invoking thread
(for a specified time) if the pool is exhausted.

Add documentation.

Polishing

spring-integration-ip/src/main/java/org/springframework/integration/ip/tcp/connection/TcpNioConnection.java

@@ -28,6 +28,7 @@ import java.nio.channels.SocketChannel;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -36,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.core.serializer.Serializer;
 import org.springframework.integration.ip.tcp.serializer.SoftEndOfStreamException;
+import org.springframework.integration.util.CompositeExecutor;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessagingException;
 import org.springframework.util.Assert;
@@ -59,7 +61,7 @@ public class TcpNioConnection extends TcpConnectionSupport {
 
 	private volatile boolean usingDirectBuffers;
 
-	private volatile Executor taskExecutor;
+	private volatile CompositeExecutor taskExecutor;
 
 	private volatile ByteBuffer rawBuffer;
 
@@ -327,7 +329,8 @@ public class TcpNioConnection extends TcpConnectionSupport {
 		this.writingToPipe = true;
 		try {
 			if (this.taskExecutor == null) {
-				this.taskExecutor = Executors.newCachedThreadPool();
+				ExecutorService executor = Executors.newCachedThreadPool();
+				this.taskExecutor = new CompositeExecutor(executor, executor);
 			}
 			// If there is no assembler running, start one
 			checkForAssembler();
@@ -356,7 +359,7 @@ public class TcpNioConnection extends TcpConnectionSupport {
 			 * avoid a deadlock (block on the write to the pipe).
 			 * Hence the count down latch.
 			 */
-			this.taskExecutor.execute(new Runnable() {
+			this.taskExecutor.execute2(new Runnable() {
 				@Override
 				public void run() {
 					try {
@@ -400,7 +403,7 @@ public class TcpNioConnection extends TcpConnectionSupport {
 				if (logger.isDebugEnabled()) {
 					logger.debug(this.getConnectionId() + " Running an assembler");
 				}
-				this.taskExecutor.execute(this);
+				this.taskExecutor.execute2(this);
 			} else {
 				this.executionControl.decrementAndGet();
 			}
@@ -443,7 +446,12 @@ public class TcpNioConnection extends TcpConnectionSupport {
 	 * @param taskExecutor the taskExecutor to set
 	 */
 	public void setTaskExecutor(Executor taskExecutor) {
-		this.taskExecutor = taskExecutor;
+		if (taskExecutor instanceof CompositeExecutor) {
+			this.taskExecutor = (CompositeExecutor) taskExecutor;
+		}
+		else {
+			this.taskExecutor = new CompositeExecutor(taskExecutor, taskExecutor);
+		}
 	}
 
 	/**

spring-integration-ip/src/test/java/org/springframework/integration/ip/tcp/connection/TcpNioConnectionTests.java

@@ -16,8 +16,10 @@
 
 package org.springframework.integration.ip.tcp.connection;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
@@ -30,6 +32,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
+import java.net.ConnectException;
 import java.net.ServerSocket;
 import java.net.Socket;
 import java.net.SocketTimeoutException;
@@ -52,6 +55,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import javax.net.ServerSocketFactory;
+import javax.net.SocketFactory;
 
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -68,7 +72,11 @@ import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.support.converter.MapMessageConverter;
 import org.springframework.integration.test.util.SocketUtils;
 import org.springframework.integration.test.util.TestUtils;
+import org.springframework.integration.util.CallerBlocksPolicy;
+import org.springframework.integration.util.CompositeExecutor;
 import org.springframework.messaging.Message;
+import org.springframework.messaging.support.ErrorMessage;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.util.ReflectionUtils;
 import org.springframework.util.ReflectionUtils.FieldCallback;
 import org.springframework.util.ReflectionUtils.FieldFilter;
@@ -553,6 +561,68 @@ public class TcpNioConnectionTests {
 		assertEquals("baz", inboundMessage.get().getHeaders().get("bar"));
 	}
 
+	@Test
+	public void testAssemblerUsesSecondaryExecutor() throws Exception {
+		final int port = SocketUtils.findAvailableServerSocket();
+		TcpNioServerConnectionFactory factory = new TcpNioServerConnectionFactory(port);
+		factory.setApplicationEventPublisher(mock(ApplicationEventPublisher.class));
+
+		CompositeExecutor compositeExec = compositeExecutor();
+
+		factory.setSoTimeout(1000);
+		factory.setTaskExecutor(compositeExec);
+		final AtomicReference<String> threadName = new AtomicReference<String>();
+		final CountDownLatch latch = new CountDownLatch(1);
+		factory.registerListener(new TcpListener() {
+
+			@Override
+			public boolean onMessage(Message<?> message) {
+				if (!(message instanceof ErrorMessage)) {
+					threadName.set(Thread.currentThread().getName());
+					latch.countDown();
+				}
+				return false;
+			}
+
+		});
+		factory.start();
+
+		Socket socket = null;
+		int n = 0;
+		while (n++ < 100) {
+			try {
+				socket = SocketFactory.getDefault().createSocket("localhost", port);
+				break;
+			}
+			catch (ConnectException e) {}
+			Thread.sleep(100);
+		}
+		assertTrue("Could not open socket to localhost:" + port, n < 100);
+		socket.getOutputStream().write("foo\r\n".getBytes());
+		socket.close();
+
+		assertTrue(latch.await(10, TimeUnit.SECONDS));
+		assertThat(threadName.get(), containsString("assembler"));
+
+		factory.stop();
+	}
+
+	private CompositeExecutor compositeExecutor() {
+		ThreadPoolTaskExecutor ioExec = new ThreadPoolTaskExecutor();
+		ioExec.setCorePoolSize(2);
+		ioExec.setQueueCapacity(10);
+		ioExec.setThreadNamePrefix("io-");
+		ioExec.setRejectedExecutionHandler(new CallerBlocksPolicy(10000));
+		ioExec.initialize();
+		ThreadPoolTaskExecutor assemblerExec = new ThreadPoolTaskExecutor();
+		assemblerExec.setCorePoolSize(2);
+		assemblerExec.setQueueCapacity(10);
+		assemblerExec.setThreadNamePrefix("assembler-");
+		assemblerExec.setRejectedExecutionHandler(new CallerBlocksPolicy(10000));
+		assemblerExec.initialize();
+		return new CompositeExecutor(ioExec, assemblerExec);
+	}
+
 	private void readFully(InputStream is, byte[] buff) throws IOException {
 		for (int i = 0; i < buff.length; i++) {
 			buff[i] = (byte) is.read();