INT-3410 TCP NIO Deadlock with Bound TE

JIRA: https://jira.spring.io/browse/INT-3410

When running a fixed thread pool with a bound queue,
and CALLER_RUNS execution rejection policy, it
was possible to deadlock the IO selector thread.

Add a `CompositeExecutor` to use different threads for
IO to those used for message assembly.

Add a `CallerBlocksPolicy` to block the invoking thread
(for a specified time) if the pool is exhausted.

Add documentation.

Polishing

spring-integration-core/src/main/java/org/springframework/integration/util/CallerBlocksPolicy.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.util;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * A {@link RejectedExecutionHandler} that blocks the caller until
+ * the executor has room in its queue, or a timeout occurs (in which
+ * case a {@link RejectedExecutionException} is thrown.
+ *
+ * @author Gary Russell
+ * @since 3.0.3
+ *
+ */
+public class CallerBlocksPolicy implements RejectedExecutionHandler {
+
+	private static final Log logger = LogFactory.getLog(CallerBlocksPolicy.class);
+
+	private final long maxWait;
+
+	/**
+	 * @param maxWait The maximum time to wait for a queue slot to be
+	 * available, in milliseconds.
+	 */
+	public CallerBlocksPolicy(long maxWait) {
+		this.maxWait = maxWait;
+	}
+
+	@Override
+	public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
+		if (!executor.isShutdown()) {
+			try {
+				BlockingQueue<Runnable> queue = executor.getQueue();
+				if (logger.isDebugEnabled()) {
+					logger.debug("Attempting to queue task execution for " + this.maxWait + " milliseconds");
+				}
+				if (!queue.offer(r, this.maxWait, TimeUnit.MILLISECONDS)) {
+					throw new RejectedExecutionException("Max wait time expired to queue task");
+				}
+				if (logger.isDebugEnabled()) {
+					logger.debug("Task execution queued");
+				}
+			}
+			catch (InterruptedException e) {
+				Thread.currentThread().interrupt();
+				throw new RejectedExecutionException("Interrupted", e);
+			}
+		}
+		else {
+			throw new RejectedExecutionException("Executor has been shut down");
+		}
+	}
+
+}

spring-integration-core/src/main/java/org/springframework/integration/util/CompositeExecutor.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.util;
+
+import java.util.concurrent.Executor;
+
+import org.springframework.util.Assert;
+
+/**
+ * An {@link Executor} that encapsulates two underlying executors. Used in cases
+ * where two distinct operation types are being used where sharing threads could
+ * adversely affect the operation of the system. For example, NIO event processing
+ * threads being used for other blocking operations (such as assembling messages).
+ * If a {@code CallerRunsPolicy} rejected execution policy is used, the NIO event thread
+ * might deadlock, and stop processing events.
+ * <p>
+ * In order to work in such an environment, the secondary executor must <b>not</b>
+ * have a {@code CallerRunsPolicy} rejected execution policy.
+ * <p>
+ * It is generally recommended to use a {@code CallerBlocksPolicy} on both
+ * executors.
+ *
+ * @author Gary Russell
+ * @since 3.0.3
+ *
+ */
+public class CompositeExecutor implements Executor {
+
+	private final Executor primaryTaskExecutor;
+
+	private final Executor secondaryTaskExecutor;
+
+	public CompositeExecutor(Executor primaryTaskExecutor, Executor secondaryTaskExecutor) {
+		Assert.notNull(primaryTaskExecutor, "'primaryTaskExecutor' cannot be null");
+		Assert.notNull(primaryTaskExecutor, "'secondaryTaskExecutor' cannot be null");
+		this.primaryTaskExecutor = primaryTaskExecutor;
+		this.secondaryTaskExecutor = secondaryTaskExecutor;
+	}
+
+	/**
+	 * Execute using the primary executor.
+	 * @param task the task to run.
+	 */
+	@Override
+	public void execute(Runnable task) {
+		this.primaryTaskExecutor.execute(task);
+	}
+
+	/**
+	 * Execute using the secondary executor.
+	 * @param task the task to run.
+	 */
+	public void execute2(Runnable task) {
+		this.secondaryTaskExecutor.execute(task);
+	}
+
+}

spring-integration-core/src/test/java/org/springframework/integration/util/CallerBlocksPolicyTests.java

@@ -0,0 +1,111 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.util;
+
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.Test;
+
+import org.springframework.core.task.TaskRejectedException;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+
+/**
+ * @author Gary Russell
+ * @since 3.0.3
+ *
+ */
+public class CallerBlocksPolicyTests {
+
+	@Test
+	public void test0() throws Exception {
+		final ThreadPoolTaskExecutor te = new ThreadPoolTaskExecutor();
+		te.setCorePoolSize(1);
+		te.setMaxPoolSize(1);
+		te.setQueueCapacity(0);
+		te.setRejectedExecutionHandler(new CallerBlocksPolicy(1000));
+		te.initialize();
+		final AtomicReference<Throwable> e = new AtomicReference<Throwable>();
+		final CountDownLatch latch = new CountDownLatch(1);
+		te.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				try {
+					te.execute(this);
+				}
+				catch (TaskRejectedException tre) {
+					e.set(tre.getCause());
+				}
+				latch.countDown();
+			}
+		});
+		assertTrue(latch.await(10,  TimeUnit.SECONDS));
+		assertThat(e.get(), instanceOf(RejectedExecutionException.class));
+		assertEquals("Max wait time expired to queue task", e.get().getMessage());
+	}
+
+	@Test
+	public void test1() throws Exception {
+		final ThreadPoolTaskExecutor te = new ThreadPoolTaskExecutor();
+		te.setCorePoolSize(2);
+		te.setMaxPoolSize(2);
+		te.setQueueCapacity(1);
+		te.setRejectedExecutionHandler(new CallerBlocksPolicy(10000));
+		te.initialize();
+		final AtomicReference<Throwable> e = new AtomicReference<Throwable>();
+		final CountDownLatch latch = new CountDownLatch(3);
+		te.execute(new Runnable() {
+
+			@Override
+			public void run() {
+				try {
+					Runnable foo = new Runnable() {
+
+						@Override
+						public void run() {
+							try {
+								Thread.sleep(1000);
+							}
+							catch (InterruptedException e) {
+								Thread.currentThread().interrupt();
+								throw new RuntimeException();
+							}
+							latch.countDown();
+						}
+					};
+					te.execute(foo);
+					te.execute(foo); // this one will be queued
+					te.execute(foo); // this one will be blocked and successful later
+				}
+				catch (TaskRejectedException tre) {
+					e.set(tre.getCause());
+				}
+			}
+		});
+		assertTrue(latch.await(10,  TimeUnit.SECONDS));
+		assertNull(e.get());
+	}
+
+}