diff --git a/spring-rabbit/src/main/java/org/springframework/amqp/rabbit/AsyncRabbitTemplate.java b/spring-rabbit/src/main/java/org/springframework/amqp/rabbit/AsyncRabbitTemplate.java
index 8f2cbb1a..f4fa65f5 100644
--- a/spring-rabbit/src/main/java/org/springframework/amqp/rabbit/AsyncRabbitTemplate.java
+++ b/spring-rabbit/src/main/java/org/springframework/amqp/rabbit/AsyncRabbitTemplate.java
@@ -438,10 +438,10 @@ public class AsyncRabbitTemplate implements SmartLifecycle, MessageListener, Ret
 				future.setNackCause("AsyncRabbitTemplate was stopped while waiting for reply");
 				future.cancel(true);
 			}
-		}
-		if (this.internalTaskScheduler) {
-			((ThreadPoolTaskScheduler) this.taskScheduler).destroy();
-			this.taskScheduler = null;
+			if (this.internalTaskScheduler) {
+				((ThreadPoolTaskScheduler) this.taskScheduler).destroy();
+				this.taskScheduler = null;
+			}
 		}
 		this.running = false;
 	}
diff --git a/spring-rabbit/src/test/java/org/springframework/amqp/rabbit/AsyncRabbitTemplateTests.java b/spring-rabbit/src/test/java/org/springframework/amqp/rabbit/AsyncRabbitTemplateTests.java
index 3af5f5cb..eebbe1d7 100644
--- a/spring-rabbit/src/test/java/org/springframework/amqp/rabbit/AsyncRabbitTemplateTests.java
+++ b/spring-rabbit/src/test/java/org/springframework/amqp/rabbit/AsyncRabbitTemplateTests.java
@@ -67,6 +67,8 @@ import org.springframework.util.concurrent.ListenableFutureCallback;
 
 /**
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 1.6
  */
 @ContextConfiguration
@@ -252,6 +254,8 @@ public class AsyncRabbitTemplateTests {
 		future.addCallback(callback);
 		assertEquals(1, TestUtils.getPropertyValue(this.template, "pending", Map.class).size());
 		this.template.stop();
+		// Second stop() to be sure that it is idempotent
+		this.template.stop();
 		try {
 			future.get(10, TimeUnit.SECONDS);
 			fail("Expected CancellationException");