AMQP: Support CorrelationData Message Headers

In preparation for https://github.com/spring-cloud/spring-cloud-stream-binder-rabbit/issues/303

Now that `CorrelationData` has a `Future<?>`, users might simply add
correlation data in a header and not receive confirm/return messages.

- No longer require channels for returns and confirms
- don't build the confirm message if there are no channels
- reduce the log level for no channels to DEBUG
- complete the user's future when a message is returned (async GW)

* Fix typo.

spring-integration-amqp/src/main/java/org/springframework/integration/amqp/outbound/AbstractAmqpOutboundEndpoint.java

@@ -63,7 +63,7 @@ import org.springframework.util.concurrent.SettableListenableFuture;
 public abstract class AbstractAmqpOutboundEndpoint extends AbstractReplyProducingMessageHandler
 		implements Lifecycle {
 
-	private static final UUID NO_ID = new UUID(0L, 0L);
+	private static final String NO_ID = new UUID(0L, 0L).toString();
 
 	private String exchangeName;
 
@@ -548,20 +548,33 @@ public abstract class AbstractAmqpOutboundEndpoint extends AbstractReplyProducin
 
 	protected CorrelationData generateCorrelationData(Message<?> requestMessage) {
 		CorrelationData correlationData = null;
+		UUID uuid = requestMessage.getHeaders().getId();
+		String messageId;
+		if (uuid == null) {
+			messageId = NO_ID;
+		}
+		else {
+			messageId = uuid.toString();
+		}
 		if (this.correlationDataGenerator != null) {
-			UUID messageId = requestMessage.getHeaders().getId();
-			if (messageId == null) {
-				messageId = NO_ID;
-			}
 			Object userData = this.correlationDataGenerator.processMessage(requestMessage);
 			if (userData != null) {
-				correlationData = new CorrelationDataWrapper(messageId.toString(), userData, requestMessage);
+				correlationData = new CorrelationDataWrapper(messageId, userData, requestMessage);
 			}
 			else {
 				this.logger.debug("'confirmCorrelationExpression' resolved to 'null'; "
 						+ "no publisher confirm will be sent to the ack or nack channel");
 			}
 		}
+		if (correlationData == null) {
+			Object correlation = requestMessage.getHeaders().get(AmqpHeaders.PUBLISH_CONFIRM_CORRELATION);
+			if (correlation instanceof CorrelationData) {
+				correlationData = (CorrelationData) correlation;
+			}
+			if (correlationData != null) {
+				correlationData = new CorrelationDataWrapper(messageId, correlationData, requestMessage);
+			}
+		}
 		return correlationData;
 	}
 
@@ -652,19 +665,21 @@ public abstract class AbstractAmqpOutboundEndpoint extends AbstractReplyProducin
 			return;
 		}
 		Object userCorrelationData = wrapper.getUserData();
-		Message<?> confirmMessage;
-		confirmMessage = buildConfirmMessage(ack, cause, wrapper, userCorrelationData);
-		if (ack && getConfirmAckChannel() != null) {
-			sendOutput(confirmMessage, getConfirmAckChannel(), true);
-		}
-		else if (!ack && getConfirmNackChannel() != null) {
-			sendOutput(confirmMessage, getConfirmNackChannel(), true);
+		MessageChannel ackChannel = getConfirmAckChannel();
+		if (ack && ackChannel != null) {
+			sendOutput(buildConfirmMessage(ack, cause, wrapper, userCorrelationData), ackChannel, true);
 		}
 		else {
-			if (logger.isInfoEnabled()) {
-				logger.info("Nowhere to send publisher confirm "
-						+ (ack ? "ack" : "nack") + " for "
-						+ userCorrelationData);
+			MessageChannel nackChannel = getConfirmNackChannel();
+			if (!ack && nackChannel != null) {
+				sendOutput(buildConfirmMessage(ack, cause, wrapper, userCorrelationData), nackChannel, true);
+			}
+			else {
+				if (logger.isDebugEnabled()) {
+					logger.debug("Nowhere to send publisher confirm "
+							+ (ack ? "ack" : "nack") + " for "
+							+ userCorrelationData);
+				}
 			}
 		}
 	}

spring-integration-amqp/src/main/java/org/springframework/integration/amqp/outbound/AsyncAmqpOutboundGateway.java

@@ -22,12 +22,14 @@ import org.springframework.amqp.core.ReturnedMessage;
 import org.springframework.amqp.rabbit.AsyncRabbitTemplate;
 import org.springframework.amqp.rabbit.AsyncRabbitTemplate.RabbitMessageFuture;
 import org.springframework.amqp.rabbit.connection.CorrelationData;
+import org.springframework.amqp.rabbit.connection.CorrelationData.Confirm;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.amqp.support.converter.MessageConverter;
 import org.springframework.integration.amqp.support.MappingUtils;
 import org.springframework.integration.handler.ReplyRequiredException;
 import org.springframework.integration.support.AbstractIntegrationMessageBuilder;
 import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageChannel;
 import org.springframework.messaging.MessageHandlingException;
 import org.springframework.messaging.MessagingException;
 import org.springframework.util.Assert;
@@ -87,11 +89,11 @@ public class AsyncAmqpOutboundGateway extends AbstractAmqpOutboundEndpoint {
 		addDelayProperty(requestMessage, amqpMessage);
 		RabbitMessageFuture future = this.template.sendAndReceive(generateExchangeName(requestMessage),
 				generateRoutingKey(requestMessage), amqpMessage);
-		future.addCallback(new FutureCallback(requestMessage));
 		CorrelationData correlationData = generateCorrelationData(requestMessage);
 		if (correlationData != null && future.getConfirm() != null) {
 			future.getConfirm().addCallback(new CorrelationCallback(correlationData, future));
 		}
+		future.addCallback(new FutureCallback(requestMessage, correlationData));
 		return null;
 	}
 
@@ -99,8 +101,11 @@ public class AsyncAmqpOutboundGateway extends AbstractAmqpOutboundEndpoint {
 
 		private final Message<?> requestMessage;
 
-		FutureCallback(Message<?> requestMessage) {
+		private final CorrelationDataWrapper correlationData;
+
+		FutureCallback(Message<?> requestMessage, CorrelationData correlationData) {
 			this.requestMessage = requestMessage;
+			this.correlationData = (CorrelationDataWrapper) correlationData;
 		}
 
 		@Override
@@ -141,18 +146,21 @@ public class AsyncAmqpOutboundGateway extends AbstractAmqpOutboundEndpoint {
 				}
 			}
 			if (ex instanceof AmqpMessageReturnedException) {
-				if (getReturnChannel() == null) {
-					logger.error("Returned message received and no return channel "
-							+ ((AmqpMessageReturnedException) ex).getReturnedMessage());
-				}
-				else {
-					AmqpMessageReturnedException amre = (AmqpMessageReturnedException) ex;
+				AmqpMessageReturnedException amre = (AmqpMessageReturnedException) ex;
+				MessageChannel returnChannel = getReturnChannel();
+				if (returnChannel != null) {
 					Message<?> returnedMessage = buildReturnedMessage(
 							new ReturnedMessage(amre.getReturnedMessage(), amre.getReplyCode(), amre.getReplyText(),
 									amre.getExchange(), amre.getRoutingKey()),
 							AsyncAmqpOutboundGateway.this.messageConverter);
-					sendOutput(returnedMessage, getReturnChannel(), true);
+					sendOutput(returnedMessage, returnChannel, true);
 				}
+				this.correlationData.setReturnedMessage(amre.getReturnedMessage());
+				/*
+				 *  Complete the user's future (if present) since the async template will only complete
+				 *  once, successfully, or with a failure.
+				 */
+				this.correlationData.getFuture().set(new Confirm(true, null));
 			}
 			else {
 				sendErrorMessage(this.requestMessage, exceptionToSend);

spring-integration-amqp/src/test/java/org/springframework/integration/amqp/outbound/AmqpOutboundEndpointTests2.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019 the original author or authors.
+ * Copyright 2019-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import java.util.Collections;
+import java.util.concurrent.TimeUnit;
 
 import org.junit.jupiter.api.Test;
 
@@ -29,16 +30,19 @@ import org.springframework.amqp.core.QueueBuilder;
 import org.springframework.amqp.core.QueueBuilder.Overflow;
 import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
 import org.springframework.amqp.rabbit.connection.ConnectionFactory;
+import org.springframework.amqp.rabbit.connection.CorrelationData;
 import org.springframework.amqp.rabbit.core.RabbitAdmin;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.amqp.rabbit.junit.RabbitAvailable;
 import org.springframework.amqp.rabbit.junit.RabbitAvailableCondition;
+import org.springframework.amqp.support.AmqpHeaders;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.integration.amqp.dsl.Amqp;
 import org.springframework.integration.config.EnableIntegration;
 import org.springframework.integration.dsl.IntegrationFlow;
+import org.springframework.integration.support.MessageBuilder;
 import org.springframework.messaging.MessageHandlingException;
 import org.springframework.messaging.support.GenericMessage;
 import org.springframework.test.annotation.DirtiesContext;
@@ -74,6 +78,17 @@ public class AmqpOutboundEndpointTests2 {
 				.isEqualTo("Message was returned by the broker");
 	}
 
+	@Test
+	void testReturnConfirmNoChannels(@Autowired IntegrationFlow flow2) throws Exception {
+		CorrelationData corr = new CorrelationData("foo");
+		flow2.getInputChannel().send(MessageBuilder.withPayload("test")
+				.setHeader("rk", "junkjunk")
+				.setHeader(AmqpHeaders.PUBLISH_CONFIRM_CORRELATION, corr)
+				.build());
+		assertThat(corr.getFuture().get(10, TimeUnit.SECONDS).isAck()).isTrue();
+		assertThat(corr.getReturnedMessage()).isNotNull();
+	}
+
 	@Test
 	@DisabledIf("#{systemEnvironment['TRAVIS'] ?: false}")
 		// needs RabbitMQ 3.7
@@ -106,6 +121,13 @@ public class AmqpOutboundEndpointTests2 {
 					.waitForConfirm(true));
 		}
 
+		@Bean
+		public IntegrationFlow flow2(RabbitTemplate template) {
+			return f -> f.handle(Amqp.outboundAdapter(template)
+					.exchangeName("")
+					.routingKeyFunction(msg -> msg.getHeaders().get("rk", String.class)));
+		}
+
 		@Bean
 		public CachingConnectionFactory cf() {
 			CachingConnectionFactory ccf = new CachingConnectionFactory(

spring-integration-amqp/src/test/java/org/springframework/integration/amqp/outbound/AsyncAmqpGatewayTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016-2019 the original author or authors.
+ * Copyright 2016-2020 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,6 +37,7 @@ import org.springframework.amqp.core.AmqpReplyTimeoutException;
 import org.springframework.amqp.rabbit.AsyncRabbitTemplate;
 import org.springframework.amqp.rabbit.AsyncRabbitTemplate.RabbitMessageFuture;
 import org.springframework.amqp.rabbit.connection.CachingConnectionFactory;
+import org.springframework.amqp.rabbit.connection.CorrelationData;
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.amqp.rabbit.junit.RabbitAvailable;
 import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer;
@@ -49,6 +50,7 @@ import org.springframework.beans.factory.BeanFactory;
 import org.springframework.integration.amqp.support.NackedAmqpMessageException;
 import org.springframework.integration.amqp.support.ReturnedAmqpMessageException;
 import org.springframework.integration.channel.DirectChannel;
+import org.springframework.integration.channel.NullChannel;
 import org.springframework.integration.channel.QueueChannel;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.test.condition.LogLevels;
@@ -220,4 +222,38 @@ class AsyncAmqpGatewayTests {
 		ccf.destroy();
 	}
 
+	@Test
+	void confirmsAndReturnsNoChannels() throws Exception {
+		CachingConnectionFactory ccf = new CachingConnectionFactory("localhost");
+		ccf.setPublisherConfirmType(CachingConnectionFactory.ConfirmType.CORRELATED);
+		ccf.setPublisherReturns(true);
+		RabbitTemplate template = new RabbitTemplate(ccf);
+		SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(ccf);
+		container.setBeanName("replyContainer");
+		container.setQueueNames("asyncRQ1");
+		container.afterPropertiesSet();
+		container.start();
+		AsyncRabbitTemplate asyncTemplate = new AsyncRabbitTemplate(template, container);
+		asyncTemplate.setEnableConfirms(true);
+		asyncTemplate.setMandatory(true);
+
+		AsyncAmqpOutboundGateway gateway = new AsyncAmqpOutboundGateway(asyncTemplate);
+		gateway.setOutputChannel(new NullChannel());
+		gateway.setExchangeName("");
+		gateway.setRoutingKey("noRoute");
+		gateway.setBeanFactory(mock(BeanFactory.class));
+		gateway.afterPropertiesSet();
+		gateway.start();
+
+		CorrelationData corr = new CorrelationData("foo");
+		gateway.handleMessage(MessageBuilder.withPayload("test")
+				.setHeader(AmqpHeaders.PUBLISH_CONFIRM_CORRELATION, corr)
+				.build());
+		assertThat(corr.getFuture().get(10, TimeUnit.SECONDS).isAck()).isTrue();
+		assertThat(corr.getReturnedMessage()).isNotNull();
+
+		asyncTemplate.stop();
+		ccf.destroy();
+	}
+
 }

src/reference/asciidoc/amqp.adoc

@@ -601,18 +601,22 @@ Version 4.1 introduced the `amqp_publishConfirmNackCause` message header.
 It contains the `cause` of a 'nack' for a publisher confirmation.
 Starting with version 4.2, if the expression resolves to a `Message<?>` instance (such as `#this`), the message emitted on the `ack`/`nack` channel is based on that message, with the additional header(s) added.
 Previously, a new message was created with the correlation data as its payload, regardless of type.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <11> The channel to which positive (`ack`) publisher confirms are sent.
 The payload is the correlation data defined by the `confirm-correlation-expression`.
 If the expression is `#root` or `#this`, the message is built from the original message, with the `amqp_publishConfirm` header set to `true`.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <12> The channel to which negative (`nack`) publisher confirmations are sent.
 The payload is the correlation data defined by the `confirm-correlation-expression` (if there is no `ErrorMessageStrategy` configured).
 If the expression is `#root` or `#this`, the message is built from the original message, with the `amqp_publishConfirm` header set to `false`.
 When there is an `ErrorMessageStrategy`, the message is an `ErrorMessage` with a `NackedAmqpMessageException` payload.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <13> When set, the adapter will synthesize a negative acknowledgment (nack) if a publisher confirm is not received within this time in milliseconds.
 Pending confirms are checked every 50% of this value, so the actual time a nack is sent will be between 1x and 1.5x this value.
+Also see <<alternative-confirms-returns>>.
 Default none (nacks will not be generated).
 <14> When set to true, the calling thread will block, waiting for a publisher confirmation.
 This requires a `RabbitTemplate` configured for confirms as well as a `confirm-correlation-expression`.
@@ -623,6 +627,7 @@ If returns are enabled and a message is returned, or any other exception occurs
 When provided, the underlying AMQP template is configured to return undeliverable messages to the adapter.
 When there is no `ErrorMessageStrategy` configured, the message is constructed from the data received from AMQP, with the following additional headers: `amqp_returnReplyCode`, `amqp_returnReplyText`, `amqp_returnExchange`, `amqp_returnRoutingKey`.
 When there is an `ErrorMessageStrategy`, the message is an `ErrorMessage` with a `ReturnedAmqpMessageException` payload.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <16> A reference to an `ErrorMessageStrategy` implementation used to build `ErrorMessage` instances when sending returned or negatively acknowledged messages.
 <17> A reference to an `AmqpHeaderMapper` to use when sending AMQP Messages.
@@ -815,15 +820,18 @@ Examples: `headers['myCorrelationData']` and `payload`.
 If the expression resolves to a `Message<?>` instance (such as `#this`), the message
 emitted on the `ack`/`nack` channel is based on that message, with the additional headers added.
 Previously, a new message was created with the correlation data as its payload, regardless of type.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <14> The channel to which positive (`ack`) publisher confirmations are sent.
 The payload is the correlation data defined by `confirm-correlation-expression`.
 If the expression is `#root` or `#this`, the message is built from the original message, with the `amqp_publishConfirm` header set to `true`.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <15> The channel to which negative (`nack`) publisher confirmations are sent.
 The payload is the correlation data defined by `confirm-correlation-expression` (if there is no `ErrorMessageStrategy` configured).
 If the expression is `#root` or `#this`, the message is built from the original message, with the `amqp_publishConfirm` header set to `false`.
 When there is an `ErrorMessageStrategy`, the message is an `ErrorMessage` with a `NackedAmqpMessageException` payload.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <16> When set, the gateway will synthesize a negative acknowledgment (nack) if a publisher confirm is not received within this time in milliseconds.
 Pending confirms are checked every 50% of this value, so the actual time a nack is sent will be between 1x and 1.5x this value.
@@ -832,6 +840,7 @@ Default none (nacks will not be generated).
 When provided, the underlying AMQP template is configured to return undeliverable messages to the adapter.
 When there is no `ErrorMessageStrategy` configured, the message is constructed from the data received from AMQP, with the following additional headers: `amqp_returnReplyCode`, `amqp_returnReplyText`, `amqp_returnExchange`, and `amqp_returnRoutingKey`.
 When there is an `ErrorMessageStrategy`, the message is an `ErrorMessage` with a `ReturnedAmqpMessageException` payload.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <18> A reference to an `ErrorMessageStrategy` implementation used to build `ErrorMessage` instances when sending returned or negatively acknowledged messages.
 <19> When set to `false`, the endpoint attempts to connect to the broker during application context initialization.
@@ -1026,23 +1035,28 @@ The payload of the confirmation is the correlation data as defined by this expre
 For `nack` instances, an additional header (`amqp_publishConfirmNackCause`) is provided.
 Examples: `headers['myCorrelationData']`, `payload`.
 If the expression resolves to a `Message<?>` instance (such as "`#this`"), the message emitted on the `ack`/`nack` channel is based on that message, with the additional headers added.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <14> The channel to which positive (`ack`) publisher confirmations are sent.
 The payload is the correlation data defined by the `confirm-correlation-expression`.
 Requires the underlying `AsyncRabbitTemplate` to have its `enableConfirms` property set to `true`.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <15> Since version 4.2.
 The channel to which negative (`nack`) publisher confirmations are sent.
 The payload is the correlation data defined by the `confirm-correlation-expression`.
 Requires the underlying `AsyncRabbitTemplate` to have its `enableConfirms` property set to `true`.
+Also see <<alternative-confirms-returns>>.
 Optional (the default is `nullChannel`).
 <16> When set, the gateway will synthesize a negative acknowledgment (nack) if a publisher confirm is not received within this time in milliseconds.
 Pending confirms are checked every 50% of this value, so the actual time a nack is sent will be between 1x and 1.5x this value.
+Also see <<alternative-confirms-returns>>.
 Default none (nacks will not be generated).
 <17> The channel to which returned messages are sent.
 When provided, the underlying AMQP template is configured to return undeliverable messages to the gateway.
 The message is constructed from the data received from AMQP, with the following additional headers: `amqp_returnReplyCode`, `amqp_returnReplyText`, `amqp_returnExchange`, and `amqp_returnRoutingKey`.
 Requires the underlying `AsyncRabbitTemplate` to have its `mandatory` property set to `true`.
+Also see <<alternative-confirms-returns>>.
 Optional.
 <18> When set to `false`, the endpoint tries to connect to the broker during application context initialization.
 Doing so allows "`fail fast`" detection of bad configuration, by logging an error message if the broker is down.
@@ -1137,6 +1151,39 @@ public class AmqpAsyncApplication {
 ----
 ====
 
+[[alternative-confirms-returns]]
+=== Alternative Mechanism for Publisher Confirms and Returns
+
+When the connection factory is configured for publisher confirms and returns, the sections above discuss the configuration of message channels to receive the confirms and returns asynchronously.
+Starting with version 5.4, there is an additional mechanism which is generally easier to use.
+
+In this case, do not configure a `confirm-correlation-expression` or the confirm and return channels.
+Instead, add a `CorrelationData` instance in the `AmqpHeaders.PUBLISH_CONFIRM_CORRELATION` header; you can then wait for the result(s) later, by checking the state of the future in the `CorrelationData` instances for which you have sent messages.
+The `returnedMessage` field will always be populated (if a message is returned) before the future is completed.
+
+====
+[source, java]
+----
+CorrelationData corr = new CorrelationData("someId"); // <--- Unique "id" is required for returns
+someFlow.getInputChannel().send(MessageBuilder.withPayload("test")
+        .setHeader("rk", "someKeyThatWontRoute")
+        .setHeader(AmqpHeaders.PUBLISH_CONFIRM_CORRELATION, corr)
+        .build());
+...
+try {
+    Confirm Confirm = corr.getFuture().get(10, TimeUnit.SECONDS);
+    Message returned = corr.getReturnedMessage();
+    if (returned !- null) {
+        // meessage could not be routed
+    }
+}
+catch { ... }
+----
+====
+
+To improve performance, you may wish to send multiple messages and wait for the confirmations later, rather than one-at-a-time.
+The returned message is the raw message after conversion; you can subclass `CorrelationData` with whatever additional data you need.
+
 [[amqp-conversion-inbound]]
 === Inbound Message Conversion
 

src/reference/asciidoc/whats-new.adoc

@@ -60,3 +60,8 @@ See <<./ip.adoc#ip-collaborating-adapters,Collaborating Channel Adapters>> and <
 
 The `spring-integration-rmi` module is deprecated with no replacement and is going to be removed in the next major version.
 See <<./rmi.adoc#rmi, RMI Support>> for more information.
+
+=== AMQP Changes
+
+The outbound endpoints now have a new mechanism for handling publisher confirms and returns.
+See <<./amqp.adoc#alternative-confirms-returns,Alternative Mechanism for Publisher Confirms and Returns>> for more information.