INT-4255: JSON Embedded Headers Message Mapper

JIRA: https://jira.spring.io/browse/INT-4255

- Support embedding headers for transports that don't support headers (TCP, Kafka, etc)
- Use the new message-aware Jackson ObjectMapper
- Provide a mechanism to more efficiently support byte[] payloads (avoid Base64 encoding)
- Support decoding "legacy" SCSt embedded headers

Polishing; PR Comments - Add Support to MQTT and TCP

Switch to simple patterns instead of regex

* Fix JavaDoc typo
* Upgrade to Jackson 2.9.1

spring-integration-mqtt/src/main/java/org/springframework/integration/mqtt/support/DefaultPahoMessageConverter.java

@@ -21,11 +21,13 @@ import org.eclipse.paho.client.mqttv3.MqttMessage;
 import org.springframework.beans.factory.BeanFactory;
 import org.springframework.beans.factory.BeanFactoryAware;
 import org.springframework.integration.handler.MessageProcessor;
+import org.springframework.integration.mapping.BytesMessageMapper;
 import org.springframework.integration.support.AbstractIntegrationMessageBuilder;
 import org.springframework.integration.support.DefaultMessageBuilderFactory;
 import org.springframework.integration.support.MessageBuilderFactory;
 import org.springframework.integration.support.utils.IntegrationUtils;
 import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageHandlingException;
 import org.springframework.messaging.MessageHeaders;
 import org.springframework.messaging.converter.MessageConversionException;
 import org.springframework.util.Assert;
@@ -51,6 +53,8 @@ public class DefaultPahoMessageConverter implements MqttMessageConverter, BeanFa
 
 	private final MessageProcessor<Boolean> retainedProcessor;
 
+	private BytesMessageMapper bytesMessageMapper;
+
 	private volatile boolean payloadAsBytes = false;
 
 	private volatile BeanFactory beanFactory;
@@ -164,8 +168,10 @@ public class DefaultPahoMessageConverter implements MqttMessageConverter, BeanFa
 
 	/**
 	 * True if the converter should not convert the message payload to a String.
+	 * Ignored if a {@link BytesMessageMapper} is provided.
 	 *
 	 * @param payloadAsBytes The payloadAsBytes to set.
+	 * @see #setBytesMessageMapper(BytesMessageMapper)
 	 */
 	public void setPayloadAsBytes(boolean payloadAsBytes) {
 		this.payloadAsBytes = payloadAsBytes;
@@ -175,6 +181,18 @@ public class DefaultPahoMessageConverter implements MqttMessageConverter, BeanFa
 		return this.payloadAsBytes;
 	}
 
+	/**
+	 * Set a {@link BytesMessageMapper} to use when mapping byte[].
+	 * {@link #setPayloadAsBytes(boolean)} is ignored when a {@link BytesMessageMapper}
+	 * is provided.
+	 * @param bytesMessageMapper the mapper.
+	 * @since 5.0
+	 * @see #setPayloadAsBytes(boolean)
+	 */
+	public void setBytesMessageMapper(BytesMessageMapper bytesMessageMapper) {
+		this.bytesMessageMapper = bytesMessageMapper;
+	}
+
 	@Override
 	public Message<?> toMessage(Object mqttMessage, MessageHeaders headers) {
 		Assert.isInstanceOf(MqttMessage.class, mqttMessage,
@@ -183,11 +201,20 @@ public class DefaultPahoMessageConverter implements MqttMessageConverter, BeanFa
 		return toMessage(null, (MqttMessage) mqttMessage);
 	}
 
+	@SuppressWarnings("unchecked")
 	@Override
 	public Message<?> toMessage(String topic, MqttMessage mqttMessage) {
 		try {
-			AbstractIntegrationMessageBuilder<Object> messageBuilder = getMessageBuilderFactory()
-					.withPayload(mqttBytesToPayload(mqttMessage))
+			AbstractIntegrationMessageBuilder<Object> messageBuilder;
+			if (this.bytesMessageMapper != null) {
+				messageBuilder = (AbstractIntegrationMessageBuilder<Object>) getMessageBuilderFactory()
+						.fromMessage(this.bytesMessageMapper.toMessage(mqttMessage.getPayload()));
+			}
+			else {
+				messageBuilder = getMessageBuilderFactory()
+					.withPayload(mqttBytesToPayload(mqttMessage));
+			}
+			messageBuilder
 					.setHeader(MqttHeaders.RECEIVED_QOS, mqttMessage.getQos())
 					.setHeader(MqttHeaders.DUPLICATE, mqttMessage.isDuplicate())
 					.setHeader(MqttHeaders.RECEIVED_RETAINED, mqttMessage.isRetained());
@@ -232,29 +259,42 @@ public class DefaultPahoMessageConverter implements MqttMessageConverter, BeanFa
 	/**
 	 * Subclasses can override this method to convert the payload to a byte[].
 	 * The default implementation accepts a byte[] or String payload.
+	 * If a {@link BytesMessageMapper} is provided, conversion to byte[]
+	 * is delegated to it, so any payload that it can handle is supported.
 	 *
 	 * @param message The outbound Message.
 	 * @return The byte[] which will become the payload of the MQTT Message.
 	 */
 	protected byte[] messageToMqttBytes(Message<?> message) {
-		Object payload = message.getPayload();
-		Assert.isTrue(payload instanceof byte[] || payload instanceof String,
-				() -> "This default converter can only handle 'byte[]' or 'String' payloads; consider adding a "
-						+ "transformer to your flow definition, or subclass this converter for "
-						+ payload.getClass().getName() + " payloads");
-		byte[] payloadBytes;
-		if (payload instanceof String) {
+		if (this.bytesMessageMapper != null) {
 			try {
-				payloadBytes = ((String) payload).getBytes(this.charset);
+				return this.bytesMessageMapper.fromMessage(message);
 			}
 			catch (Exception e) {
-				throw new MessageConversionException("failed to convert Message to object", e);
+				throw new MessageHandlingException(message, "Failed to map outbound message", e);
 			}
 		}
 		else {
-			payloadBytes = (byte[]) payload;
+			Object payload = message.getPayload();
+			Assert.isTrue(payload instanceof byte[] || payload instanceof String,
+					() -> "This default converter can only handle 'byte[]' or 'String' payloads; consider adding a "
+							+ "transformer to your flow definition, or provide a BytesMessageMapper, "
+							+ "or subclass this converter for "
+							+ payload.getClass().getName() + " payloads");
+			byte[] payloadBytes;
+			if (payload instanceof String) {
+				try {
+					payloadBytes = ((String) payload).getBytes(this.charset);
+				}
+				catch (Exception e) {
+					throw new MessageConversionException("failed to convert Message to object", e);
+				}
+			}
+			else {
+				payloadBytes = (byte[]) payload;
+			}
+			return payloadBytes;
 		}
-		return payloadBytes;
 	}
 
 }

spring-integration-mqtt/src/test/java/org/springframework/integration/mqtt/BackToBackAdapterTests.java

@@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 
+import java.util.Collections;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -45,8 +46,11 @@ import org.springframework.integration.mqtt.event.MqttMessageDeliveredEvent;
 import org.springframework.integration.mqtt.event.MqttMessageSentEvent;
 import org.springframework.integration.mqtt.inbound.MqttPahoMessageDrivenChannelAdapter;
 import org.springframework.integration.mqtt.outbound.MqttPahoMessageHandler;
+import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter;
 import org.springframework.integration.mqtt.support.MqttHeaders;
 import org.springframework.integration.support.MessageBuilder;
+import org.springframework.integration.support.json.EmbeddedJsonHeadersMessageMapper;
+import org.springframework.integration.support.json.JacksonJsonUtils;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageChannel;
 import org.springframework.messaging.MessagingException;
@@ -104,6 +108,39 @@ public class BackToBackAdapterTests {
 		assertEquals("mqtt-foo", out.getHeaders().get(MqttHeaders.RECEIVED_TOPIC));
 	}
 
+	@Test
+	public void testJson() {
+		MqttPahoMessageHandler adapter = new MqttPahoMessageHandler("tcp://localhost:1883", "si-test-out");
+		adapter.setDefaultTopic("mqtt-foo");
+		adapter.setBeanFactory(mock(BeanFactory.class));
+		EmbeddedJsonHeadersMessageMapper mapper = new EmbeddedJsonHeadersMessageMapper(
+				JacksonJsonUtils.messagingAwareMapper("org.springframework"));
+		DefaultPahoMessageConverter converter = new DefaultPahoMessageConverter();
+		converter.setBytesMessageMapper(mapper);
+		adapter.setConverter(converter);
+		adapter.afterPropertiesSet();
+		adapter.start();
+		MqttPahoMessageDrivenChannelAdapter inbound = new MqttPahoMessageDrivenChannelAdapter("tcp://localhost:1883",
+				"si-test-in", "mqtt-foo");
+		QueueChannel outputChannel = new QueueChannel();
+		inbound.setOutputChannel(outputChannel);
+		ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler();
+		taskScheduler.initialize();
+		inbound.setTaskScheduler(taskScheduler);
+		inbound.setBeanFactory(mock(BeanFactory.class));
+		inbound.setConverter(converter);
+		inbound.afterPropertiesSet();
+		inbound.start();
+		adapter.handleMessage(new GenericMessage<Foo>(new Foo("bar"), Collections.singletonMap("baz", "qux")));
+		adapter.stop();
+		Message<?> out = outputChannel.receive(10000);
+		assertNotNull(out);
+		inbound.stop();
+		assertEquals(new Foo("bar"), out.getPayload());
+		assertEquals("mqtt-foo", out.getHeaders().get(MqttHeaders.RECEIVED_TOPIC));
+		assertEquals("qux", out.getHeaders().get("baz"));
+	}
+
 	@Test
 	public void testAddRemoveTopic() {
 		MqttPahoMessageHandler adapter = new MqttPahoMessageHandler("tcp://localhost:1883", "si-test-out");
@@ -355,4 +392,57 @@ public class BackToBackAdapterTests {
 
 	}
 
+	public static class Foo {
+
+		private String bar;
+
+		public Foo() {
+			super();
+		}
+
+		public Foo(String bar) {
+			this.bar = bar;
+		}
+
+		public String getBar() {
+			return this.bar;
+		}
+
+		public void setBar(String bar) {
+			this.bar = bar;
+		}
+
+		@Override
+		public int hashCode() {
+			final int prime = 31;
+			int result = 1;
+			result = prime * result + ((this.bar == null) ? 0 : this.bar.hashCode());
+			return result;
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			if (this == obj) {
+				return true;
+			}
+			if (obj == null) {
+				return false;
+			}
+			if (getClass() != obj.getClass()) {
+				return false;
+			}
+			Foo other = (Foo) obj;
+			if (this.bar == null) {
+				if (other.bar != null) {
+					return false;
+				}
+			}
+			else if (!this.bar.equals(other.bar)) {
+				return false;
+			}
+			return true;
+		}
+
+	}
+
 }