INT-4098: IMAP Content Rendering Consistency

JIRA: https://jira.spring.io/browse/INT-4098

Polishing - What's New

Schema Polishing

Also fix a test to align it with changes to the test mail server.

spring-integration-mail/src/main/java/org/springframework/integration/mail/AbstractMailReceiver.java

@@ -17,6 +17,7 @@
 package org.springframework.integration.mail;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.util.Date;
 import java.util.LinkedList;
@@ -104,6 +105,8 @@ public abstract class AbstractMailReceiver extends IntegrationObjectSupport impl
 
 	private volatile boolean embeddedPartsAsBytes = true;
 
+	private volatile boolean simpleContent;
+
 	public AbstractMailReceiver() {
 		this.url = null;
 	}
@@ -244,6 +247,38 @@ public abstract class AbstractMailReceiver extends IntegrationObjectSupport impl
 		this.embeddedPartsAsBytes = embeddedPartsAsBytes;
 	}
 
+	/**
+	 * {@link MimeMessage#getContent()} returns just the email body.
+	 *
+	 * <pre class="code">
+	 * foo
+	 * </pre>
+	 *
+	 * Some subclasses, such as {@code IMAPMessage} return some headers with the body.
+	 *
+	 * <pre class="code">
+	 * To: foo@bar
+	 * From: bar@baz
+	 * Subject: Test Email
+	 *
+	 *  foo
+	 * </pre>
+	 *
+	 * Starting with version 5.0, messages emitted by mail receivers will render the
+	 * content in the same way as the {@link MimeMessage} implementation returned by
+	 * javamail. In versions 2.2 through 4.3, the content was always just the body,
+	 * regardless of the underlying message type (unless a header mapper was provided,
+	 * in which case the payload was rendered by the underlying {@link MimeMessage}.
+	 * <p>To revert to the previous behavior, set this flag to true. In addition, even
+	 * if a header mapper is provided, the payload will just be the email body.
+	 * @param simpleContent true to render simple content.
+	 *
+	 * @since 5.0
+	 */
+	public void setSimpleContent(boolean simpleContent) {
+		this.simpleContent = simpleContent;
+	}
+
 	protected Folder getFolder() {
 		return this.folder;
 	}
@@ -366,7 +401,14 @@ public abstract class AbstractMailReceiver extends IntegrationObjectSupport impl
 	private Object extractContent(MimeMessage message, Map<String, Object> headers) {
 		Object content;
 		try {
-			content = message.getContent();
+			MimeMessage theMessage;
+			if (this.simpleContent) {
+				theMessage = new IntegrationMimeMessage(message);
+			}
+			else {
+				theMessage = message;
+			}
+			content = theMessage.getContent();
 			if (content instanceof String) {
 				String mailContentType = (String) headers.get(MailHeaders.CONTENT_TYPE);
 				if (mailContentType != null && mailContentType.toLowerCase().startsWith("text")) {
@@ -553,9 +595,25 @@ public abstract class AbstractMailReceiver extends IntegrationObjectSupport impl
 
 		private final MimeMessage source;
 
+		private final Object content;
+
 		private IntegrationMimeMessage(MimeMessage source) throws MessagingException {
 			super(source);
 			this.source = source;
+			if (AbstractMailReceiver.this.simpleContent) {
+				this.content = null;
+			}
+			else {
+				Object complexContent;
+				try {
+					complexContent = source.getContent();
+				}
+				catch (IOException e) {
+					complexContent = "Unable to extract content; see logs: " + e.getMessage();
+					AbstractMailReceiver.this.logger.error("Failed to extract content from " + source, e);
+				}
+				this.content = complexContent;
+			}
 		}
 
 		@Override
@@ -584,6 +642,16 @@ public abstract class AbstractMailReceiver extends IntegrationObjectSupport impl
 			return this.source.getLineCount();
 		}
 
+		@Override
+		public Object getContent() throws IOException, MessagingException {
+			if (AbstractMailReceiver.this.simpleContent) {
+				return super.getContent();
+			}
+			else {
+				return this.content;
+			}
+		}
+
 	}
 
 }

spring-integration-mail/src/main/java/org/springframework/integration/mail/config/ImapIdleChannelAdapterParser.java

@@ -98,6 +98,7 @@ public class ImapIdleChannelAdapterParser extends AbstractChannelAdapterParser {
 		}
 		IntegrationNamespaceUtils.setReferenceIfAttributeDefined(receiverBuilder, element, "header-mapper");
 		IntegrationNamespaceUtils.setValueIfAttributeDefined(receiverBuilder, element, "embedded-parts-as-bytes");
+		IntegrationNamespaceUtils.setValueIfAttributeDefined(receiverBuilder, element, "simple-content");
 
 		return receiverBuilder.getBeanDefinition();
 	}

spring-integration-mail/src/main/java/org/springframework/integration/mail/config/MailInboundChannelAdapterParser.java

@@ -95,6 +95,7 @@ public class MailInboundChannelAdapterParser extends AbstractPollingInboundChann
 		}
 		IntegrationNamespaceUtils.setReferenceIfAttributeDefined(receiverBuilder, element, "header-mapper");
 		IntegrationNamespaceUtils.setValueIfAttributeDefined(receiverBuilder, element, "embedded-parts-as-bytes");
+		IntegrationNamespaceUtils.setValueIfAttributeDefined(receiverBuilder, element, "simple-content");
 
 		return receiverBuilder.getBeanDefinition();
 	}

spring-integration-mail/src/main/java/org/springframework/integration/mail/config/MailReceiverFactoryBean.java

@@ -85,6 +85,8 @@ public class MailReceiverFactoryBean implements FactoryBean<MailReceiver>, Dispo
 
 	private Boolean embeddedPartsAsBytes;
 
+	private Boolean simpleContent;
+
 	public void setStoreUri(String storeUri) {
 		this.storeUri = storeUri;
 	}
@@ -141,6 +143,10 @@ public class MailReceiverFactoryBean implements FactoryBean<MailReceiver>, Dispo
 		this.embeddedPartsAsBytes = embeddedPartsAsBytes;
 	}
 
+	public void setSimpleContent(Boolean simpleContent) {
+		this.simpleContent = simpleContent;
+	}
+
 	@Override
 	public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
 		this.beanFactory = beanFactory;
@@ -230,6 +236,9 @@ public class MailReceiverFactoryBean implements FactoryBean<MailReceiver>, Dispo
 		if (this.embeddedPartsAsBytes != null) {
 			receiver.setEmbeddedPartsAsBytes(this.embeddedPartsAsBytes);
 		}
+		if (this.simpleContent != null) {
+			receiver.setSimpleContent(this.simpleContent);
+		}
 		receiver.afterPropertiesSet();
 		return receiver;
 	}

spring-integration-mail/src/main/resources/org/springframework/integration/mail/config/spring-integration-mail-5.0.xsd

@@ -269,6 +269,22 @@
 				]]></xsd:documentation>
 			</xsd:annotation>
 		</xsd:attribute>
+		<xsd:attribute name="simple-content">
+			<xsd:annotation>
+				<xsd:documentation>
+	When 'true', messages produced by the source will be rendered by 'MimeMessage.getContent()'
+	which is usually just the body for a simple text email. When false (default) the content
+	is rendered by the 'getContent()' method on the actual message returned by the underlying
+	javamail implementation.
+	For example, an IMAP message is rendered with some message headers.
+	This attribute is provided so that users can enable the previous behavior, which just
+	rendered the body.
+				</xsd:documentation>
+			</xsd:annotation>
+			<xsd:simpleType>
+				<xsd:union memberTypes="xsd:boolean xsd:string"/>
+			</xsd:simpleType>
+		</xsd:attribute>
 		<xsd:attributeGroup ref="inboundMappingGroup" />
 	</xsd:complexType>
 

spring-integration-mail/src/test/java/org/springframework/integration/mail/ImapMailReceiverTests.java

@@ -159,7 +159,29 @@ public class ImapMailReceiverTests {
 		testIdleWithServerGuts(receiver, true);
 	}
 
-	public void testIdleWithServerGuts(ImapMailReceiver receiver, boolean mapped) throws MessagingException {
+	@Test
+	public void testIdleWithServerDefaultSearchSimple() throws Exception {
+		ImapMailReceiver receiver = new ImapMailReceiver("imap://user:pw@localhost:" + imapIdleServer.getPort()
+				+ "/INBOX");
+		receiver.setSimpleContent(true);
+		testIdleWithServerGuts(receiver, false, true);
+		assertTrue(imapIdleServer.assertReceived("searchWithUserFlag"));
+	}
+
+	@Test
+	public void testIdleWithMessageMappingSimple() throws Exception {
+		ImapMailReceiver receiver = new ImapMailReceiver("imap://user:pw@localhost:" + imapIdleServer.getPort()
+				+ "/INBOX");
+		receiver.setSimpleContent(true);
+		receiver.setHeaderMapper(new DefaultMailHeaderMapper());
+		testIdleWithServerGuts(receiver, true, true);
+	}
+
+	public void testIdleWithServerGuts(ImapMailReceiver receiver, boolean mapped) throws Exception {
+		testIdleWithServerGuts(receiver, mapped, false);
+	}
+
+	public void testIdleWithServerGuts(ImapMailReceiver receiver, boolean mapped, boolean simple) throws Exception {
 		imapIdleServer.resetServer();
 		Properties mailProps = new Properties();
 		mailProps.put("mail.debug", "true");
@@ -187,6 +209,14 @@ public class ImapMailReceiverTests {
 			assertNotNull(received);
 			assertNotNull(received.getPayload().getReceivedDate());
 			assertTrue(received.getPayload().getLineCount() > -1);
+			if (simple) {
+				assertThat(received.getPayload().getContent(),
+						equalTo(TestMailServer.MailServer.MailHandler.BODY + "\r\n"));
+			}
+			else {
+				assertThat(received.getPayload().getContent(),
+						equalTo(TestMailServer.MailServer.MailHandler.MESSAGE + "\r\n"));
+			}
 		}
 		else {
 			org.springframework.messaging.Message<?> received = channel.receive(10000);
@@ -199,6 +229,12 @@ public class ImapMailReceiverTests {
 			assertThat((String) received.getHeaders().get(MailHeaders.FROM), equalTo("Bar <bar@baz>"));
 			assertThat(((String[]) received.getHeaders().get(MailHeaders.TO))[0], equalTo("Foo <foo@bar>"));
 			assertThat((String) received.getHeaders().get(MailHeaders.SUBJECT), equalTo("Test Email"));
+			if (simple) {
+				assertThat(received.getPayload(), equalTo(TestMailServer.MailServer.MailHandler.BODY + "\r\n"));
+			}
+			else {
+				assertThat(received.getPayload(), equalTo(TestMailServer.MailServer.MailHandler.MESSAGE + "\r\n"));
+			}
 		}
 		assertNotNull(channel.receive(10000)); // new message after idle
 		assertNull(channel.receive(10000)); // no new message after second and third idle

spring-integration-mail/src/test/java/org/springframework/integration/mail/config/ImapIdleChannelAdapterParserTests-context.xml

@@ -36,6 +36,7 @@
 			channel="channel"
 			auto-startup="false"
 			header-mapper="mapper"
+			simple-content="true"
 			embedded-parts-as-bytes="false"
 			should-delete-messages="true"/>
 

spring-integration-mail/src/test/java/org/springframework/integration/mail/config/ImapIdleChannelAdapterParserTests.java

@@ -86,6 +86,7 @@ public class ImapIdleChannelAdapterParserTests {
 		assertNull(adapterAccessor.getPropertyValue("adviceChain"));
 		assertEquals(Boolean.FALSE, receiverAccessor.getPropertyValue("embeddedPartsAsBytes"));
 		assertNotNull(receiverAccessor.getPropertyValue("headerMapper"));
+		assertEquals(Boolean.TRUE, receiverAccessor.getPropertyValue("simpleContent"));
 	}
 
 	@Test
@@ -109,6 +110,7 @@ public class ImapIdleChannelAdapterParserTests {
 		assertSame(context.getBean("errorChannel"), adapterAccessor.getPropertyValue("errorChannel"));
 		assertEquals(Boolean.TRUE, receiverAccessor.getPropertyValue("embeddedPartsAsBytes"));
 		assertNull(receiverAccessor.getPropertyValue("headerMapper"));
+		assertEquals(Boolean.FALSE, receiverAccessor.getPropertyValue("simpleContent"));
 	}
 
 	@Test

spring-integration-mail/src/test/java/org/springframework/integration/mail/config/InboundChannelAdapterParserTests-context.xml

@@ -24,6 +24,7 @@
 
 	<mail:inbound-channel-adapter id="imapShouldDeleteTrue"
 		store-uri="imap:test" channel="testChannel" should-delete-messages="true"
+		simple-content="true"
 		auto-startup="false" />
 
 	<mail:inbound-channel-adapter id="imapShouldDeleteFalse"

spring-integration-mail/src/test/java/org/springframework/integration/mail/config/InboundChannelAdapterParserTests.java

@@ -105,16 +105,20 @@ public class InboundChannelAdapterParserTests {
 	public void imapShouldDeleteTrue() {
 		AbstractMailReceiver receiver = this.getReceiver("imapShouldDeleteTrue");
 		assertEquals(ImapMailReceiver.class, receiver.getClass());
-		Boolean value = (Boolean) new DirectFieldAccessor(receiver).getPropertyValue("shouldDeleteMessages");
+		DirectFieldAccessor receiverAccessor = new DirectFieldAccessor(receiver);
+		Boolean value = (Boolean) receiverAccessor.getPropertyValue("shouldDeleteMessages");
 		assertTrue(value);
+		assertEquals(Boolean.TRUE, receiverAccessor.getPropertyValue("simpleContent"));
 	}
 
 	@Test
 	public void imapShouldDeleteFalse() {
 		AbstractMailReceiver receiver = this.getReceiver("imapShouldDeleteFalse");
 		assertEquals(ImapMailReceiver.class, receiver.getClass());
-		Boolean value = (Boolean) new DirectFieldAccessor(receiver).getPropertyValue("shouldDeleteMessages");
+		DirectFieldAccessor receiverAccessor = new DirectFieldAccessor(receiver);
+		Boolean value = (Boolean) receiverAccessor.getPropertyValue("shouldDeleteMessages");
 		assertFalse(value);
+		assertEquals(Boolean.FALSE, receiverAccessor.getPropertyValue("simpleContent"));
 	}
 
 

spring-integration-mail/src/test/java/org/springframework/integration/mail/config/Pop3Tests.java

@@ -67,7 +67,7 @@ public class Pop3Tests {
 		assertEquals("Foo <foo@bar>", headers.get(MailHeaders.TO, String[].class)[0]);
 		assertEquals("Bar <bar@baz>", headers.get(MailHeaders.FROM));
 		assertEquals("Test Email", headers.get(MailHeaders.SUBJECT));
-		assertEquals("foo\r\n", message.getPayload());
+		assertEquals("foo\r\n\r\n", message.getPayload());
 	}
 
 }

spring-integration-test/src/main/java/org/springframework/integration/test/mail/TestMailServer.java

@@ -440,8 +440,10 @@ public class TestMailServer {
 
 		public abstract class MailHandler implements Runnable {
 
-			protected static final String MESSAGE =
-					"To: Foo <foo@bar>\r\nFrom: Bar <bar@baz>\r\nSubject: Test Email\r\n\r\nfoo";
+			public static final String BODY = "foo\r\n";
+
+			public static final String MESSAGE =
+					"To: Foo <foo@bar>\r\nFrom: Bar <bar@baz>\r\nSubject: Test Email\r\n\r\n" + BODY;
 
 			protected final Socket socket;
 

src/reference/asciidoc/mail.adoc

@@ -55,6 +55,31 @@ Spring Integration provides the `ImapIdleChannelAdapter` which is itself a Messa
 It delegates to an instance of the `ImapMailReceiver` but enables asynchronous reception of Mail Messages.
 There are examples in the next section of configuring both types of inbound Channel Adapter with Spring Integration's namespace support in the 'mail' schema.
 
+[[imap-format-important]]
+[IMPORTANT]
+====
+Normally, when `IMAPMessage.getContent()` method is called, certain headers as well as the body are rendered (for a simple text email):
+
+[source]
+----
+To: foo@bar
+From: bar@baz
+Subject: Test Email
+
+foo
+----
+
+With a simple `MimeMessage`, `getContent()` just returns the mail body (`foo` in this case).
+
+Starting with _version 2.2_, the framework eagerly fetches IMAP messages and exposes them as an internal subclass of `MimeMessage`.
+This had the undesired side effect of changing the `getContent()` behavior.
+This inconsistency was further exacerbated by the <<mail-mapping, Mail Mapping>> enhancement in _version 4.3_ in that, when a header mapper was provided, the payload was rendered by the `IMAPMessage.getContent()` method.
+This meant that IMAP content differed depending on whether or not a header mapper was provided.
+Starting with _version 5.0_, messages originating from an IMAP source will now render the content in accordance with `IMAPMessage.getContent()` behavior, regardless of whether a header mapper is provided.
+If you are not using a header mapper, and you wish to revert to the previous behavior of just rendering the body, set the `simpleContent` boolean property on the mail receiver to `true`.
+This property now controls the rendering regardless of whether a header mapper is used; it now allows the simple body-only rendering when a header mapper is provided.
+====
+
 [[mail-mapping]]
 === Inbound Mail Message Mapping
 

src/reference/asciidoc/whats-new.adoc

@@ -28,3 +28,8 @@ The gateway now correctly sets the `errorChannel` header when the gateway method
 Previously, the header was not populated.
 This had the effect that synchronous downstream flows (running on the calling thread) would send the exception to the configured channel but an exception on an async downstream flow would be sent to the default `errorChannel` instead.
 See <<gateway-error-handling>> for more information.
+
+==== Mail Changes
+
+Some inconsistencies with rendering IMAP mail content have been resolved.
+See <<imap-format-important, the note in the Mail-Receiving Channel Adapter Section>> for more information.