INT-4015: Streaming Remote File Inbound Adapter

JIRA: https://jira.spring.io/browse/INT-4015
      https://jira.spring.io/browse/INT-3854

Initial commit.

Reworked to emit an input stream and use the file splitter.

Add StreamTransformer.

Add CLOSABLE_RESOURCE header so we can close the session automatically.

Implement INT-3854, FTP, SFTP

(S)FTP Namespace Changes

Docs - also fixes a PDF overflow

Polishing - PR Comments

checkstyle fixes

Polishing - Add Namespace for StreamParser

Polishing - PR Comments

spring-integration-core/src/main/java/org/springframework/integration/IntegrationMessageHeaderAccessor.java

@@ -16,6 +16,7 @@
 
 package org.springframework.integration;
 
+import java.io.Closeable;
 import java.util.Date;
 import java.util.Map;
 
@@ -50,6 +51,8 @@ public class IntegrationMessageHeaderAccessor extends MessageHeaderAccessor {
 
 	public static final String DUPLICATE_MESSAGE = "duplicateMessage";
 
+	public static final String CLOSEABLE_RESOURCE = "closableResource";
+
 	public IntegrationMessageHeaderAccessor(Message<?> message) {
 		super(message);
 	}
@@ -76,6 +79,19 @@ public class IntegrationMessageHeaderAccessor extends MessageHeaderAccessor {
 		return this.getHeader(PRIORITY, Integer.class);
 	}
 
+	/**
+	 * If the payload was created by a {@link Closeable} that needs to remain
+	 * open until the payload is consumed, the resource will be added to this
+	 * header. After the payload is consumed the {@link Closeable} should be
+	 * closed. Usually this must occur in an endpoint close to the message
+	 * origin in the flow, and in the same JVM.
+	 * @return the {@link Closeable}.
+	 * @since 4.3
+	 */
+	public Closeable getCloseableResource() {
+		return this.getHeader(CLOSEABLE_RESOURCE, Closeable.class);
+	}
+
 	@SuppressWarnings("unchecked")
 	public <T> T getHeader(String key, Class<T> type) {
 		Object value = getHeader(key);

spring-integration-core/src/main/java/org/springframework/integration/config/xml/IntegrationNamespaceHandler.java

@@ -53,6 +53,7 @@ public class IntegrationNamespaceHandler extends AbstractIntegrationNamespaceHan
 		registerBeanDefinitionParser("json-to-object-transformer", new JsonToObjectTransformerParser());
 		registerBeanDefinitionParser("payload-serializing-transformer", new PayloadSerializingTransformerParser());
 		registerBeanDefinitionParser("payload-deserializing-transformer", new PayloadDeserializingTransformerParser());
+		registerBeanDefinitionParser("stream-transformer", new StreamTransformerParser());
 		registerBeanDefinitionParser("claim-check-in", new ClaimCheckInParser());
 		registerBeanDefinitionParser("syslog-to-map-transformer", new SyslogToMapTransformerParser());
 		registerBeanDefinitionParser("claim-check-out", new ClaimCheckOutParser());

spring-integration-core/src/main/java/org/springframework/integration/config/xml/StreamTransformerParser.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.config.xml;
+
+import org.springframework.integration.transformer.StreamTransformer;
+
+/**
+ * Parser for {@code <stream-transformer/>} element.
+ *
+ * @author Gary Russell
+ * @since 4.3
+ *
+ */
+public class StreamTransformerParser extends ObjectToStringTransformerParser {
+
+	@Override
+	protected String getTransformerClassName() {
+		return StreamTransformer.class.getName();
+	}
+
+}

spring-integration-core/src/main/java/org/springframework/integration/transformer/StreamTransformer.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.InputStream;
+
+import org.springframework.integration.IntegrationMessageHeaderAccessor;
+import org.springframework.messaging.Message;
+import org.springframework.util.Assert;
+import org.springframework.util.FileCopyUtils;
+
+/**
+ * Transforms an InputStream payload to a byte[] or String (if a
+ * charset is provided).
+ *
+ * @author Gary Russell
+ * @since 4.3
+ *
+ */
+public class StreamTransformer extends AbstractTransformer {
+
+	private final String charset;
+
+	/**
+	 * Construct an instance to transform an {@link InputStream} to
+	 * a {@code byte[]}.
+	 */
+	public StreamTransformer() {
+		this(null);
+	}
+
+	/**
+	 * Construct an instance with the charset to convert the stream to a
+	 * String; if null a {@code byte[]} will be produced instead.
+	 * @param charset the charset.
+	 */
+	public StreamTransformer(String charset) {
+		this.charset = charset;
+	}
+
+	@Override
+	protected Object doTransform(Message<?> message) throws Exception {
+		Assert.isTrue(message.getPayload() instanceof InputStream, "payload must be an InputStream");
+		InputStream stream = (InputStream) message.getPayload();
+		ByteArrayOutputStream baos = new ByteArrayOutputStream();
+		FileCopyUtils.copy(stream, baos);
+		Closeable closeableResource = new IntegrationMessageHeaderAccessor(message).getCloseableResource();
+		if (closeableResource != null) {
+			closeableResource.close();
+		}
+		return this.charset == null ? baos.toByteArray() : baos.toString(this.charset);
+	}
+
+}

spring-integration-core/src/main/resources/org/springframework/integration/config/spring-integration-4.3.xsd

@@ -1826,6 +1826,8 @@
 					<xsd:element name="map-to-object-transformer" type="map-to-object-transformer-type"/>
 					<xsd:element name="object-to-json-transformer" type="object-to-json-transformer-type"/>
 					<xsd:element name="json-to-object-transformer" type="json-to-object-transformer-type"/>
+					<xsd:element name="stream-transformer" type="stream-transformer-type"/>
+					<xsd:element name="syslog-to-map-transformer" type="specialized-transformer-type"/>
 					<xsd:element name="claim-check-in" type="claimCheckInTypeChain"/>
 					<xsd:element name="claim-check-out" type="claimCheckOutTypeChain"/>
 					<xsd:element name="control-bus" type="control-bus-type"/>
@@ -2686,11 +2688,49 @@
 			</xsd:documentation>
 		</xsd:annotation>
 		<xsd:complexType>
-			<xsd:attributeGroup ref="inputOutputChannelGroup" />
-			<xsd:attribute name="id" type="xsd:string" />
+			<xsd:complexContent>
+				<xsd:extension base="specialized-transformer-type">
+					<xsd:attributeGroup ref="inputOutputChannelGroup" />
+				</xsd:extension>
+			</xsd:complexContent>
 		</xsd:complexType>
 	</xsd:element>
 
+	<xsd:element name="stream-transformer">
+		<xsd:annotation>
+			<xsd:documentation>
+				Defines a Consumer Endpoint for the
+				'org.springframework.integration.transformer.StreamTransformer'
+				that converts an 'InputStream' payload to a byte[] or String.
+				Providing a 'charset' signals that the conversion to String is
+				required.
+			</xsd:documentation>
+		</xsd:annotation>
+		<xsd:complexType>
+			<xsd:complexContent>
+				<xsd:extension base="stream-transformer-type">
+					<xsd:attributeGroup ref="inputOutputChannelGroup" />
+				</xsd:extension>
+			</xsd:complexContent>
+		</xsd:complexType>
+	</xsd:element>
+
+	<xsd:complexType name="stream-transformer-type">
+		<xsd:complexContent>
+			<xsd:extension base="specialized-transformer-type">
+				<xsd:attribute name="charset" type="xsd:string">
+					<xsd:annotation>
+						<xsd:documentation>
+							Allows you to specify the Charset (e.g., US-ASCII,
+							ISO-8859-1, UTF-8) to be used when transforming byte[].
+							None by default, meaning the payload will be byte[].
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+			</xsd:extension>
+		</xsd:complexContent>
+	</xsd:complexType>
+
     <!-- Claim Check -->
 
 	<xsd:element name="claim-check-in" type="claimCheckInType">
@@ -2789,8 +2829,7 @@
 	</xsd:complexType>
 
 	<xsd:complexType name="specialized-transformer-type">
-		<xsd:choice minOccurs="1" maxOccurs="unbounded">
-			<xsd:any processContents="strict" namespace="##other" minOccurs="0" maxOccurs="unbounded" />
+		<xsd:choice minOccurs="0" maxOccurs="1">
 			<xsd:element ref="poller" />
 		</xsd:choice>
 		<xsd:attribute name="id" type="xsd:string" />

spring-integration-core/src/test/java/org/springframework/integration/config/xml/ObjectToStringTransformerParserTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,22 +21,26 @@ import static org.junit.Assert.assertNotNull;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.integration.endpoint.AbstractEndpoint;
+import org.springframework.integration.test.util.TestUtils;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageChannel;
 import org.springframework.messaging.PollableChannel;
-import org.springframework.integration.endpoint.AbstractEndpoint;
 import org.springframework.messaging.support.GenericMessage;
-import org.springframework.integration.test.util.TestUtils;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
 /**
  * @author Mark Fisher
+ * @author Gary Russell
  */
 @ContextConfiguration
 @RunWith(SpringJUnit4ClassRunner.class)
+@DirtiesContext
 public class ObjectToStringTransformerParserTests {
 
 	@Autowired

spring-integration-core/src/test/java/org/springframework/integration/config/xml/StreamTransformerParserTests-context.xml

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans:beans xmlns="http://www.springframework.org/schema/integration"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:beans="http://www.springframework.org/schema/beans"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans
+			http://www.springframework.org/schema/beans/spring-beans.xsd
+			http://www.springframework.org/schema/integration
+			http://www.springframework.org/schema/integration/spring-integration.xsd">
+
+	<channel id="directInput"/>
+
+	<channel id="queueInput">
+		<queue capacity="1"/>
+	</channel>
+
+	<channel id="output">
+		<queue capacity="1"/>
+	</channel>
+
+	<stream-transformer input-channel="directInput" output-channel="output"/>
+
+	<stream-transformer input-channel="queueInput" output-channel="output">
+		<poller fixed-delay="10000"/>
+	</stream-transformer>
+
+	<chain input-channel="charsetChannel" output-channel="output">
+		<stream-transformer id="withCharset" charset="UTF-8" />
+	</chain>
+
+</beans:beans>

spring-integration-core/src/test/java/org/springframework/integration/config/xml/StreamTransformerParserTests.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.config.xml;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageChannel;
+import org.springframework.messaging.PollableChannel;
+import org.springframework.messaging.support.GenericMessage;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+/**
+ * @author Mark Fisher
+ * @author Gary Russell
+ */
+@ContextConfiguration
+@RunWith(SpringJUnit4ClassRunner.class)
+@DirtiesContext
+public class StreamTransformerParserTests {
+
+	@Autowired
+	@Qualifier("directInput")
+	private MessageChannel directInput;
+
+	@Autowired
+	@Qualifier("charsetChannel")
+	private MessageChannel charsetChannel;
+
+	@Autowired
+	@Qualifier("queueInput")
+	private MessageChannel queueInput;
+
+	@Autowired
+	@Qualifier("output")
+	private PollableChannel output;
+
+	@Test
+	public void directChannelWithStringMessage() {
+		this.directInput.send(new GenericMessage<InputStream>(new ByteArrayInputStream("foo".getBytes())));
+		Message<?> result = output.receive(0);
+		assertNotNull(result);
+		assertArrayEquals("foo".getBytes(), (byte[]) result.getPayload());
+	}
+
+	@Test
+	public void queueChannelWithStringMessage() {
+		this.queueInput.send(new GenericMessage<InputStream>(new ByteArrayInputStream("foo".getBytes())));
+		Message<?> result = output.receive(3000);
+		assertNotNull(result);
+		assertArrayEquals("foo".getBytes(), (byte[]) result.getPayload());
+	}
+
+	@Test
+	public void charset() {
+		this.charsetChannel.send(new GenericMessage<InputStream>(new ByteArrayInputStream("foo".getBytes())));
+		Message<?> result = output.receive(0);
+		assertNotNull(result);
+		assertEquals("foo", result.getPayload());
+	}
+
+}

spring-integration-core/src/test/java/org/springframework/integration/config/xml/SyslogTransformerParserTests-context.xml

@@ -7,7 +7,16 @@
 
 	<int:syslog-to-map-transformer id="toMap" input-channel="toMapChannel" output-channel="out" />
 
+	<int:syslog-to-map-transformer id="withPollerContextLoads" input-channel="nullChannel" output-channel="nullChannel">
+		<int:poller fixed-delay="50000" />
+	</int:syslog-to-map-transformer>
+
+	<int:chain input-channel="toMapChannel" output-channel="out">
+		<int:syslog-to-map-transformer />
+	</int:chain>
+
 	<int:channel id="out">
 		<int:queue />
 	</int:channel>
+
 </beans>