INT-3737: (S)FTP Outbound Gateway Streaming GET

JIRA: https://jira.spring.io/browse/INT-3737

Support returning an `InputStream` from a GET operation.

This can be used in conjuction with a `<file:splitter/>` to stream a text file.

The user is responsible for releasing the session when the download is complete.

The session object is stored in the message header and `RemoteFileUtils.closeSession()` should be called
to clean up and close the session.

INT-3737: Polishing and Docs

Fix `CachedSession.close()` bug

Preventing `Caused by: java.io.IOException: Previous raw read was not finalized`
when `Session` is returned to the pool, but `readingRaw` hasn't been finalized
because the real `close()` isn't invoked in case of normal return to pool.

spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2013 the original author or authors.
+ * Copyright 2002-2015 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,8 @@ public abstract class FileHeaders {
 
 	public static final String REMOTE_FILE = PREFIX + "remoteFile";
 
+	public static final String REMOTE_SESSION = PREFIX + "remoteSession";
+
 	public static final String RENAME_TO = PREFIX + "renameTo";
 
 }

spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2014 the original author or authors.
+ * Copyright 2013-2015 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -98,6 +98,14 @@ public class RemoteFileTemplate<F> implements RemoteFileOperations<F>, Initializ
 		this.sessionFactory = sessionFactory;
 	}
 
+	/**
+	 * @return this template's {@link SessionFactory}.
+	 * @since 4.2
+	 */
+	public SessionFactory<F> getSessionFactory() {
+		return sessionFactory;
+	}
+
 	/**
 	 * Determine whether the remote directory should automatically be created when
 	 * sending files to the remote system.

spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java

@@ -40,12 +40,14 @@ import org.springframework.integration.file.FileHeaders;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.AbstractFileInfo;
 import org.springframework.integration.file.remote.RemoteFileTemplate;
+import org.springframework.integration.file.remote.RemoteFileUtils;
 import org.springframework.integration.file.remote.SessionCallback;
 import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.file.remote.session.SessionFactory;
 import org.springframework.integration.file.support.FileExistsMode;
 import org.springframework.integration.handler.AbstractReplyProducingMessageHandler;
 import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor;
+import org.springframework.integration.support.AbstractIntegrationMessageBuilder;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageHandlingException;
 import org.springframework.messaging.MessagingException;
@@ -170,7 +172,12 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
 		/**
 		 * Recursive (ls, mget)
 		 */
-		RECURSIVE("-R");
+		RECURSIVE("-R"),
+
+		/**
+		 * Streaming 'get' (returns InputStream); user must call {@link RemoteFileUtils#closeSession(Session)}.
+		 */
+		STREAM("-stream");
 
 		private String option;
 
@@ -364,7 +371,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
 				Command.GET.equals(this.command)) {
 			Assert.isNull(this.filter, "Filters are not supported with the rm and get commands");
 		}
-		if (Command.GET.equals(this.command)
+		if ((Command.GET.equals(this.command) && !options.contains(Option.STREAM))
 				|| Command.MGET.equals(this.command)) {
 			Assert.notNull(this.localDirectoryExpression, "localDirectory must not be null");
 			if (this.localDirectoryExpression instanceof LiteralExpression) {
@@ -449,19 +456,37 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
 		final String remoteFilePath =  this.fileNameProcessor.processMessage(requestMessage);
 		final String remoteFilename = this.getRemoteFilename(remoteFilePath);
 		final String remoteDir = this.getRemoteDirectory(remoteFilePath, remoteFilename);
-		File payload = this.remoteFileTemplate.execute(new SessionCallback<F, File>() {
-
-			@Override
-			public File doInSession(Session<F> session) throws IOException {
-				return AbstractRemoteFileOutboundGateway.this.get(requestMessage, session, remoteDir, remoteFilePath,
-						remoteFilename, true);
-
+		Session<F> session = null;
+		Object payload;
+		if (this.options.contains(Option.STREAM)) {
+			session = this.remoteFileTemplate.getSessionFactory().getSession();
+			try {
+				payload = session.readRaw(remoteFilePath);
 			}
-		});
-		return this.getMessageBuilderFactory().withPayload(payload)
+			catch (IOException e) {
+				throw new MessageHandlingException(requestMessage, "Failed to get the remote file ["
+						+ remoteFilePath
+						+ "] as a stream", e);
+			}
+		}
+		else {
+			payload = this.remoteFileTemplate.execute(new SessionCallback<F, File>() {
+
+				@Override
+				public File doInSession(Session<F> session) throws IOException {
+					return AbstractRemoteFileOutboundGateway.this.get(requestMessage, session, remoteDir, remoteFilePath,
+							remoteFilename, true);
+
+				}
+			});
+		}
+		AbstractIntegrationMessageBuilder<Object> builder = this.getMessageBuilderFactory().withPayload(payload)
 			.setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir)
-			.setHeader(FileHeaders.REMOTE_FILE, remoteFilename)
-			.build();
+			.setHeader(FileHeaders.REMOTE_FILE, remoteFilename);
+		if (session != null) {
+			builder.setHeader(FileHeaders.REMOTE_SESSION, session);
+		}
+		return builder.build();
 	}
 
 	private Object doMget(final Message<?> requestMessage) {

spring-integration-file/src/main/java/org/springframework/integration/file/remote/session/CachingSessionFactory.java

@@ -190,7 +190,15 @@ public class CachingSessionFactory<F> implements SessionFactory<F>, DisposableBe
 				else if (this.dirty) {
 					this.targetSession.close();
 				}
-				pool.releaseItem(targetSession);
+				if (this.targetSession.isOpen()) {
+					try {
+						this.targetSession.finalizeRaw();
+					}
+					catch (IOException e) {
+						//No-op in this context
+					}
+				}
+				pool.releaseItem(this.targetSession);
 				released = true;
 			}
 		}

spring-integration-file/src/main/java/org/springframework/integration/file/splitter/FileSplitter.java

@@ -38,6 +38,7 @@ import org.springframework.integration.file.splitter.FileSplitter.FileMarker.Mar
 import org.springframework.integration.splitter.AbstractMessageSplitter;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageHandlingException;
+import org.springframework.util.StringUtils;
 
 /**
  * The {@link AbstractMessageSplitter} implementation to split the {@link File}
@@ -147,11 +148,11 @@ public class FileSplitter extends AbstractMessageSplitter {
 			else {
 				reader = new InputStreamReader((InputStream) payload, this.charset);
 			}
-			filePath = ":stream:";
+			filePath = buildPathFromMessage(message, ":stream:");
 		}
 		else if (payload instanceof Reader) {
 			reader = (Reader) payload;
-			filePath = ":reader:";
+			filePath = buildPathFromMessage(message, ":reader:");
 		}
 		else {
 			return message;
@@ -242,7 +243,6 @@ public class FileSplitter extends AbstractMessageSplitter {
 		}
 	}
 
-
 	@Override
 	protected boolean willAddHeaders(Message<?> message) {
 		Object payload = message.getPayload();
@@ -268,6 +268,17 @@ public class FileSplitter extends AbstractMessageSplitter {
 		}
 	}
 
+	private String buildPathFromMessage(Message<?> message, String defaultPath) {
+		String remoteDir = (String) message.getHeaders().get(FileHeaders.REMOTE_DIRECTORY);
+		String remoteFile = (String) message.getHeaders().get(FileHeaders.REMOTE_FILE);
+		if (StringUtils.hasText(remoteDir) && StringUtils.hasText(remoteFile)) {
+			return remoteDir + remoteFile;
+		}
+		else {
+			return defaultPath;
+		}
+	}
+
 	public static class FileMarker implements Serializable {
 
 		private static final long serialVersionUID = 8514605438145748406L;