INT-3412 (S)FTP Append, rmdir, Client Access

JIRA: https://jira.spring.io/browse/INT-3412

Initial commit - review only.

TODO:
- SFTP Tests
- Namespace/Adapter support for file append
- Docs

INT-3412 Polishing

- Addressed PR comments
- Completed SFTP implementation
- Added namespace/parser support for `FileExistsMode` (append, etc)
- Added SFTP Tests
- Created Embedded SFTP server for tests (similar to FTP)
- Converted tests that needed a real server to use the embedded server

INT-3412 Docs and Polish (PR Comments)

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpNamespaceHandler.java

@@ -17,7 +17,6 @@
 package org.springframework.integration.sftp.config;
 
 import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHandler;
-import org.springframework.integration.file.config.RemoteFileOutboundChannelAdapterParser;
 
 /**
  * Provides namespace support for using SFTP.
@@ -30,9 +29,10 @@ import org.springframework.integration.file.config.RemoteFileOutboundChannelAdap
  */
 public class SftpNamespaceHandler extends AbstractIntegrationNamespaceHandler {
 
+	@Override
 	public void init() {
 		registerBeanDefinitionParser("inbound-channel-adapter", new SftpInboundChannelAdapterParser());
-		registerBeanDefinitionParser("outbound-channel-adapter", new RemoteFileOutboundChannelAdapterParser());
+		registerBeanDefinitionParser("outbound-channel-adapter", new SftpOutboundChannelAdapterParser());
 		registerBeanDefinitionParser("outbound-gateway", new SftpOutboundGatewayParser());
 	}
 

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpOutboundChannelAdapterParser.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp.config;
+
+import org.springframework.integration.file.config.RemoteFileOutboundChannelAdapterParser;
+import org.springframework.integration.file.remote.RemoteFileOperations;
+import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
+
+/**
+ * Parser for SFTP Outbound Channel Adapters.
+ *
+ * @author Gary Russell
+ * @since 4.1
+ *
+ */
+public class SftpOutboundChannelAdapterParser extends RemoteFileOutboundChannelAdapterParser {
+
+	@Override
+	protected Class<? extends RemoteFileOperations<?>> getTemplateClass() {
+		return SftpRemoteFileTemplate.class;
+	}
+
+}

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/config/SftpOutboundGatewayParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2013 the original author or authors.
+ * Copyright 2002-2014 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,9 +16,11 @@
 package org.springframework.integration.sftp.config;
 
 import org.springframework.integration.file.config.AbstractRemoteFileOutboundGatewayParser;
+import org.springframework.integration.file.remote.RemoteFileOperations;
 import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter;
 import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter;
 import org.springframework.integration.sftp.gateway.SftpOutboundGateway;
+import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
 
 /**
  * @author Gary Russell
@@ -28,6 +30,7 @@ import org.springframework.integration.sftp.gateway.SftpOutboundGateway;
  */
 public class SftpOutboundGatewayParser extends AbstractRemoteFileOutboundGatewayParser {
 
+	@Override
 	public String getGatewayClassName() {
 		return SftpOutboundGateway.class.getName();
 	}
@@ -42,4 +45,9 @@ public class SftpOutboundGatewayParser extends AbstractRemoteFileOutboundGateway
 		return SftpRegexPatternFileListFilter.class.getName();
 	}
 
+	@Override
+	protected Class<? extends RemoteFileOperations<?>> getTemplateClass() {
+		return SftpRemoteFileTemplate.class;
+	}
+
 }

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/DefaultSftpSessionFactory.java

@@ -21,7 +21,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import org.springframework.beans.factory.BeanCreationException;
 import org.springframework.core.io.Resource;
-import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.file.remote.session.SessionFactory;
 import org.springframework.integration.file.remote.session.SharedSessionCapable;
 import org.springframework.util.Assert;
@@ -314,7 +313,7 @@ public class DefaultSftpSessionFactory implements SessionFactory<LsEntry>, Share
 
 
 	@Override
-	public Session<LsEntry> getSession() {
+	public SftpSession getSession() {
 		Assert.hasText(this.host, "host must not be empty");
 		Assert.hasText(this.user, "user must not be empty");
 		Assert.isTrue(this.port >= 0, "port must be a positive number");

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplate.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp.session;
+
+import java.io.IOException;
+
+import org.springframework.integration.file.remote.ClientCallback;
+import org.springframework.integration.file.remote.RemoteFileTemplate;
+import org.springframework.integration.file.remote.SessionCallback;
+import org.springframework.integration.file.remote.session.Session;
+import org.springframework.integration.file.remote.session.SessionFactory;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import com.jcraft.jsch.SftpException;
+
+/**
+ * SFTP version of {@code RemoteFileTemplate} providing type-safe access to
+ * the underlying ChannelSftp object.
+ *
+ * @author Gary Russell
+ * @since 4.1
+ *
+ */
+public class SftpRemoteFileTemplate extends RemoteFileTemplate<LsEntry> {
+
+	public SftpRemoteFileTemplate(SessionFactory<LsEntry> sessionFactory) {
+		super(sessionFactory);
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public <T, C> T executeWithClient(final ClientCallback<C, T> callback) {
+		return doExecuteWithClient((ClientCallback<ChannelSftp, T>) callback);
+	}
+
+	protected <T> T doExecuteWithClient(final ClientCallback<ChannelSftp, T> callback) {
+		return execute(new SessionCallback<LsEntry, T>() {
+
+			@Override
+			public T doInSession(Session<LsEntry> session) throws IOException {
+				return callback.doWithClient((ChannelSftp) session.getClientInstance());
+			}
+		});
+	}
+
+	@Override
+	public boolean exists(final String path) {
+		return executeWithClient(new ClientCallback<ChannelSftp, Boolean>() {
+
+			@Override
+			public Boolean doWithClient(ChannelSftp client) {
+				try {
+					return client.stat(path) != null;
+				}
+				catch (SftpException e) {
+					return false;
+				}
+			}
+		});
+	}
+
+
+
+}

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpSession.java

@@ -47,7 +47,7 @@ import com.jcraft.jsch.SftpException;
  * @author Gary Russell
  * @since 2.0
  */
-class SftpSession implements Session<LsEntry> {
+public class SftpSession implements Session<LsEntry> {
 
 	private final Log logger = LogFactory.getLog(this.getClass());
 
@@ -159,6 +159,17 @@ class SftpSession implements Session<LsEntry> {
 		}
 	}
 
+	@Override
+	public void append(InputStream inputStream, String destination) throws IOException {
+		Assert.state(this.channel != null, "session is not connected");
+		try {
+			this.channel.put(inputStream, destination, ChannelSftp.APPEND);
+		}
+		catch (SftpException e) {
+			throw new NestedIOException("failed to write file", e);
+		}
+	}
+
 	@Override
 	public void close() {
 		this.closed = true;
@@ -223,6 +234,17 @@ class SftpSession implements Session<LsEntry> {
 		return true;
 	}
 
+	@Override
+	public boolean rmdir(String remoteDirectory) throws IOException {
+		try {
+			this.channel.rmdir(remoteDirectory);
+		}
+		catch (SftpException e) {
+			throw new NestedIOException("failed to remove remote directory '" + remoteDirectory + "'.", e);
+		}
+		return true;
+	}
+
 	@Override
 	public boolean exists(String path) {
 		try {
@@ -251,4 +273,9 @@ class SftpSession implements Session<LsEntry> {
 		}
 	}
 
+	@Override
+	public ChannelSftp getClientInstance() {
+		return this.channel;
+	}
+
 }

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/TestSftpServer.java

@@ -0,0 +1,198 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.sshd.SshServer;
+import org.apache.sshd.common.NamedFactory;
+import org.apache.sshd.common.file.FileSystemView;
+import org.apache.sshd.common.file.nativefs.NativeFileSystemFactory;
+import org.apache.sshd.common.file.nativefs.NativeFileSystemView;
+import org.apache.sshd.server.Command;
+import org.apache.sshd.server.PasswordAuthenticator;
+import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
+import org.apache.sshd.server.session.ServerSession;
+import org.apache.sshd.server.sftp.SftpSubsystem;
+import org.junit.rules.TemporaryFolder;
+
+import org.springframework.beans.factory.DisposableBean;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.util.SocketUtils;
+
+/**
+ * @author Gary Russell
+ * @since 4.1
+ *
+ */
+public class TestSftpServer implements InitializingBean, DisposableBean {
+
+	private final SshServer server = SshServer.setUpDefaultServer();
+
+	private final int port = SocketUtils.findAvailableTcpPort();
+
+	private final TemporaryFolder sftpFolder;
+
+	private final TemporaryFolder localFolder;
+
+	private volatile File sftpRootFolder;
+
+	private volatile File sourceSftpDirectory;
+
+	private volatile File targetSftpDirectory;
+
+	private volatile File sourceLocalDirectory;
+
+	private volatile File targetLocalDirectory;
+
+	public TestSftpServer() {
+		this.sftpFolder = new TemporaryFolder() {
+
+			@Override
+			public void create() throws IOException {
+				super.create();
+				sftpRootFolder = this.newFolder("test");
+				sourceSftpDirectory = new File(sftpRootFolder, "sftpSource");
+				sourceSftpDirectory.mkdir();
+				File file = new File(sourceSftpDirectory, "sftpSource1.txt");
+				file.createNewFile();
+				FileOutputStream fos = new FileOutputStream(file);
+				fos.write("source1".getBytes());
+				fos.close();
+				file = new File(sourceSftpDirectory, "sftpSource2.txt");
+				file.createNewFile();
+				fos = new FileOutputStream(file);
+				fos.write("source2".getBytes());
+				fos.close();
+
+				File subSourceFtpDirectory = new File(sourceSftpDirectory, "subSftpSource");
+				subSourceFtpDirectory.mkdir();
+				file = new File(subSourceFtpDirectory, "subSftpSource1.txt");
+				file.createNewFile();
+				fos = new FileOutputStream(file);
+				fos.write("subSource1".getBytes());
+				fos.close();
+
+				targetSftpDirectory = new File(sftpRootFolder, "sftpTarget");
+				targetSftpDirectory.mkdir();
+			}
+		};
+		this.localFolder  = new TemporaryFolder() {
+
+			@Override
+			public void create() throws IOException {
+				super.create();
+				File rootFolder = this.newFolder("test");
+				sourceLocalDirectory = new File(rootFolder, "localSource");
+				sourceLocalDirectory.mkdirs();
+				File file = new File(sourceLocalDirectory, "localSource1.txt");
+				file.createNewFile();
+				file = new File(sourceLocalDirectory, "localSource2.txt");
+				file.createNewFile();
+
+				File subSourceLocalDirectory = new File(sourceLocalDirectory, "subLocalSource");
+				subSourceLocalDirectory.mkdir();
+				file = new File(subSourceLocalDirectory, "subLocalSource1.txt");
+				file.createNewFile();
+
+				targetLocalDirectory = new File(rootFolder, "slocalTarget");
+				targetLocalDirectory.mkdir();
+			}
+		};
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public void afterPropertiesSet() throws Exception {
+		server.setPasswordAuthenticator(new PasswordAuthenticator() {
+
+			@Override
+			public boolean authenticate(String arg0, String arg1, ServerSession arg2) {
+				return true;
+			}
+		});
+		server.setPort(port);
+		server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider("hostkey.ser"));
+		SftpSubsystem.Factory sftp = new SftpSubsystem.Factory();
+		server.setSubsystemFactories(Arrays.<NamedFactory<Command>>asList(sftp));
+		server.setFileSystemFactory(new NativeFileSystemFactory() {
+
+			@Override
+			public FileSystemView createFileSystemView(org.apache.sshd.common.Session session) {
+				return new NativeFileSystemView(session.getUsername(), false) {
+
+					@Override
+					public String getVirtualUserDir() {
+						return sftpRootFolder.getAbsolutePath();
+					}
+				};
+			}
+
+		});
+		this.sftpFolder.create();
+		this.localFolder.create();
+		server.start();
+	}
+
+	@Override
+	public void destroy() throws Exception {
+		this.server.stop();
+		this.sftpFolder.delete();
+		this.localFolder.delete();
+	}
+
+	public File getSourceLocalDirectory() {
+		return this.sourceLocalDirectory;
+	}
+
+	public File getTargetLocalDirectory() {
+		return this.targetLocalDirectory;
+	}
+
+	public String getTargetLocalDirectoryName() {
+		return this.targetLocalDirectory.getAbsolutePath() + File.separator;
+	}
+
+	public File getTargetSftpDirectory() {
+		return this.targetSftpDirectory;
+	}
+
+	public void recursiveDelete(File file) {
+		File[] files = file.listFiles();
+		if (files != null) {
+			for (File each : files) {
+				recursiveDelete(each);
+			}
+		}
+		if (!(file.equals(this.targetSftpDirectory) || file.equals(this.targetLocalDirectory))) {
+			file.delete();
+		}
+	}
+
+	public DefaultSftpSessionFactory getSessionFactory() {
+		DefaultSftpSessionFactory factory = new DefaultSftpSessionFactory(true);
+		factory.setHost("localhost");
+		factory.setPort(this.port);
+		factory.setUser("foo");
+		factory.setPassword("foo");
+		return factory;
+	}
+
+}

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/TestSftpServerConfig.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+
+/**
+ * @author Gary Russell
+ * @since 4.1
+ *
+ */
+@Configuration
+public class TestSftpServerConfig {
+
+	@Bean
+	public TestSftpServer sftpServer() {
+		return new TestSftpServer();
+	}
+
+	@Bean
+	public DefaultSftpSessionFactory sftpSessionFactory(TestSftpServer server) {
+		return sftpServer().getSessionFactory();
+	}
+
+}

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java

@@ -45,11 +45,11 @@ import org.springframework.beans.factory.BeanFactory;
 import org.springframework.integration.expression.ExpressionUtils;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
-import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.metadata.PropertiesPersistingMetadataStore;
 import org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter;
 import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.integration.sftp.session.SftpSession;
 import org.springframework.integration.sftp.session.SftpTestSessionFactory;
 import org.springframework.integration.test.util.TestUtils;
 import org.springframework.messaging.Message;
@@ -93,7 +93,6 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 		ftpSessionFactory.setPassword("frog");
 		ftpSessionFactory.setHost("foo.com");
 
-
 		SftpInboundFileSynchronizer synchronizer = spy(new SftpInboundFileSynchronizer(ftpSessionFactory));
 		synchronizer.setDeleteRemoteFiles(true);
 		synchronizer.setPreserveTimestamp(true);
@@ -168,7 +167,7 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 		}
 
 		@Override
-		public Session<LsEntry> getSession() {
+		public SftpSession getSession() {
 			if (this.sftpEntries.size() == 0) {
 				this.init();
 			}
@@ -184,7 +183,8 @@ public class SftpInboundRemoteFileSystemSynchronizerTests {
 
 				when(jschSession.openChannel("sftp")).thenReturn(channel);
 				return SftpTestSessionFactory.createSftpSession(jschSession);
-			} catch (Exception e) {
+			}
+			catch (Exception e) {
 				throw new RuntimeException("Failed to create mock sftp session", e);
 			}
 		}

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpOutboundTests.java

@@ -56,6 +56,7 @@ import org.springframework.integration.file.remote.session.CachingSessionFactory
 import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.file.remote.session.SessionFactory;
 import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.integration.sftp.session.SftpSession;
 import org.springframework.integration.sftp.session.SftpTestSessionFactory;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.test.util.TestUtils;
@@ -375,7 +376,7 @@ public class SftpOutboundTests {
 	public static class TestSftpSessionFactory extends DefaultSftpSessionFactory {
 
 		@Override
-		public Session<LsEntry> getSession() {
+		public SftpSession getSession() {
 			try {
 				ChannelSftp channel = mock(ChannelSftp.class);
 

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml

@@ -14,17 +14,17 @@
 
 	<int:channel id="inboundGet"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundGet"
 							  command="get"
 							  expression="payload"
-							  local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory.toUpperCase()"
-							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
+							  local-directory-expression="@sftpServer.targetLocalDirectoryName + #remoteDirectory.toUpperCase()"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
 	<int:channel id="invalidDirExpression"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="invalidDirExpression"
 							  command="get"
 							  expression="payload"
@@ -33,40 +33,40 @@
 
 	<int:channel id="inboundMGet"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMGet"
 							  command="mget"
 							  expression="payload"
-							  local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory"
-							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
+							  local-directory-expression="@sftpServer.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
 	<int:channel id="inboundMGetRecursive"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMGetRecursive"
 							  command="mget"
 							  expression="payload"
 							  command-options="-R"
-							  local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory"
-							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
+							  local-directory-expression="@sftpServer.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
 	<int:channel id="inboundMGetRecursiveFiltered"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMGetRecursiveFiltered"
 							  command="mget"
 							  expression="payload"
 							  command-options="-R"
 							  filename-regex="(subSftpSource|.*1.txt)"
-							  local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory"
-							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
+							  local-directory-expression="@sftpServer.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
 	<int:channel id="inboundMPut"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMPut"
 							  command="mput"
 							  auto-create-directory="true"
@@ -77,7 +77,7 @@
 
 	<int:channel id="inboundMPutRecursive"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMPutRecursive"
 							  command="mput"
 							  command-options="-R"
@@ -89,7 +89,7 @@
 
 	<int:channel id="inboundMPutRecursiveFiltered"/>
 
-	<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
 							  request-channel="inboundMPutRecursiveFiltered"
 							  command="mput"
 							  command-options="-R"
@@ -100,32 +100,37 @@
 							  remote-directory="sftpTarget"
 							  reply-channel="output"/>
 
-	<bean id="ftpSessionFactory" class="org.mockito.Mockito" factory-method="mock">
-		<constructor-arg value="org.springframework.integration.file.remote.session.SessionFactory" />
-	</bean>
+	<int:channel id="appending" />
 
-	<beans profile="realSSH">
-		<bean id="ftpSessionFactory"
-			  class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
-			<property name="host" value="localhost"/>
-			<property name="user" value="ftptest"/>
-			<property name="password" value="ftptest"/>
-		</bean>
-	</beans>
+	<bean id="sftpServerConfig" class="org.springframework.integration.sftp.TestSftpServerConfig" />
 
-	<beans profile="realSSHSharedSession">
-		<bean id="ftpSessionFactory" 
-				class="org.springframework.integration.file.remote.session.CachingSessionFactory">
-			<constructor-arg>
-				<bean
-			  		class="org.springframework.integration.sftp.session.DefaultSftpSessionFactory">
-					<constructor-arg value="true"/>
-					<property name="host" value="localhost"/>
-					<property name="user" value="ftptest"/>
-					<property name="password" value="ftptest"/>
-				</bean>
-			</constructor-arg>
-		</bean>
-	</beans>
+	<int-sftp:outbound-channel-adapter id="appender"
+							  session-factory="sftpSessionFactory"
+							  channel="appending"
+							  mode="APPEND"
+							  use-temporary-file-name="false"
+							  remote-directory="sftpTarget"
+							  auto-create-directory="true"
+							  remote-file-separator="/"	/>
+
+	<int:channel id="ignoring" />
+
+	<int-sftp:outbound-channel-adapter id="ignore"
+							  session-factory="sftpSessionFactory"
+							  channel="ignoring"
+							  mode="IGNORE"
+							  remote-directory="sftpTarget"
+							  auto-create-directory="true"
+							  remote-file-separator="/"	/>
+
+	<int:channel id="failing" />
+
+	<int-sftp:outbound-channel-adapter id="fail"
+							  session-factory="sftpSessionFactory"
+							  channel="failing"
+							  mode="FAIL"
+							  remote-directory="sftpTarget"
+							  auto-create-directory="true"
+							  remote-file-separator="/"	/>
 
 </beans>

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013 the original author or authors.
+ * Copyright 2013-2014 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package org.springframework.integration.sftp.outbound;
 
 import static org.hamcrest.Matchers.anyOf;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
@@ -25,8 +26,6 @@ import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -44,29 +43,28 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
-import org.springframework.beans.DirectFieldAccessor;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.integration.channel.DirectChannel;
+import org.springframework.integration.file.FileHeaders;
+import org.springframework.integration.file.remote.SessionCallback;
 import org.springframework.integration.file.remote.session.Session;
-import org.springframework.integration.file.remote.session.SessionFactory;
-import org.springframework.integration.sftp.session.SftpFileInfo;
+import org.springframework.integration.sftp.TestSftpServer;
+import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
+import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.test.util.TestUtils;
 import org.springframework.messaging.Message;
+import org.springframework.messaging.MessagingException;
 import org.springframework.messaging.PollableChannel;
 import org.springframework.messaging.support.GenericMessage;
-import org.springframework.test.annotation.IfProfileValue;
-import org.springframework.test.annotation.ProfileValueUtils;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 import org.springframework.util.FileCopyUtils;
 
 import com.jcraft.jsch.ChannelSftp.LsEntry;
-import com.jcraft.jsch.SftpATTRS;
 
 /**
- * Run with -Dspring-profiles-active=realSSH to run with a real SSH server.
- *
- * Assumes ftptest account on localhost with the following directory tree in the user's root...
+ * Runs against an embedded SFTP Server with the following directory tree:
  *
  * <pre class="code">
  *  $ tree sftpSource/
@@ -113,81 +111,30 @@ public class SftpServerOutboundTests {
 	private DirectChannel inboundMPutRecursiveFiltered;
 
 	@Autowired
-	private SessionFactory<SftpFileInfo> sessionFactory;
+	private DefaultSftpSessionFactory sessionFactory;
+
+	@Autowired
+	private DirectChannel appending;
+
+	@Autowired
+	private DirectChannel ignoring;
+
+	@Autowired
+	private DirectChannel failing;
+
+	@Autowired
+	private TestSftpServer sftpServer;
 
 	@Before
-	public void setup() throws Exception {
-		purge();
-		setUpMocksIfNeeded();
-	}
-
-	@SuppressWarnings({ "rawtypes", "unchecked" })
-	private void setUpMocksIfNeeded() throws IOException {
-		String profile = ProfileValueUtils.retrieveProfileValueSource(this.getClass()).get("spring.profiles.active");
-		boolean usingMocks = profile == null || ! profile.startsWith("realSSH");
-		if (usingMocks) {
-			Session session = mock(Session.class);
-			when(sessionFactory.getSession()).thenReturn(session);
-			LsEntry entry1 = mock(LsEntry.class);
-			SftpATTRS attrs1 = mock(SftpATTRS.class);
-			when(entry1.getAttrs()).thenReturn(attrs1);
-			when(entry1.getFilename()).thenReturn("sftpSource1.txt");
-			LsEntry entry2 = mock(LsEntry.class);
-			SftpATTRS attrs2 = mock(SftpATTRS.class);
-			when(entry2.getAttrs()).thenReturn(attrs2);
-			when(entry2.getFilename()).thenReturn("sftpSource2.txt");
-			LsEntry entry3 = mock(LsEntry.class);
-			when(entry3.getFilename()).thenReturn("subSftpSource");
-			SftpATTRS attrs3 = mock(SftpATTRS.class);
-			when(entry3.getAttrs()).thenReturn(attrs3);
-			when(attrs3.isDir()).thenReturn(true);
-			LsEntry entry4 = mock(LsEntry.class);
-			SftpATTRS attrs4 = mock(SftpATTRS.class);
-			when(entry4.getAttrs()).thenReturn(attrs4);
-			// recursion uses a DFA to update the filename to include the subdirectory
-			new DirectFieldAccessor(entry4).setPropertyValue("filename", "subSftpSource1.txt");
-			when(entry4.getFilename()).thenCallRealMethod();
-			when(session.list("sftpSource/sftpSource1.txt")).thenReturn(new LsEntry[] {
-					entry1
-			});
-			when(session.list("sftpSource/")).thenReturn(new LsEntry[] {
-					entry1, entry2, entry3
-			});
-			when(session.list("sftpSource/subSftpSource/")).thenReturn(new LsEntry[] {
-					entry4
-			});
-			when(session.list("sftpSource/subSftpSource/subSftpSource1.txt")).thenReturn(new LsEntry[] {
-					entry4
-			});
-		}
-	}
-
 	@After
-	public void purge() {
-		File local = new File("/tmp/sftpOutboundTests/");
-		purge(local);
-		local.delete();
-	}
-
-	private void purge(File local) {
-		File[] files = local.listFiles();
-		if (files != null) {
-			for (File file : files) {
-				if (file.isDirectory()) {
-					this.purge(file);
-				}
-				file.delete();
-			}
-		}
+	public void setup() {
+		this.sftpServer.recursiveDelete(sftpServer.getTargetLocalDirectory());
+		this.sftpServer.recursiveDelete(sftpServer.getTargetSftpDirectory());
 	}
 
 	@Test
 	public void testInt2866LocalDirectoryExpressionGET() {
-		Session<?> session = null;
-		boolean sharedSession = "realSSHSharedSession".equals(System.getProperty("spring.profiles.active"));
-		if (sharedSession) {
-			session = this.sessionFactory.getSession();
-		}
+		Session<?> session = this.sessionFactory.getSession();
 		String dir = "sftpSource/";
 		this.inboundGet.send(new GenericMessage<Object>(dir + "sftpSource1.txt"));
 		Message<?> result = this.output.receive(1000);
@@ -203,11 +150,9 @@ public class SftpServerOutboundTests {
 		localFile = (File) result.getPayload();
 		assertThat(localFile.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
 				Matchers.containsString(dir.toUpperCase()));
-		if (sharedSession) {
-			Session<?> session2 = this.sessionFactory.getSession();
-			assertSame(TestUtils.getPropertyValue(session, "targetSession.jschSession"),
-					TestUtils.getPropertyValue(session2, "targetSession.jschSession"));
-		}
+		Session<?> session2 = this.sessionFactory.getSession();
+		assertSame(TestUtils.getPropertyValue(session, "jschSession"),
+				TestUtils.getPropertyValue(session2, "jschSession"));
 	}
 
 	@Test
@@ -294,7 +239,6 @@ public class SftpServerOutboundTests {
 	 * Only runs with a real server (see class javadocs).
 	 */
 	@Test
-	@IfProfileValue(name="spring.profiles.active", value="realSSH")
 	public void testInt3100RawGET() throws Exception {
 		Session<?> session = this.sessionFactory.getSession();
 		ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -311,7 +255,6 @@ public class SftpServerOutboundTests {
 	}
 
 	@Test
-	@IfProfileValue(name="spring.profiles.active", value="realSSHSharedSession")
 	public void testInt3047ConcurrentSharedSession() throws Exception {
 		final Session<?> session1 = this.sessionFactory.getSession();
 		final Session<?> session2 = this.sessionFactory.getSession();
@@ -371,12 +314,11 @@ public class SftpServerOutboundTests {
 	}
 
 	@Test
-	@IfProfileValue(name="spring.profiles.active", value="realSSH")
 	public void testInt3088MPutNotRecursive() {
 		String dir = "sftpSource/";
 		this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
 		while (output.receive(0) != null) { }
-		this.inboundMPut.send(new GenericMessage<File>(new File("/tmp/sftpOutboundTests/sftpSource")));
+		this.inboundMPut.send(new GenericMessage<File>(this.sftpServer.getSourceLocalDirectory()));
 		@SuppressWarnings("unchecked")
 		Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
 		assertNotNull(out);
@@ -385,19 +327,18 @@ public class SftpServerOutboundTests {
 				not(equalTo(out.getPayload().get(1))));
 		assertThat(
 				out.getPayload().get(0),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt")));
 		assertThat(
 				out.getPayload().get(1),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt")));
 	}
 
 	@Test
-	@IfProfileValue(name="spring.profiles.active", value="realSSH")
 	public void testInt3088MPutRecursive() {
 		String dir = "sftpSource/";
 		this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
 		while (output.receive(0) != null) { }
-		this.inboundMPutRecursive.send(new GenericMessage<File>(new File("/tmp/sftpOutboundTests/sftpSource")));
+		this.inboundMPutRecursive.send(new GenericMessage<File>(this.sftpServer.getSourceLocalDirectory()));
 		@SuppressWarnings("unchecked")
 		Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
 		assertNotNull(out);
@@ -406,25 +347,24 @@ public class SftpServerOutboundTests {
 				not(equalTo(out.getPayload().get(1))));
 		assertThat(
 				out.getPayload().get(0),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt"),
-						equalTo("sftpTarget/subSftpSource/subSlocalTarget1.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
+						equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
 		assertThat(
 				out.getPayload().get(1),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt"),
-						equalTo("sftpTarget/subSftpSource/subSlocalTarget1.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
+						equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
 		assertThat(
 				out.getPayload().get(2),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt"),
-						equalTo("sftpTarget/subSftpSource/subSlocalTarget1.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
+						equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
 	}
 
 	@Test
-	@IfProfileValue(name="spring.profiles.active", value="realSSH")
 	public void testInt3088MPutRecursiveFiltered() {
 		String dir = "sftpSource/";
 		this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
 		while (output.receive(0) != null) { }
-		this.inboundMPutRecursiveFiltered.send(new GenericMessage<File>(new File("/tmp/sftpOutboundTests/sftpSource")));
+		this.inboundMPutRecursiveFiltered.send(new GenericMessage<File>(this.sftpServer.getSourceLocalDirectory()));
 		@SuppressWarnings("unchecked")
 		Message<List<String>> out = (Message<List<String>>) this.output.receive(1000);
 		assertNotNull(out);
@@ -433,12 +373,47 @@ public class SftpServerOutboundTests {
 				not(equalTo(out.getPayload().get(1))));
 		assertThat(
 				out.getPayload().get(0),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt"),
-						equalTo("sftpTarget/subSftpSource/subSlocalTarget1.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
+						equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
 		assertThat(
 				out.getPayload().get(1),
-				anyOf(equalTo("sftpTarget/slocalTarget1.txt"), equalTo("sftpTarget/slocalTarget2.txt"),
-						equalTo("sftpTarget/subSftpSource/subSlocalTarget1.txt")));
+				anyOf(equalTo("sftpTarget/localSource1.txt"), equalTo("sftpTarget/localSource2.txt"),
+						equalTo("sftpTarget/subLocalSource/subLocalSource1.txt")));
+	}
+
+	@Test
+	public void testInt3412FileMode() {
+		Message<String> m = MessageBuilder.withPayload("foo")
+				.setHeader(FileHeaders.FILENAME, "appending.txt")
+				.build();
+		appending.send(m);
+		appending.send(m);
+
+		SftpRemoteFileTemplate template = new SftpRemoteFileTemplate(sessionFactory);
+		assertLength6(template);
+
+		ignoring.send(m);
+		assertLength6(template);
+		try {
+			failing.send(m);
+			fail("Expected exception");
+		}
+		catch (MessagingException e) {
+			assertThat(e.getCause().getCause().getMessage(), containsString("The destination file already exists"));
+		}
+
+	}
+
+	private void assertLength6(SftpRemoteFileTemplate template) {
+		LsEntry[] files = template.execute(new SessionCallback<LsEntry, LsEntry[]>() {
+
+			@Override
+			public LsEntry[] doInSession(Session<LsEntry> session) throws IOException {
+				return session.list("sftpTarget/appending.txt");
+			}
+		});
+		assertEquals(1, files.length);
+		assertEquals(6, files[0].getAttrs().getSize());
 	}
 
 }

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplateTests.java

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2014 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp.session;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.expression.common.LiteralExpression;
+import org.springframework.integration.file.DefaultFileNameGenerator;
+import org.springframework.integration.file.remote.ClientCallbackWithoutResult;
+import org.springframework.integration.file.remote.SessionCallback;
+import org.springframework.integration.file.remote.SessionCallbackWithoutResult;
+import org.springframework.integration.file.remote.session.Session;
+import org.springframework.integration.sftp.TestSftpServer;
+import org.springframework.integration.sftp.TestSftpServerConfig;
+import org.springframework.messaging.support.GenericMessage;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.ChannelSftp.LsEntry;
+import com.jcraft.jsch.SftpATTRS;
+import com.jcraft.jsch.SftpException;
+
+/**
+ * @author Gary Russell
+ * @since 4.1
+ *
+ */
+@ContextConfiguration(classes=TestSftpServerConfig.class)
+@RunWith(SpringJUnit4ClassRunner.class)
+@DirtiesContext
+public class SftpRemoteFileTemplateTests {
+
+	@Autowired
+	private TestSftpServer sftpServer;
+
+	@Autowired
+	private DefaultSftpSessionFactory sessionFactory;
+
+	@Before
+	@After
+	public void setup() {
+		this.sftpServer.recursiveDelete(sftpServer.getTargetLocalDirectory());
+		this.sftpServer.recursiveDelete(sftpServer.getTargetSftpDirectory());
+	}
+
+	@Test
+	public void testINT3412AppendStatRmdir() {
+		SftpRemoteFileTemplate template = new SftpRemoteFileTemplate(sessionFactory);
+		DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator();
+		fileNameGenerator.setExpression("'foobar.txt'");
+		template.setFileNameGenerator(fileNameGenerator);
+		template.setRemoteDirectoryExpression(new LiteralExpression("foo/"));
+		template.setUseTemporaryFileName(false);
+		template.execute(new SessionCallback<LsEntry, Boolean>() {
+
+			@Override
+			public Boolean doInSession(Session<LsEntry> session) throws IOException {
+				session.mkdir("foo/");
+				return session.mkdir("foo/bar/");
+			}
+
+		});
+		template.append(new GenericMessage<String>("foo"));
+		template.append(new GenericMessage<String>("bar"));
+		assertTrue(template.exists("foo/foobar.txt"));
+		template.executeWithClient(new ClientCallbackWithoutResult<ChannelSftp>() {
+
+			@Override
+			public void doWithClientWithoutResult(ChannelSftp client) {
+				try {
+					SftpATTRS file = client.lstat("foo/foobar.txt");
+					assertEquals(6, file.getSize());
+				}
+				catch (SftpException e) {
+					throw new RuntimeException(e);
+				}
+			}
+		});
+		template.execute(new SessionCallbackWithoutResult<LsEntry>() {
+
+			@Override
+			public void doInSessionWithoutResult(Session<LsEntry> session) throws IOException {
+				assertTrue(session.remove("foo/foobar.txt"));
+				assertTrue(session.rmdir("foo/bar/"));
+				LsEntry[] files = session.list("foo/");
+				assertEquals(0, files.length);
+				assertTrue(session.rmdir("foo/"));
+			}
+		});
+		assertFalse(template.exists("foo"));
+	}
+
+}

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/session/SftpTestSessionFactory.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2014 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,17 +15,15 @@
  */
 package org.springframework.integration.sftp.session;
 
-import org.springframework.integration.file.remote.session.Session;
-
-import com.jcraft.jsch.ChannelSftp.LsEntry;
 
 /**
  * @author Oleg Zhurakousky
- * 
+ * @author Gary Russell
+ *
  */
 public class SftpTestSessionFactory {
 
-	public static Session<LsEntry> createSftpSession(com.jcraft.jsch.Session jschSession) {
+	public static SftpSession createSftpSession(com.jcraft.jsch.Session jschSession) {
 		SftpSession sftpSession = new SftpSession(jschSession);
 		sftpSession.connect();
 		return sftpSession;