Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-3172 (S)FTP - Add Recursion Option to LS Cmd

Browse Source 
When recursing, the returned filenames are relative to
the top level directory.

INT-3172 (S)FTP - Add Recursion to MGET Command

https://jira.springsource.org/browse/INT-3172

INT-3172 Polishing

- PR Comments
- Add Tests for SFTP (tested with Mock and real SSH)
- Enhance test to ensure subdir is in 3rd file retrieved
- Exclude "special" directories ('.' and '..') from recursion

INT-3172 Make FtpServerRule Safe For Concurrency

Previously the port was static.

- Make the port an instance variable
- Channel the `@ ClassRule` to a `@ Rule`
- Register an appropriate DefaultFtpSessionFactory bean declaration

INT-3172 Polishing; Docs

- Rename FtpServerRule to TestFtpServer
- Remove restriction on filters for MGET
- Add test for filtered MGET
- Reference docs, what's new

INT-3172 More Polish

- Fix type in reference
- Fix error message now that filter is allowed with MGET
- Remove test for filter with MGET (now valid)

INT-3172: Fix tests and typos

INT-3172: this.remoteFileSeparator for recursion
This commit is contained in:
Gary Russell
2013-10-18 12:04:55 +03:00
committed by Artem Bilan
parent dd479a3ce7
commit 6d9e48b9ac
12 changed files with 478 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View File

@@ -66,22 +66,27 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
* Enumeration of commands supported by the gateways.
*/
public static enum Command {
/**
* List remote files.
*/
LS("ls"),
/**
* Retrieve a remote file.
*/
GET("get"),
/**
* Remove a remote file (path - including wildcards).
*/
RM("rm"),
/**
* Retrieve multiple files matching a wildcard path.
*/
MGET("mget"),
/**
* Move (rename) a remote file.
*/
@@ -112,34 +117,46 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
*
*/
public static enum Option {
/**
* Don't return full file information; just the name (ls).
*/
NAME_ONLY("-1"),
/**
* Include directories {@code .} and {@code ..} in the results (ls).
* Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls).
*/
ALL("-a"),
/**
* Do not sort the results (ls with NAME_ONLY).
*/
NOSORT("-f"),
/**
* Include directories in the results (ls).
*/
SUBDIRS("-dirs"),
/**
* Include links in the results (ls).
*/
LINKS("-links"),
/**
* Preserve the server timestamp (get, mget).
*/
PRESERVE_TIMESTAMP("-P"),
/**
* Throw an exception if no files returned (mget).
*/
EXCEPTION_WHEN_EMPTY("-x");
EXCEPTION_WHEN_EMPTY("-x"),
/**
* Recursive (ls, mget)
*/
RECURSIVE("-R");
private String option;
@@ -272,9 +289,9 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
protected void onInit() {
super.onInit();
Assert.notNull(this.command, "command must not be null");
if (Command.RM.equals(this.command) || Command.MGET.equals(this.command) ||
if (Command.RM.equals(this.command) ||
Command.GET.equals(this.command)) {
Assert.isNull(this.filter, "Filters are not supported with the rm, get, and mget commands");
Assert.isNull(this.filter, "Filters are not supported with the rm and get commands");
}
if (Command.GET.equals(this.command)
|| Command.MGET.equals(this.command)) {
@@ -305,6 +322,11 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
}
}
}
if (Command.MGET.equals(this.command)) {
Assert.isTrue(!(this.options.contains(Option.SUBDIRS)),
"Cannot use " + Option.SUBDIRS.toString() + " when using 'mget' use " + Option.RECURSIVE.toString() +
" to obtain files in subdirectories");
}
if (this.getBeanFactory() != null) {
this.fileNameProcessor.setBeanFactory(this.getBeanFactory());
this.renameProcessor.setBeanFactory(this.getBeanFactory());
@@ -398,21 +420,7 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
}
protected List<?> ls(Session<F> session, String dir) throws IOException {
List<F> lsFiles = new ArrayList<F>();
F[] files = session.list(dir);
if (!ObjectUtils.isEmpty(files)) {
Collection<F> filteredFiles = this.filterFiles(files);
for (F file : filteredFiles) {
if (file != null) {
if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) {
lsFiles.add(file);
}
}
}
}
else {
return lsFiles;
}
List<F> lsFiles = listFilesInRemoteDir(session, dir, "");
if (!this.options.contains(Option.LINKS)) {
purgeLinks(lsFiles);
}
@@ -441,6 +449,32 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
}
}
private List<F> listFilesInRemoteDir(Session<F> session, String directory, String subDirectory) throws IOException {
List<F> lsFiles = new ArrayList<F>();
F[] files = session.list(directory + subDirectory);
boolean recursion = this.options.contains(Option.RECURSIVE);
if (!ObjectUtils.isEmpty(files)) {
Collection<F> filteredFiles = this.filterFiles(files);
for (F file : filteredFiles) {
String fileName = this.getFilename(file);
if (file != null) {
if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) {
if (recursion && StringUtils.hasText(subDirectory)) {
lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory));
}
else {
lsFiles.add(file);
}
}
if (recursion && this.isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) {
lsFiles.addAll(listFilesInRemoteDir(session, directory, subDirectory + fileName + this.remoteFileSeparator));
}
}
}
}
return lsFiles;
}
protected final List<F> filterFiles(F[] files) {
return (this.filter != null) ? this.filter.filterFiles(files) : Arrays.asList(files);
}
@@ -523,6 +557,22 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
protected List<File> mGet(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
if (this.options.contains(Option.RECURSIVE)) {
if (logger.isWarnEnabled() && !("*".equals(remoteFilename))) {
logger.warn("File name pattern must be '*' when using recursion");
}
if (this.options.contains(Option.NAME_ONLY)) {
this.options.remove(Option.NAME_ONLY);
}
return mGetWithRecursion(message, session, remoteDirectory, remoteFilename);
}
else {
return mGetWithoutRecursion(message, session, remoteDirectory, remoteFilename);
}
}
private List<File> mGetWithoutRecursion(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
String path = this.generateFullPath(remoteDirectory, remoteFilename);
String[] fileNames = session.listNames(path);
if (fileNames == null) {
@@ -549,6 +599,30 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
return files;
}
private List<File> mGetWithRecursion(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
List<File> files = new ArrayList<File>();
@SuppressWarnings("unchecked")
List<AbstractFileInfo<F>> fileNames = (List<AbstractFileInfo<F>>) this.ls(session, remoteDirectory);
if (fileNames.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) {
throw new MessagingException("No files found at " + remoteDirectory
+ " with pattern " + remoteFilename);
}
for (AbstractFileInfo<F> lsEntry : fileNames) {
String fullFileName = remoteDirectory + this.getFilename(lsEntry);
/*
* With recursion, the filename might contain subdirectory information
* normalize each file separately.
*/
String fileName = this.getRemoteFilename(fullFileName);
String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName);
File file = this.get(message, session, actualRemoteDirectory,
fullFileName, fileName, false);
files.add(file);
}
return files;
}
private String getRemoteDirectory(String remoteFilePath, String remoteFilename) {
String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf(remoteFilename));
if (remoteDir.length() == 0) {
@@ -626,8 +700,11 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
abstract protected String getFilename(F file);
abstract protected String getFilename(AbstractFileInfo<F> file);
abstract protected long getModified(F file);
abstract protected List<AbstractFileInfo<F>> asFileInfoList(Collection<F> files);
abstract protected F enhanceNameWithSubDirectory(F file, String directory);
}

116
spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View File

@@ -86,21 +86,6 @@ public class RemoteFileOutboundGatewayTests {
}
}
@Test
public void testBadFilterMGet() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "mget", "payload");
gw.setFilter(new TestPatternFilter(""));
try {
gw.onInit();
fail("Exception expected");
}
catch (IllegalArgumentException e) {
assertTrue(e.getMessage().startsWith("Filters are not supported"));
}
}
@Test
public void testBadFilterRm() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
@@ -389,9 +374,6 @@ public class RemoteFileOutboundGatewayTests {
assertEquals("foo/bar", madeDirs.get(1));
}
/**
* @return
*/
public TestLsEntry[] fileList() {
TestLsEntry[] files = new TestLsEntry[6];
files[0] = new TestLsEntry("f2", 123, false, false, 1234, "-r--r--r--");
@@ -424,6 +406,83 @@ public class RemoteFileOutboundGatewayTests {
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}
public TestLsEntry[] level1List() {
return new TestLsEntry[] {
new TestLsEntry("f1", 123, false, false, 1234, "-r--r--r--"),
new TestLsEntry("d1", 0, true, false, 12345, "drw-r--r--"),
new TestLsEntry("f2", 12345, false, false, 123456, "-rw-r--r--")
};
}
public TestLsEntry[] level2List() {
return new TestLsEntry[] {
new TestLsEntry("d2", 0, true, false, 12345, "drw-r--r--"),
new TestLsEntry("f3", 12345, false, false, 123456, "-rw-r--r--")
};
}
public TestLsEntry[] level3List() {
return new TestLsEntry[] {
new TestLsEntry("f4", 12345, false, false, 123456, "-rw-r--r--")
};
}
@Test
public void testLs_f_R() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Session session = mock(Session.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "ls", "payload");
gw.setOptions("-f -R");
gw.afterPropertiesSet();
when(sessionFactory.getSession()).thenReturn(session);
TestLsEntry[] level1 = level1List();
TestLsEntry[] level2 = level2List();
TestLsEntry[] level3 = level3List();
when(session.list("testremote/x/")).thenReturn(level1);
when(session.list("testremote/x/d1/")).thenReturn(level2);
when(session.list("testremote/x/d1/d2/")).thenReturn(level3);
@SuppressWarnings("unchecked")
Message<List<TestLsEntry>> out = (Message<List<TestLsEntry>>) gw
.handleRequestMessage(new GenericMessage<String>("testremote/x"));
assertEquals(4, out.getPayload().size());
assertEquals("f1", out.getPayload().get(0).getFilename());
assertEquals("d1/d2/f4", out.getPayload().get(1).getFilename());
assertEquals("d1/f3", out.getPayload().get(2).getFilename());
assertEquals("f2", out.getPayload().get(3).getFilename());
assertEquals("testremote/x/",
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}
@Test
public void testLs_f_R_dirs() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Session session = mock(Session.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "ls", "payload");
gw.setOptions("-f -R -dirs");
gw.afterPropertiesSet();
when(sessionFactory.getSession()).thenReturn(session);
TestLsEntry[] level1 = level1List();
TestLsEntry[] level2 = level2List();
TestLsEntry[] level3 = level3List();
when(session.list("testremote/x/")).thenReturn(level1);
when(session.list("testremote/x/d1/")).thenReturn(level2);
when(session.list("testremote/x/d1/d2/")).thenReturn(level3);
@SuppressWarnings("unchecked")
Message<List<TestLsEntry>> out = (Message<List<TestLsEntry>>) gw
.handleRequestMessage(new GenericMessage<String>("testremote/x"));
assertEquals(6, out.getPayload().size());
assertEquals("f1", out.getPayload().get(0).getFilename());
assertEquals("d1", out.getPayload().get(1).getFilename());
assertEquals("d1/d2", out.getPayload().get(2).getFilename());
assertEquals("d1/d2/f4", out.getPayload().get(3).getFilename());
assertEquals("d1/f3", out.getPayload().get(4).getFilename());
assertEquals("f2", out.getPayload().get(5).getFilename());
assertEquals("testremote/x/",
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}
@Test
public void testLs_None() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
@@ -775,6 +834,11 @@ class TestRemoteFileOutboundGateway extends AbstractRemoteFileOutboundGateway<Te
return file.getFilename();
}
@Override
protected String getFilename(AbstractFileInfo<TestLsEntry> file) {
return file.getFilename();
}
@Override
protected long getModified(TestLsEntry file) {
return file.getModified();
@@ -786,18 +850,24 @@ class TestRemoteFileOutboundGateway extends AbstractRemoteFileOutboundGateway<Te
return new ArrayList<AbstractFileInfo<TestLsEntry>>(files);
}
@Override
protected TestLsEntry enhanceNameWithSubDirectory(TestLsEntry file, String directory) {
file.setFilename(directory + file.getFilename());
return file;
}
}
class TestLsEntry extends AbstractFileInfo<TestLsEntry> {
private final String filename;
private final int size;
private volatile String filename;
private final long size;
private final boolean dir;
private final boolean link;
private final long modified;
private final String permissions;
public TestLsEntry(String filename, int size, boolean dir, boolean link,
public TestLsEntry(String filename, long size, boolean dir, boolean link,
long modified, String permissions) {
this.filename = filename;
this.size = size;
@@ -835,6 +905,10 @@ class TestLsEntry extends AbstractFileInfo<TestLsEntry> {
return this;
}
public void setFilename(String filename) {
this.filename = filename;
}
}
class TestPatternFilter extends AbstractSimplePatternFileListFilter<TestLsEntry>{

17
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2011 the original author or authors.
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,6 +21,7 @@ import java.util.Collection;
import java.util.List;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
import org.springframework.integration.file.remote.session.SessionFactory;
@@ -28,7 +29,7 @@ import org.springframework.integration.ftp.session.FtpFileInfo;
/**
* Outbound Gateway for performing remote file operations via FTP/FTPS.
*
*
* @author Gary Russell
* @since 2.1
*/
@@ -54,6 +55,11 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway<FTPFil
return file.getName();
}
@Override
protected String getFilename(AbstractFileInfo<FTPFile> file) {
return file.getFilename();
}
@Override
protected long getModified(FTPFile file) {
return file.getTimestamp().getTimeInMillis();
@@ -69,4 +75,11 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway<FTPFil
}
@Override
protected FTPFile enhanceNameWithSubDirectory(FTPFile file, String directory) {
file.setName(directory + file.getName());
return file;
}
}

42
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java → spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java
View File

@@ -20,6 +20,9 @@ import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import org.apache.ftpserver.FtpServer;
import org.apache.ftpserver.FtpServerFactory;
import org.apache.ftpserver.ftplet.Authentication;
@@ -32,18 +35,25 @@ import org.apache.ftpserver.usermanager.impl.BaseUser;
import org.apache.ftpserver.usermanager.impl.ConcurrentLoginPermission;
import org.apache.ftpserver.usermanager.impl.TransferRatePermission;
import org.apache.ftpserver.usermanager.impl.WritePermission;
import org.junit.rules.ExternalResource;
import org.junit.rules.TemporaryFolder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.ftp.session.DefaultFtpSessionFactory;
import org.springframework.integration.test.util.SocketUtils;
/**
* Embedded FTP Server for test cases; exposes an associated session factory
* as a @Bean.
*
* @author Artem Bilan
* @author Gary Russell
* @since 3.0
*/
public class FtpServerRule extends ExternalResource {
@Configuration
public class TesFtpServer {
public static int FTP_PORT = SocketUtils.findAvailableServerSocket();
private final int ftpPort = SocketUtils.findAvailableServerSocket();
private final TemporaryFolder ftpFolder;
@@ -61,7 +71,7 @@ public class FtpServerRule extends ExternalResource {
private volatile FtpServer server;
public FtpServerRule(final String root) {
public TesFtpServer(final String root) {
this.ftpFolder = new TemporaryFolder() {
@Override
@@ -124,8 +134,22 @@ public class FtpServerRule extends ExternalResource {
return targetLocalDirectory;
}
@Override
protected void before() throws Throwable {
public String getTargetLocalDirectoryName() {
return targetLocalDirectory.getAbsolutePath() + File.separator;
}
@Bean
public DefaultFtpSessionFactory ftpSessionFactory() {
DefaultFtpSessionFactory factory = new DefaultFtpSessionFactory();
factory.setHost("localhost");
factory.setPort(this.ftpPort);
factory.setUsername("foo");
factory.setPassword("foo");
return factory;
}
@PostConstruct
public void before() throws Throwable {
this.ftpFolder.create();
this.localFolder.create();
@@ -133,7 +157,7 @@ public class FtpServerRule extends ExternalResource {
serverFactory.setUserManager(new TestUserManager(this.ftpRootFolder.getAbsolutePath()));
ListenerFactory factory = new ListenerFactory();
factory.setPort(FTP_PORT);
factory.setPort(ftpPort);
serverFactory.addListener("default", factory.createListener());
server = serverFactory.createServer();
@@ -141,8 +165,8 @@ public class FtpServerRule extends ExternalResource {
}
@Override
protected void after() {
@PreDestroy
public void after() {
this.server.stop();
this.ftpFolder.delete();
this.localFolder.delete();

35
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml
View File

@@ -8,15 +8,10 @@
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="ftpSessionFactory" class="org.springframework.integration.ftp.session.DefaultFtpSessionFactory">
<property name="host" value="localhost"/>
<property name="port" value="#{T(org.springframework.integration.ftp.FtpServerRule).FTP_PORT}"/>
<property name="username" value="foo"/>
<property name="password" value="foo"/>
<bean id="ftpServer" class="org.springframework.integration.ftp.TesFtpServer">
<constructor-arg value="FtpServerOutboundTests"/>
</bean>
<int:spel-function id="localDir" class="org.springframework.integration.ftp.outbound.FtpServerOutboundTests" method="localDirectory"/>
<int:channel id="output">
<int:queue/>
</int:channel>
@@ -27,7 +22,7 @@
request-channel="inboundGet"
command="get"
expression="payload"
local-directory-expression="#localDir() + #remoteDirectory.toUpperCase()"
local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory.toUpperCase()"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
@@ -46,9 +41,31 @@
request-channel="inboundMGet"
command="mget"
expression="payload"
local-directory-expression="#localDir() + #remoteDirectory"
local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMGetRecursive"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMGetRecursive"
command="mget"
expression="payload"
command-options="-R"
local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMGetRecursiveFiltered"/>
<int-ftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMGetRecursiveFiltered"
command="mget"
expression="payload"
command-options="-R"
filename-regex="(subFtpSource|.*1.txt)"
local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
</beans>

56
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java
View File

@@ -16,6 +16,7 @@
package org.springframework.integration.ftp.outbound;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -25,7 +26,6 @@ import java.util.List;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -33,7 +33,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.channel.DirectChannel;
import org.springframework.integration.core.PollableChannel;
import org.springframework.integration.ftp.FtpServerRule;
import org.springframework.integration.ftp.TesFtpServer;
import org.springframework.integration.message.GenericMessage;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@@ -46,8 +46,8 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
@RunWith(SpringJUnit4ClassRunner.class)
public class FtpServerOutboundTests {
@ClassRule
public static final FtpServerRule FTP_SERVER = new FtpServerRule(FtpServerOutboundTests.class.getSimpleName());
@Autowired
public TesFtpServer ftpServer;
@Autowired
private PollableChannel output;
@@ -61,10 +61,16 @@ public class FtpServerOutboundTests {
@Autowired
private DirectChannel inboundMGet;
@Autowired
private DirectChannel inboundMGetRecursive;
@Autowired
private DirectChannel inboundMGetRecursiveFiltered;
@Before
public void setup() {
FtpServerRule.recursiveDelete(FTP_SERVER.getTargetLocalDirectory());
FtpServerRule.recursiveDelete(FTP_SERVER.getTargetFtpDirectory());
TesFtpServer.recursiveDelete(ftpServer.getTargetLocalDirectory());
TesFtpServer.recursiveDelete(ftpServer.getTargetFtpDirectory());
}
@Test
@@ -125,9 +131,43 @@ public class FtpServerOutboundTests {
}
}
public static String localDirectory() {
return FTP_SERVER.getTargetLocalDirectory().getAbsolutePath() + File.separator;
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursive() {
String dir = "ftpSource/";
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
assertEquals(3, localFiles.size());
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir));
}
assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir + "subFtpSource"));
}
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() {
String dir = "ftpSource/";
this.inboundMGetRecursiveFiltered.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
// should have filtered ftpSource2.txt
assertEquals(2, localFiles.size());
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir));
}
assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir + "subFtpSource"));
}
}

17
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2011 the original author or authors.
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway;
import org.springframework.integration.file.remote.session.SessionFactory;
@@ -29,7 +30,7 @@ import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* Outbound Gateway for performing remote file operations via SFTP.
*
*
* @author Gary Russell
* @since 2.1
*/
@@ -59,6 +60,11 @@ public class SftpOutboundGateway extends AbstractRemoteFileOutboundGateway<LsEnt
return file.getFilename();
}
@Override
protected String getFilename(AbstractFileInfo<LsEntry> file) {
return file.getFilename();
}
@Override
protected List<AbstractFileInfo<LsEntry>> asFileInfoList(Collection<LsEntry> files) {
List<AbstractFileInfo<LsEntry>> canonicalFiles = new ArrayList<AbstractFileInfo<LsEntry>>();
@@ -73,4 +79,11 @@ public class SftpOutboundGateway extends AbstractRemoteFileOutboundGateway<LsEnt
return ((long)file.getAttrs().getMTime()) * 1000;
}
@Override
protected LsEntry enhanceNameWithSubDirectory(LsEntry file, String directory) {
DirectFieldAccessor accessor = new DirectFieldAccessor(file);
accessor.setPropertyValue("filename", directory + file.getFilename());
return file;
}
}

23
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml
View File

@@ -41,6 +41,29 @@
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMGetRecursive"/>
<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMGetRecursive"
command="mget"
expression="payload"
command-options="-R"
local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMGetRecursiveFiltered"/>
<int-sftp:outbound-gateway session-factory="ftpSessionFactory"
request-channel="inboundMGetRecursiveFiltered"
command="mget"
expression="payload"
command-options="-R"
filename-regex="(subSftpSource|.*1.txt)"
local-directory-expression="'/tmp/sftpOutboundTests/' + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
reply-channel="output"/>
<bean id="ftpSessionFactory" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="org.springframework.integration.file.remote.session.SessionFactory" />
</bean>

51
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java
View File

@@ -16,6 +16,7 @@
package org.springframework.integration.sftp.outbound;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -32,6 +33,7 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.channel.DirectChannel;
@@ -80,6 +82,12 @@ public class SftpServerOutboundTests {
@Autowired
private DirectChannel inboundMGet;
@Autowired
private DirectChannel inboundMGetRecursive;
@Autowired
private DirectChannel inboundMGetRecursiveFiltered;
@Autowired
private SessionFactory<SftpFileInfo> sessionFactory;
@@ -110,7 +118,9 @@ public class SftpServerOutboundTests {
LsEntry entry4 = mock(LsEntry.class);
SftpATTRS attrs4 = mock(SftpATTRS.class);
when(entry4.getAttrs()).thenReturn(attrs4);
when(entry4.getFilename()).thenReturn("subSftpSource1.txt");
// recursion uses a DFA to update the filename to include the subdirectory
new DirectFieldAccessor(entry4).setPropertyValue("filename", "subSftpSource1.txt");
when(entry4.getFilename()).thenCallRealMethod();
when(session.list("sftpSource/sftpSource1.txt")).thenReturn(new LsEntry[] {
entry1
});
@@ -203,4 +213,43 @@ public class SftpServerOutboundTests {
}
}
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursive() {
String dir = "sftpSource/";
this.inboundMGetRecursive.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
assertEquals(3, localFiles.size());
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir));
}
assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir + "subSftpSource"));
}
@Test
@SuppressWarnings("unchecked")
public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() {
String dir = "sftpSource/";
this.inboundMGetRecursiveFiltered.send(new GenericMessage<Object>(dir + "*"));
Message<?> result = this.output.receive(1000);
assertNotNull(result);
List<File> localFiles = (List<File>) result.getPayload();
// should have filtered sftpSource2.txt
assertEquals(2, localFiles.size());
for (File file : localFiles) {
assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir));
}
assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"),
Matchers.containsString(dir + "subSftpSource"));
}
}

29
src/reference/docbook/ftp.xml
View File

@@ -351,6 +351,7 @@ protected void postProcessClientBeforeConnect(T client) throws IOException {
<listitem>-f - do not sort the list</listitem>
<listitem>-dirs - include directories (excluded by default)</listitem>
<listitem>-links - include symbolic links (excluded by default)</listitem>
<listitem>-R - list the remote directory recursively</listitem>
</itemizedlist>
</para>
<para>
@@ -366,6 +367,13 @@ protected void postProcessClientBeforeConnect(T client) throws IOException {
The remote directory that the <emphasis>ls</emphasis> command acted on is provided
in the <classname>file_remoteDirectory</classname> header.
</para>
<para>
When using the recursive option (<code>-R</code>), the <code>fileName</code> includes any subdirectory
elements, representing a relative path to the file (relative to the remote directory). If the <code>-dirs</code>
option is included, each recursive directory is also returned as an element in the list. In this case,
it is recommended that the <code>-1</code> is not used because you would not be able to determine files Vs.
directories, which is achievable using the <code>FileInfo</code> objects.
</para>
<para><emphasis role="bold">get</emphasis></para>
<para>
<emphasis>get</emphasis> retrieves a remote file and supports the following option:
@@ -399,6 +407,27 @@ protected void postProcessClientBeforeConnect(T client) throws IOException {
for the filenames is
provided in the <classname>file_remoteFile</classname> header.
</para>
<note>
<title>Notes for when using recursion (<code>-R</code>)</title>
<para>
The pattern is ignored, and <code>*</code> is assumed. By
default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a
<classname>FileListFilter</classname>; directories in the tree can also be filtered this way.
A <classname>FileListFilter</classname> can be provided by reference or by <code>filename-pattern</code>
or <code>filename-regex</code> attributes. For example,
<code>filename-regex="(subDir|.*1.txt)"</code> will retrieve all files ending with <code>1.txt</code> in the
remote directory and the subdirectory <code>subDir</code>. If a subdirectory is filtered, no additional
traversal of that subdirectory is performed.
</para>
<para>
The <code>-dirs</code> option is not allowed (the recursive mget uses the recursive <code>ls</code> to
obtain the directory tree and the directories themselves cannot be included in the list).
</para>
<para>
Typically, you would use the <code>#remoteDirectory</code> variable in the <code>local-directory-expression</code>
so that the remote directory structure is retained locally.
</para>
</note>
<para><emphasis role="bold">rm</emphasis></para>
<para>
The <emphasis>rm</emphasis> command has no options.

29
src/reference/docbook/sftp.xml
View File

@@ -387,6 +387,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
<listitem>-f - do not sort the list</listitem>
<listitem>-dirs - include directories (excluded by default)</listitem>
<listitem>-links - include symbolic links (excluded by default)</listitem>
<listitem>-R - list the remote directory recursively</listitem>
</itemizedlist>
</para>
<para>
@@ -402,6 +403,13 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
The remote directory that the <emphasis>ls</emphasis> command acted on is provided
in the <classname>file_remoteDirectory</classname> header.
</para>
<para>
When using the recursive option (<code>-R</code>), the <code>fileName</code> includes any subdirectory
elements, representing a relative path to the file (relative to the remote directory). If the <code>-dirs</code>
option is included, each recursive directory is also returned as an element in the list. In this case,
it is recommended that the <code>-1</code> is not used because you would not be able to determine files Vs.
directories, which is achievable using the <code>FileInfo</code> objects.
</para>
<para><emphasis role="bold">get</emphasis></para>
<para>
<emphasis>get</emphasis> retrieves a remote file and supports the following option:
@@ -435,6 +443,27 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp
for the filenames is
provided in the <classname>file_remoteFile</classname> header.
</para>
<note>
<title>Notes for when using recursion (<code>-R</code>)</title>
<para>
The pattern is ignored, and <code>*</code> is assumed. By
default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a
<classname>FileListFilter</classname>; directories in the tree can also be filtered this way.
A <classname>FileListFilter</classname> can be provided by reference or by <code>filename-pattern</code>
or <code>filename-regex</code> attributes. For example,
<code>filename-regex="(subDir|.*1.txt)"</code> will retrieve all files ending with <code>1.txt</code> in the
remote directory and the subdirectory <code>subDir</code>. If a subdirectory is filtered, no additional
traversal of that subdirectory is performed.
</para>
<para>
The <code>-dirs</code> option is not allowed (the recursive mget uses the recursive <code>ls</code> to
obtain the directory tree and the directories themselves cannot be included in the list).
</para>
<para>
Typically, you would use the <code>#remoteDirectory</code> variable in the <code>local-directory-expression</code>
so that the remote directory structure is retained locally.
</para>
</note>
<para><emphasis role="bold">rm</emphasis></para>
<para>
The <emphasis>rm</emphasis> command has no options.

30
src/reference/docbook/whats-new.xml
View File

@@ -238,17 +238,25 @@
<section id="3.0-xFTP-gw">
<title>FTP, SFTP and FTPS Gateways</title>
<para>
The gateways now support the <code>mv</code> command, enabling the renaming of remote
files.
</para>
<para>
The <code>local-filename-generator-expression</code> attribute is now supported,
enabling the naming of local files during transfer. By default, the same
name as the remote file is used.
</para>
<para>
The <code>local-directory-expression</code> attribute is now supported,
enabling the naming of local directories during transfer based on the remote directory.
<itemizedlist>
<listitem>
The gateways now support the <code>mv</code> command, enabling the renaming of remote
files.
</listitem>
<listitem>
The gateways now support recursive <code>ls</code> and <code>mget</code> commands, enabling
the retrieval of a remote file tree.
</listitem>
<listitem>
The <code>local-filename-generator-expression</code> attribute is now supported,
enabling the naming of local files during transfer. By default, the same
name as the remote file is used.
</listitem>
<listitem>
The <code>local-directory-expression</code> attribute is now supported,
enabling the naming of local directories during transfer based on the remote directory.
</listitem>
</itemizedlist>
</para>
<para>
For more information, see <xref linkend="ftp-outbound-gateway"/> and <xref linkend="sftp-outbound-gateway"/>.