INT-4149: Improve (S)FTP Recursive MGET

JIRA: https://jira.spring.io/browse/INT-4149

Add an option to always pass directories while recursing.

* Fix typos in docs
* Revert `SyslogReceivingChannelAdapterTests` `Thread.sleep()` fixes

spring-integration-file/src/main/java/org/springframework/integration/file/config/FileListFilterFactoryBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2015 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ import org.springframework.integration.file.filters.SimplePatternFileListFilter;
 /**
  * @author Mark Fisher
  * @author Gunnar Hillert
+ * @author Gary Russell
  * @since 1.0.3
  */
 public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<File>> {
@@ -48,6 +49,8 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 
 	private volatile Boolean preventDuplicates;
 
+	private volatile Boolean alwaysAcceptDirectories;
+
 	private final Object monitor = new Object();
 
 	public void setFilter(FileListFilter<File> filter) {
@@ -76,6 +79,18 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 		this.preventDuplicates = preventDuplicates;
 	}
 
+	/**
+	 * Set to true to indicate that the pattern should not be applied to directories.
+	 * Used for recursive scans for file patterns, for example in gateway recursive
+	 * mget operations. Only applies when a pattern or regex is provided.
+	 * @param alwaysAcceptDirectories true to always pass directories.
+	 * @since 5.0
+	 */
+	public void setAlwaysAcceptDirectories(Boolean alwaysAcceptDirectories) {
+		this.alwaysAcceptDirectories = alwaysAcceptDirectories;
+	}
+
+	@Override
 	public FileListFilter<File> getObject() throws Exception {
 		if (this.result == null) {
 			synchronized (this.monitor) {
@@ -85,10 +100,12 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 		return this.result;
 	}
 
+	@Override
 	public Class<?> getObjectType() {
 		return (this.result != null) ? this.result.getClass() : FileListFilter.class;
 	}
 
+	@Override
 	public boolean isSingleton() {
 		return true;
 	}
@@ -132,10 +149,18 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 				filtersNeeded.add(new AcceptOnceFileListFilter<File>());
 			}
 			if (this.filenamePattern != null) {
-				filtersNeeded.add(new SimplePatternFileListFilter(this.filenamePattern));
+				SimplePatternFileListFilter patternFilter = new SimplePatternFileListFilter(this.filenamePattern);
+				if (this.alwaysAcceptDirectories != null) {
+					patternFilter.setAlwaysAcceptDirectories(this.alwaysAcceptDirectories);
+				}
+				filtersNeeded.add(patternFilter);
 			}
 			if (this.filenameRegex != null) {
-				filtersNeeded.add(new RegexPatternFileListFilter(this.filenameRegex));
+				RegexPatternFileListFilter regexFilter = new RegexPatternFileListFilter(this.filenameRegex);
+				if (this.alwaysAcceptDirectories != null) {
+					regexFilter.setAlwaysAcceptDirectories(this.alwaysAcceptDirectories);
+				}
+				filtersNeeded.add(regexFilter);
 			}
 		}
 

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractDirectoryAwareFileListFilter.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.file.filters;
+
+/**
+ * A file list filter that can be configured to always accept (pass) directories.
+ * This permits, for example, pattern matching on just files when using recursion
+ * to examine a directory tree.
+ *
+ * @author Gary Russell
+ * @since 5.0
+ *
+ */
+public abstract class AbstractDirectoryAwareFileListFilter<F> extends AbstractFileListFilter<F> {
+
+	private boolean alwaysAcceptDirectories;
+
+	/**
+	 * Set to true so that filters that support this feature can unconditionally pass
+	 * directories; default false.
+	 * @param alwaysAcceptDirectories true to always pass directories.
+	 */
+	public void setAlwaysAcceptDirectories(boolean alwaysAcceptDirectories) {
+		this.alwaysAcceptDirectories = alwaysAcceptDirectories;
+	}
+
+	protected boolean alwaysAccept(F file) {
+		return file != null && this.alwaysAcceptDirectories && isDirectory(file);
+	}
+
+	/**
+	 * Subclasses must implement this method to indicate whether the file
+	 * is a directory or not.
+	 * @param file the file.
+	 * @return true if it's a directory.
+	 */
+	protected abstract boolean isDirectory(F file);
+
+}

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java

@@ -30,7 +30,8 @@ import org.springframework.util.Assert;
  * @param <F> the type of file entry
  * @since 2.0
  */
-public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractFileListFilter<F> implements InitializingBean {
+public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractDirectoryAwareFileListFilter<F>
+		implements InitializingBean {
 
 	private volatile Pattern pattern;
 
@@ -59,7 +60,7 @@ public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractFile
 
 	@Override
 	public boolean accept(F file) {
-		return (file != null) && this.pattern.matcher(this.getFilename(file)).matches();
+		return alwaysAccept(file) || (file != null && this.pattern.matcher(getFilename(file)).matches());
 	}
 
 	/**

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java

@@ -29,7 +29,7 @@ import org.springframework.util.AntPathMatcher;
  * @see org.springframework.integration.file.filters.AbstractRegexPatternFileListFilter
  * @since 2.0
  */
-public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractFileListFilter<F> {
+public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractDirectoryAwareFileListFilter<F> {
 
 	private final AntPathMatcher matcher = new AntPathMatcher();
 
@@ -46,7 +46,7 @@ public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractFil
 	 */
 	@Override
 	public final boolean accept(F file) {
-		return this.matcher.match(this.path, this.getFilename(file));
+		return alwaysAccept(file) || (file != null && this.matcher.match(this.path, this.getFilename(file)));
 	}
 
 	/**

spring-integration-file/src/main/java/org/springframework/integration/file/filters/RegexPatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import java.util.regex.Pattern;
  * Implementation of AbstractRegexPatternMatchingFileListFilter for java.io.File instances.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  */
 public class RegexPatternFileListFilter extends AbstractRegexPatternFileListFilter<File> {
 
@@ -40,4 +41,9 @@ public class RegexPatternFileListFilter extends AbstractRegexPatternFileListFilt
 		return (file != null) ? file.getName() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(File file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/filters/SimplePatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import java.io.File;
  * This filter only filters on the name of the file, the rest of the path is ignored.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class SimplePatternFileListFilter extends AbstractSimplePatternFileListFilter<File> {
@@ -37,4 +38,10 @@ public class SimplePatternFileListFilter extends AbstractSimplePatternFileListFi
 		return file.getName();
 	}
 
+
+	@Override
+	protected boolean isDirectory(File file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-file/src/test/java/org/springframework/integration/file/config/FileListFilterFactoryBeanTests.java

@@ -16,8 +16,8 @@
 
 package org.springframework.integration.file.config;
 
-import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
@@ -35,6 +35,7 @@ import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.filters.SimplePatternFileListFilter;
+import org.springframework.integration.test.util.TestUtils;
 
 /**
  * @author Mark Fisher
@@ -117,7 +118,9 @@ public class FileListFilterFactoryBeanTests {
 				new DirectFieldAccessor(result).getPropertyValue("fileFilters");
 		Iterator<FileListFilter<?>> iterator = filters.iterator();
 		assertTrue(iterator.next() instanceof AcceptOnceFileListFilter);
-		assertThat(iterator.next(), is(instanceOf(SimplePatternFileListFilter.class)));
+		FileListFilter<?> patternFilter = iterator.next();
+		assertThat(patternFilter, is(instanceOf(SimplePatternFileListFilter.class)));
+		assertFalse(TestUtils.getPropertyValue(patternFilter, "alwaysAcceptDirectories", Boolean.class));
 	}
 
 	@Test
@@ -125,10 +128,12 @@ public class FileListFilterFactoryBeanTests {
 		FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
 		factory.setIgnoreHidden(false);
 		factory.setFilenamePattern(("foo"));
+		factory.setAlwaysAcceptDirectories(true);
 		factory.setPreventDuplicates(Boolean.FALSE);
 		FileListFilter<File> result = factory.getObject();
 		assertFalse(result instanceof CompositeFileListFilter);
 		assertThat(result, is(instanceOf(SimplePatternFileListFilter.class)));
+		assertTrue(TestUtils.getPropertyValue(result, "alwaysAcceptDirectories", Boolean.class));
 	}
 
 	private static class TestFilter extends AbstractFileListFilter<File> {

spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java

@@ -1051,7 +1051,7 @@ public class RemoteFileOutboundGatewayTests {
 
 	static class TestRemoteFileOutboundGateway extends AbstractRemoteFileOutboundGateway<TestLsEntry> {
 
-		@SuppressWarnings({ "rawtypes", "unchecked" })
+		@SuppressWarnings("unchecked")
 		TestRemoteFileOutboundGateway(SessionFactory sessionFactory,
 				String command, String expression) {
 			super(sessionFactory, Command.toCommand(command), expression);
@@ -1180,6 +1180,11 @@ public class RemoteFileOutboundGatewayTests {
 			return file.getFilename();
 		}
 
+		@Override
+		protected boolean isDirectory(TestLsEntry file) {
+			return file.isDirectory();
+		}
+
 	}
 
 }

spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpRegexPatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import org.springframework.integration.file.filters.AbstractRegexPatternFileList
  * Implementation of {@link AbstractRegexPatternFileListFilter} for FTP.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class FtpRegexPatternFileListFilter extends AbstractRegexPatternFileListFilter<FTPFile> {
@@ -44,4 +45,9 @@ public class FtpRegexPatternFileListFilter extends AbstractRegexPatternFileListF
 		return (file != null) ? file.getName() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(FTPFile file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpSimplePatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,6 +24,7 @@ import org.springframework.integration.file.filters.AbstractSimplePatternFileLis
  * Implementation of {@link AbstractSimplePatternFileListFilter} for FTP.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class FtpSimplePatternFileListFilter extends AbstractSimplePatternFileListFilter<FTPFile> {
@@ -38,4 +39,9 @@ public class FtpSimplePatternFileListFilter extends AbstractSimplePatternFileLis
 		return (file != null) ? file.getName() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(FTPFile file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml

@@ -52,10 +52,16 @@
 							  command="mget"
 							  expression="payload"
 							  command-options="-R"
+							  filter="startDotTxtFilter"
 							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
+	<bean id="startDotTxtFilter" class="org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter">
+		<constructor-arg value="*.txt" />
+		<property name="alwaysAcceptDirectories" value="true" />
+	</bean>
+
 	<int:channel id="inboundMGetRecursiveFiltered"/>
 
 	<int-ftp:outbound-gateway session-factory="ftpSessionFactory"

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpRegexPatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import com.jcraft.jsch.ChannelSftp.LsEntry;
  * Implementation of {@link AbstractRegexPatternFileListFilter} for SFTP.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class SftpRegexPatternFileListFilter extends AbstractRegexPatternFileListFilter<ChannelSftp.LsEntry> {
@@ -45,4 +46,9 @@ public class SftpRegexPatternFileListFilter extends AbstractRegexPatternFileList
 		return (entry != null) ? entry.getFilename() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(LsEntry file) {
+		return file.getAttrs().isDir();
+	}
+
 }

spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpSimplePatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import com.jcraft.jsch.ChannelSftp.LsEntry;
  * Implementation of {@link AbstractSimplePatternFileListFilter} for SFTP.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class SftpSimplePatternFileListFilter extends AbstractSimplePatternFileListFilter<ChannelSftp.LsEntry> {
@@ -39,4 +40,9 @@ public class SftpSimplePatternFileListFilter extends AbstractSimplePatternFileLi
 		return (entry != null) ? entry.getFilename() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(LsEntry file) {
+		return file.getAttrs().isDir();
+	}
+
 }

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml

@@ -51,10 +51,17 @@
 							  command="mget"
 							  expression="payload"
 							  command-options="-R"
+							  filter="dotStarDotTxtFilter"
 							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
+	<bean id="dotStarDotTxtFilter"
+				class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
+		<constructor-arg value="^.*\.txt$" />
+		<property name="alwaysAcceptDirectories" value="true" />
+	</bean>
+
 	<int:channel id="inboundMGetRecursiveFiltered"/>
 
 	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"

spring-integration-syslog/src/test/java/org/springframework/integration/syslog/inbound/SyslogReceivingChannelAdapterTests.java

@@ -74,6 +74,7 @@ public class SyslogReceivingChannelAdapterTests {
 		factory.afterPropertiesSet();
 		factory.start();
 		UdpSyslogReceivingChannelAdapter adapter = (UdpSyslogReceivingChannelAdapter) factory.getObject();
+		Thread.sleep(1000);
 		byte[] buf = "<157>JUL 26 22:08:35 WEBERN TESTING[70729]: TEST SYSLOG MESSAGE".getBytes("UTF-8");
 		DatagramPacket packet = new DatagramPacket(buf, buf.length, new InetSocketAddress("localhost", port));
 		DatagramSocket socket = new DatagramSocket();
@@ -115,6 +116,7 @@ public class SyslogReceivingChannelAdapterTests {
 			return null;
 		}).when(logger).debug(anyString());
 		new DirectFieldAccessor(adapter).setPropertyValue("logger", logger);
+		Thread.sleep(1000);
 		byte[] buf = "<157>JUL 26 22:08:35 WEBERN TESTING[70729]: TEST SYSLOG MESSAGE\n".getBytes("UTF-8");
 		Socket socket = SocketFactory.getDefault().createSocket("localhost", port);
 		socket.getOutputStream().write(buf);
@@ -142,6 +144,7 @@ public class SyslogReceivingChannelAdapterTests {
 		DefaultMessageConverter defaultMessageConverter = new DefaultMessageConverter();
 		defaultMessageConverter.setAsMap(false);
 		adapter.setConverter(defaultMessageConverter);
+		Thread.sleep(1000);
 		byte[] buf = "<157>JUL 26 22:08:35 WEBERN TESTING[70729]: TEST SYSLOG MESSAGE".getBytes("UTF-8");
 		DatagramPacket packet = new DatagramPacket(buf, buf.length, new InetSocketAddress("localhost", port));
 		DatagramSocket socket = new DatagramSocket();
@@ -188,6 +191,7 @@ public class SyslogReceivingChannelAdapterTests {
 			return null;
 		}).when(logger).debug(anyString());
 		new DirectFieldAccessor(adapter).setPropertyValue("logger", logger);
+		Thread.sleep(1000);
 		byte[] buf = ("253 <14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - " +
 				"[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"]" +
 				"[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"] Removing instance")
@@ -217,6 +221,7 @@ public class SyslogReceivingChannelAdapterTests {
 		factory.afterPropertiesSet();
 		factory.start();
 		UdpSyslogReceivingChannelAdapter adapter = (UdpSyslogReceivingChannelAdapter) factory.getObject();
+		Thread.sleep(1000);
 		byte[] buf = ("<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - " +
 				"[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"]" +
 				"[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"] Removing instance")

src/reference/asciidoc/ftp.adoc

@@ -790,6 +790,8 @@ By default, the entire remote tree is retrieved.
 However, files in the tree can be filtered, by providing a`FileListFilter`; directories in the tree can also be filtered this way.
 A `FileListFilter` can be provided by reference or by `filename-pattern` or `filename-regex` attributes.
 For example, `filename-regex="(subDir|.*1.txt)"` will retrieve all files ending with `1.txt` in the remote directory and the subdirectory `subDir`.
+However, see below for an alternative available in _version 5.0_.
+
 If a subdirectory is filtered, no additional traversal of that subdirectory is performed.
 
 The `-dirs` option is not allowed (the recursive mget uses the recursive `ls` to obtain the directory tree and the directories themselves cannot be included in the list).
@@ -797,6 +799,25 @@ The `-dirs` option is not allowed (the recursive mget uses the recursive `ls` to
 Typically, you would use the `#remoteDirectory` variable in the `local-directory-expression` so that the remote directory structure is retained locally.
 =====
 
+Starting with _version 5.0_, the `FtpSimplePatternFileListFilter` and `FtpRegexPatternFileListFilter` can be configured to always pass directories by setting the `alwaysAcceptDirectorties` to `true`.
+This allows recursion for a simple pattern; examples follow:
+
+[code, xml]
+----
+<bean id="startDotTxtFilter" class="org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter">
+    <constructor-arg value="*.txt" />
+    <property name="alwaysAcceptDirectories" value="true" />
+</bean>
+
+<bean id="dotStarDotTxtFilter"
+            class="org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter">
+    <constructor-arg value="^.*\.txt$" />
+    <property name="alwaysAcceptDirectories" value="true" />
+</bean>
+----
+
+and provide one of these filters using `filter` property on the gateway.
+
 See also <<ftp-partial>>.
 
 *put*

src/reference/asciidoc/sftp.adoc

@@ -879,6 +879,8 @@ By default, the entire remote tree is retrieved.
 However, files in the tree can be filtered, by providing a`FileListFilter`; directories in the tree can also be filtered this way.
 A `FileListFilter` can be provided by reference or by `filename-pattern` or `filename-regex` attributes.
 For example, `filename-regex="(subDir|.*1.txt)"` will retrieve all files ending with `1.txt` in the remote directory and the subdirectory `subDir`.
+However, see below for an alternative available in _version 5.0_.
+
 If a subdirectory is filtered, no additional traversal of that subdirectory is performed.
 
 The `-dirs` option is not allowed (the recursive mget uses the recursive `ls` to obtain the directory tree and the directories themselves cannot be included in the list).
@@ -886,6 +888,26 @@ The `-dirs` option is not allowed (the recursive mget uses the recursive `ls` to
 Typically, you would use the `#remoteDirectory` variable in the `local-directory-expression` so that the remote directory structure is retained locally.
 =====
 
+Starting with _version 5.0_, the `SftpSimplePatternFileListFilter` and `SftpRegexPatternFileListFilter` can be configured to always pass directories by setting the `alwaysAcceptDirectorties` to `true`.
+This allows recursion for a simple pattern; examples follow:
+
+[code, xml]
+----
+<bean id="startDotTxtFilter" class="org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter">
+    <constructor-arg value="*.txt" />
+    <property name="alwaysAcceptDirectories" value="true" />
+</bean>
+
+<bean id="dotStarDotTxtFilter"
+            class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
+    <constructor-arg value="^.*\.txt$" />
+    <property name="alwaysAcceptDirectories" value="true" />
+</bean>
+----
+
+and provide one of these filters using `filter` property on the gateway.
+
+
 See also <<sftp-partial>>
 
 *put*

src/reference/asciidoc/whats-new.adoc

@@ -46,6 +46,10 @@ See <<file-reading>> for more information.
 
 The inbound channel adapters now have a property `max-fetch-size` which is used to limit the number of files fetched during a poll when there are no files currently in the local directory.
 
+The regex and pattern filters can now be configured to always pass directories.
+This can be useful when using recursion in the outbound gateways.
+See <<ftp-outbound-gateway>> and <<sftp-outbound-gateway>> for more information.
+
 ==== Integration Properties
 
 Since _version 4.3.2_ a new `spring.integration.readOnly.headers` global property has been added to customize the list of headers which should not be copied to a newly created `Message` by the `MessageBuilder`.