INT-4149: Improve (S)FTP Recursive MGET

JIRA: https://jira.spring.io/browse/INT-4149

Add an option to always pass directories while recursing.

* Fix typos in docs
* Revert `SyslogReceivingChannelAdapterTests` `Thread.sleep()` fixes

spring-integration-file/src/main/java/org/springframework/integration/file/config/FileListFilterFactoryBean.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2015 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,6 +32,7 @@ import org.springframework.integration.file.filters.SimplePatternFileListFilter;
 /**
  * @author Mark Fisher
  * @author Gunnar Hillert
+ * @author Gary Russell
  * @since 1.0.3
  */
 public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<File>> {
@@ -48,6 +49,8 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 
 	private volatile Boolean preventDuplicates;
 
+	private volatile Boolean alwaysAcceptDirectories;
+
 	private final Object monitor = new Object();
 
 	public void setFilter(FileListFilter<File> filter) {
@@ -76,6 +79,18 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 		this.preventDuplicates = preventDuplicates;
 	}
 
+	/**
+	 * Set to true to indicate that the pattern should not be applied to directories.
+	 * Used for recursive scans for file patterns, for example in gateway recursive
+	 * mget operations. Only applies when a pattern or regex is provided.
+	 * @param alwaysAcceptDirectories true to always pass directories.
+	 * @since 5.0
+	 */
+	public void setAlwaysAcceptDirectories(Boolean alwaysAcceptDirectories) {
+		this.alwaysAcceptDirectories = alwaysAcceptDirectories;
+	}
+
+	@Override
 	public FileListFilter<File> getObject() throws Exception {
 		if (this.result == null) {
 			synchronized (this.monitor) {
@@ -85,10 +100,12 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 		return this.result;
 	}
 
+	@Override
 	public Class<?> getObjectType() {
 		return (this.result != null) ? this.result.getClass() : FileListFilter.class;
 	}
 
+	@Override
 	public boolean isSingleton() {
 		return true;
 	}
@@ -132,10 +149,18 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
 				filtersNeeded.add(new AcceptOnceFileListFilter<File>());
 			}
 			if (this.filenamePattern != null) {
-				filtersNeeded.add(new SimplePatternFileListFilter(this.filenamePattern));
+				SimplePatternFileListFilter patternFilter = new SimplePatternFileListFilter(this.filenamePattern);
+				if (this.alwaysAcceptDirectories != null) {
+					patternFilter.setAlwaysAcceptDirectories(this.alwaysAcceptDirectories);
+				}
+				filtersNeeded.add(patternFilter);
 			}
 			if (this.filenameRegex != null) {
-				filtersNeeded.add(new RegexPatternFileListFilter(this.filenameRegex));
+				RegexPatternFileListFilter regexFilter = new RegexPatternFileListFilter(this.filenameRegex);
+				if (this.alwaysAcceptDirectories != null) {
+					regexFilter.setAlwaysAcceptDirectories(this.alwaysAcceptDirectories);
+				}
+				filtersNeeded.add(regexFilter);
 			}
 		}
 

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractDirectoryAwareFileListFilter.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.file.filters;
+
+/**
+ * A file list filter that can be configured to always accept (pass) directories.
+ * This permits, for example, pattern matching on just files when using recursion
+ * to examine a directory tree.
+ *
+ * @author Gary Russell
+ * @since 5.0
+ *
+ */
+public abstract class AbstractDirectoryAwareFileListFilter<F> extends AbstractFileListFilter<F> {
+
+	private boolean alwaysAcceptDirectories;
+
+	/**
+	 * Set to true so that filters that support this feature can unconditionally pass
+	 * directories; default false.
+	 * @param alwaysAcceptDirectories true to always pass directories.
+	 */
+	public void setAlwaysAcceptDirectories(boolean alwaysAcceptDirectories) {
+		this.alwaysAcceptDirectories = alwaysAcceptDirectories;
+	}
+
+	protected boolean alwaysAccept(F file) {
+		return file != null && this.alwaysAcceptDirectories && isDirectory(file);
+	}
+
+	/**
+	 * Subclasses must implement this method to indicate whether the file
+	 * is a directory or not.
+	 * @param file the file.
+	 * @return true if it's a directory.
+	 */
+	protected abstract boolean isDirectory(F file);
+
+}

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java

@@ -30,7 +30,8 @@ import org.springframework.util.Assert;
  * @param <F> the type of file entry
  * @since 2.0
  */
-public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractFileListFilter<F> implements InitializingBean {
+public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractDirectoryAwareFileListFilter<F>
+		implements InitializingBean {
 
 	private volatile Pattern pattern;
 
@@ -59,7 +60,7 @@ public abstract class AbstractRegexPatternFileListFilter<F> extends AbstractFile
 
 	@Override
 	public boolean accept(F file) {
-		return (file != null) && this.pattern.matcher(this.getFilename(file)).matches();
+		return alwaysAccept(file) || (file != null && this.pattern.matcher(getFilename(file)).matches());
 	}
 
 	/**

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java

@@ -29,7 +29,7 @@ import org.springframework.util.AntPathMatcher;
  * @see org.springframework.integration.file.filters.AbstractRegexPatternFileListFilter
  * @since 2.0
  */
-public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractFileListFilter<F> {
+public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractDirectoryAwareFileListFilter<F> {
 
 	private final AntPathMatcher matcher = new AntPathMatcher();
 
@@ -46,7 +46,7 @@ public abstract class AbstractSimplePatternFileListFilter<F> extends AbstractFil
 	 */
 	@Override
 	public final boolean accept(F file) {
-		return this.matcher.match(this.path, this.getFilename(file));
+		return alwaysAccept(file) || (file != null && this.matcher.match(this.path, this.getFilename(file)));
 	}
 
 	/**

spring-integration-file/src/main/java/org/springframework/integration/file/filters/RegexPatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import java.util.regex.Pattern;
  * Implementation of AbstractRegexPatternMatchingFileListFilter for java.io.File instances.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  */
 public class RegexPatternFileListFilter extends AbstractRegexPatternFileListFilter<File> {
 
@@ -40,4 +41,9 @@ public class RegexPatternFileListFilter extends AbstractRegexPatternFileListFilt
 		return (file != null) ? file.getName() : null;
 	}
 
+	@Override
+	protected boolean isDirectory(File file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/filters/SimplePatternFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import java.io.File;
  * This filter only filters on the name of the file, the rest of the path is ignored.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
 public class SimplePatternFileListFilter extends AbstractSimplePatternFileListFilter<File> {
@@ -37,4 +38,10 @@ public class SimplePatternFileListFilter extends AbstractSimplePatternFileListFi
 		return file.getName();
 	}
 
+
+	@Override
+	protected boolean isDirectory(File file) {
+		return file.isDirectory();
+	}
+
 }

spring-integration-file/src/test/java/org/springframework/integration/file/config/FileListFilterFactoryBeanTests.java

@@ -16,8 +16,8 @@
 
 package org.springframework.integration.file.config;
 
-import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.CoreMatchers.is;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertThat;
@@ -35,6 +35,7 @@ import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.filters.SimplePatternFileListFilter;
+import org.springframework.integration.test.util.TestUtils;
 
 /**
  * @author Mark Fisher
@@ -117,7 +118,9 @@ public class FileListFilterFactoryBeanTests {
 				new DirectFieldAccessor(result).getPropertyValue("fileFilters");
 		Iterator<FileListFilter<?>> iterator = filters.iterator();
 		assertTrue(iterator.next() instanceof AcceptOnceFileListFilter);
-		assertThat(iterator.next(), is(instanceOf(SimplePatternFileListFilter.class)));
+		FileListFilter<?> patternFilter = iterator.next();
+		assertThat(patternFilter, is(instanceOf(SimplePatternFileListFilter.class)));
+		assertFalse(TestUtils.getPropertyValue(patternFilter, "alwaysAcceptDirectories", Boolean.class));
 	}
 
 	@Test
@@ -125,10 +128,12 @@ public class FileListFilterFactoryBeanTests {
 		FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
 		factory.setIgnoreHidden(false);
 		factory.setFilenamePattern(("foo"));
+		factory.setAlwaysAcceptDirectories(true);
 		factory.setPreventDuplicates(Boolean.FALSE);
 		FileListFilter<File> result = factory.getObject();
 		assertFalse(result instanceof CompositeFileListFilter);
 		assertThat(result, is(instanceOf(SimplePatternFileListFilter.class)));
+		assertTrue(TestUtils.getPropertyValue(result, "alwaysAcceptDirectories", Boolean.class));
 	}
 
 	private static class TestFilter extends AbstractFileListFilter<File> {

spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java

@@ -1051,7 +1051,7 @@ public class RemoteFileOutboundGatewayTests {
 
 	static class TestRemoteFileOutboundGateway extends AbstractRemoteFileOutboundGateway<TestLsEntry> {
 
-		@SuppressWarnings({ "rawtypes", "unchecked" })
+		@SuppressWarnings("unchecked")
 		TestRemoteFileOutboundGateway(SessionFactory sessionFactory,
 				String command, String expression) {
 			super(sessionFactory, Command.toCommand(command), expression);
@@ -1180,6 +1180,11 @@ public class RemoteFileOutboundGatewayTests {
 			return file.getFilename();
 		}
 
+		@Override
+		protected boolean isDirectory(TestLsEntry file) {
+			return file.isDirectory();
+		}
+
 	}
 
 }