INT-4115: (S)FtpPersistentFileFilter by Default

JIRA: https://jira.spring.io/browse/INT-4115

Apply `(S)FtpPersistentAcceptOnceFileListFilter` for the `(S)FtpInboundFileSynchronizer` by default to avoid cases to sync the same remote files to the local directory again.
Especially when `localFileName` strategy is applied and we end up with new local files, but with the same remote content

Accept `(S)FtpPersistentAcceptOnceFileListFilter` for streaming adapters

Make `doSetFilter()` as `protected final`

Fix tests after rebase

Fix "What's New" after rebase

Compose `PersistentAcceptOnceFileListFilter` together with the regex or pattern filters
Document such a behavior

Address PR comments for formatting and typo

spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java

@@ -25,6 +25,7 @@ import org.springframework.beans.factory.xml.ParserContext;
 import org.springframework.integration.config.ExpressionFactoryBean;
 import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
 import org.springframework.util.StringUtils;
@@ -36,6 +37,7 @@ import org.springframework.util.StringUtils;
  * @author Mark Fisher
  * @author Gary Russell
  * @author Artem Bilan
+ *
  * @since 2.0
  */
 public abstract class AbstractRemoteFileInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@@ -59,8 +61,10 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
 		String remoteFileSeparator = element.getAttribute("remote-file-separator");
 		synchronizerBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
 		IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "temporary-file-suffix");
+
 		FileParserUtils.configureFilter(synchronizerBuilder, element, parserContext,
-				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass());
+				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass(),
+				getPersistentAcceptOnceFileListFilterClass());
 
 		// build the MessageSource
 		BeanDefinitionBuilder messageSourceBuilder =
@@ -94,4 +98,6 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
 
 	protected abstract Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass();
 
+	protected abstract Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass();
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileStreamingInboundChannelAdapterParser.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 the original author or authors.
+ * Copyright 2016-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import org.springframework.beans.factory.xml.ParserContext;
 import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
 import org.springframework.integration.core.MessageSource;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.RemoteFileOperations;
 import org.springframework.util.StringUtils;
@@ -33,6 +34,8 @@ import org.springframework.util.StringUtils;
  * Abstract base class for parsing remote file streaming inbound channel adapters.
  *
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 4.3
  */
 public abstract class AbstractRemoteFileStreamingInboundChannelAdapterParser
@@ -56,7 +59,7 @@ public abstract class AbstractRemoteFileStreamingInboundChannelAdapterParser
 		String remoteFileSeparator = element.getAttribute("remote-file-separator");
 		messageSourceBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
 		FileParserUtils.configureFilter(messageSourceBuilder, element, parserContext,
-				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass());
+				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass(), getPersistentAcceptOnceFileListFilterClass());
 
 		String comparator = element.getAttribute("comparator");
 		if (StringUtils.hasText(comparator)) {
@@ -74,4 +77,6 @@ public abstract class AbstractRemoteFileStreamingInboundChannelAdapterParser
 
 	protected abstract Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass();
 
+	protected abstract Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass();
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java

@@ -20,12 +20,16 @@ import org.w3c.dom.Element;
 
 import org.springframework.beans.factory.config.BeanDefinition;
 import org.springframework.beans.factory.support.BeanDefinitionBuilder;
+import org.springframework.beans.factory.support.ManagedList;
 import org.springframework.beans.factory.xml.ParserContext;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
 import org.springframework.integration.file.DefaultFileNameGenerator;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
+import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.RemoteFileOperations;
+import org.springframework.integration.metadata.SimpleMetadataStore;
 import org.springframework.util.StringUtils;
 
 /**
@@ -34,6 +38,7 @@ import org.springframework.util.StringUtils;
  * @author David Turanski
  * @author Gary Russell
  * @author Artem Bilan
+ *
  * @since 3.0
  *
  */
@@ -93,7 +98,8 @@ public final class FileParserUtils {
 	}
 
 	static void configureFilter(BeanDefinitionBuilder synchronizerBuilder, Element element, ParserContext parserContext,
-			Class<? extends FileListFilter<?>> patternClass, Class<? extends FileListFilter<?>> regexClass) {
+			Class<? extends FileListFilter<?>> patternClass, Class<? extends FileListFilter<?>> regexClass,
+			Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> persistentAcceptOnceFileListFilterClass) {
 		String filter = element.getAttribute("filter");
 		String filterExpression = element.getAttribute("filter-expression");
 		String fileNamePattern = element.getAttribute("filename-pattern");
@@ -132,16 +138,44 @@ public final class FileParserUtils {
 				synchronizerBuilder.addPropertyValue("filter", expressionFilterBeanDefinition);
 			}
 			else if (hasFileNamePattern) {
-				BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition(patternClass);
-				filterBuilder.addConstructorArgValue(fileNamePattern);
-				synchronizerBuilder.addPropertyValue("filter", filterBuilder.getBeanDefinition());
+				BeanDefinition patternFilter =
+						BeanDefinitionBuilder.genericBeanDefinition(patternClass)
+								.addConstructorArgValue(fileNamePattern)
+								.getBeanDefinition();
+
+				composeFilters(synchronizerBuilder, persistentAcceptOnceFileListFilterClass, patternFilter);
 			}
-			else if (hasFileNameRegex) {
-				BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition(regexClass);
-				filterBuilder.addConstructorArgValue(fileNameRegex);
-				synchronizerBuilder.addPropertyValue("filter", filterBuilder.getBeanDefinition());
+			else {
+				BeanDefinition regexFilter = BeanDefinitionBuilder.genericBeanDefinition(regexClass)
+						.addConstructorArgValue(fileNameRegex)
+						.getBeanDefinition();
+
+				composeFilters(synchronizerBuilder, persistentAcceptOnceFileListFilterClass, regexFilter);
 			}
 		}
 	}
 
+	private static void composeFilters(BeanDefinitionBuilder synchronizerBuilder,
+			Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> persistentAcceptOnceFileListFilterClass,
+			BeanDefinition filter) {
+		BeanDefinition persistentFilter =
+				BeanDefinitionBuilder.genericBeanDefinition(persistentAcceptOnceFileListFilterClass)
+						.addConstructorArgValue(
+								BeanDefinitionBuilder
+										.genericBeanDefinition(SimpleMetadataStore.class)
+										.getBeanDefinition())
+						.addConstructorArgValue("remoteFileMessageSource")
+						.getBeanDefinition();
+
+		ManagedList<BeanDefinition> filters = new ManagedList<>();
+		filters.add(filter);
+		filters.add(persistentFilter);
+
+		BeanDefinition compositeFilterDefinition =
+				BeanDefinitionBuilder.genericBeanDefinition(CompositeFileListFilter.class)
+						.addConstructorArgValue(filters)
+						.getBeanDefinition();
+		synchronizerBuilder.addPropertyValue("filter", compositeFilterDefinition);
+	}
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java

@@ -27,7 +27,6 @@ import org.springframework.integration.dsl.ComponentsRegistration;
 import org.springframework.integration.dsl.MessageSourceSpec;
 import org.springframework.integration.expression.FunctionExpression;
 import org.springframework.integration.file.FileReadingMessageSource;
-import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer;
@@ -51,8 +50,6 @@ public abstract class RemoteFileInboundChannelAdapterSpec<F, S extends RemoteFil
 
 	protected final AbstractInboundFileSynchronizer<F> synchronizer;
 
-	private CompositeFileListFilter<F> filter;
-
 	private ExpressionFileListFilter<F> expressionFileListFilter;
 
 	protected RemoteFileInboundChannelAdapterSpec(AbstractInboundFileSynchronizer<F> synchronizer) {
@@ -172,19 +169,7 @@ public abstract class RemoteFileInboundChannelAdapterSpec<F, S extends RemoteFil
 	 * @return the spec.
 	 */
 	public S filter(FileListFilter<F> filter) {
-		if (this.filter == null) {
-			if (filter instanceof CompositeFileListFilter) {
-				this.filter = (CompositeFileListFilter<F>) filter;
-			}
-			else {
-				this.filter = new CompositeFileListFilter<>();
-				this.filter.addFilter(filter);
-			}
-			this.synchronizer.setFilter(this.filter);
-		}
-		else {
-			this.filter.addFilter(filter);
-		}
+		this.synchronizer.setFilter(filter);
 		return _this();
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java

@@ -25,7 +25,6 @@ import org.springframework.integration.dsl.ComponentsRegistration;
 import org.springframework.integration.dsl.MessageSourceSpec;
 import org.springframework.integration.expression.FunctionExpression;
 import org.springframework.integration.file.FileReadingMessageSource;
-import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
@@ -49,8 +48,6 @@ public abstract class RemoteFileStreamingInboundChannelAdapterSpec<F,
 		extends MessageSourceSpec<S, MS>
 		implements ComponentsRegistration {
 
-	private CompositeFileListFilter<F> filter;
-
 	private ExpressionFileListFilter<F> expressionFileListFilter;
 
 	/**
@@ -101,19 +98,7 @@ public abstract class RemoteFileStreamingInboundChannelAdapterSpec<F,
 	 * @return the spec.
 	 */
 	public S filter(FileListFilter<F> filter) {
-		if (this.filter == null) {
-			if (filter instanceof CompositeFileListFilter) {
-				this.filter = (CompositeFileListFilter<F>) filter;
-			}
-			else {
-				this.filter = new CompositeFileListFilter<F>();
-				this.filter.addFilter(filter);
-			}
-			this.target.setFilter(this.filter);
-		}
-		else {
-			this.filter.addFilter(filter);
-		}
+		this.target.setFilter(filter);
 		return _this();
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java

@@ -46,6 +46,8 @@ import org.springframework.util.Assert;
  * referencing a remote file.
  *
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 4.3
  *
  */
@@ -111,6 +113,10 @@ public abstract class AbstractRemoteFileStreamingMessageSource<F>
 	 * @param filter the file list filter.
 	 */
 	public void setFilter(FileListFilter<F> filter) {
+		doSetFilter(filter);
+	}
+
+	protected final void doSetFilter(FileListFilter<F> filter) {
 		this.filter = filter;
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java

@@ -152,11 +152,14 @@ public abstract class AbstractInboundFileSynchronizer<F>
 
 	/**
 	 * Specify an expression that evaluates to the full path to the remote directory.
-	 *
 	 * @param remoteDirectoryExpression The remote directory expression.
 	 * @since 4.2
 	 */
 	public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) {
+		doSetRemoteDirectoryExpression(remoteDirectoryExpression);
+	}
+
+	protected final void doSetRemoteDirectoryExpression(Expression remoteDirectoryExpression) {
 		Assert.notNull(remoteDirectoryExpression, "'remoteDirectoryExpression' must not be null");
 		this.remoteDirectoryExpression = remoteDirectoryExpression;
 	}
@@ -166,6 +169,10 @@ public abstract class AbstractInboundFileSynchronizer<F>
 	 * @param filter the file list filter.
 	 */
 	public void setFilter(FileListFilter<F> filter) {
+		doSetFilter(filter);
+	}
+
+	protected final void doSetFilter(FileListFilter<F> filter) {
 		this.filter = filter;
 	}