INT-4215: Add ChainFileListFilter

JIRA: https://jira.spring.io/browse/INT-4215
Fixes GH-1998 (https://github.com/spring-projects/spring-integration/issues/1998)

Just make an `CompositeFileListFilter` extension which chains result of the previous filter to the next

**Cherry-pick 4.3.x**

build.gradle

@@ -135,7 +135,7 @@ subprojects { subproject ->
 		springSecurityVersion = '4.1.0.RELEASE'
 		springSocialTwitterVersion = '1.1.2.RELEASE'
 		springRetryVersion = '1.1.3.RELEASE'
-		springVersion = project.hasProperty('springVersion') ? project.springVersion : '4.3.6.BUILD-SNAPSHOT'
+		springVersion = project.hasProperty('springVersion') ? project.springVersion : '4.3.6.RELEASE'
 		springWsVersion = '2.3.0.RELEASE'
 		xmlUnitVersion = '1.6'
 		xstreamVersion = '1.4.7'

spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.file.filters;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.springframework.util.Assert;
+
+/**
+ * The {@link CompositeFileListFilter} extension which chains the result
+ * of the previous filter to the next one. If a filter in the chain returns
+ * an empty list, the remaining filters are not invoked.
+ * @param <F> The type that will be filtered.
+ *
+ * @author Artem Bilan
+ * @author Gary Russell
+ *
+ * @since 4.3.7
+ *
+ */
+public class ChainFileListFilter<F> extends CompositeFileListFilter<F> {
+
+	@Override
+	public List<F> filterFiles(F[] files) {
+		Assert.notNull(files, "'files' should not be null");
+		List<F> leftOver = Arrays.asList(files);
+		for (FileListFilter<F> fileFilter : this.fileFilters) {
+			if (leftOver.size() == 0) {
+				break;
+			}
+			@SuppressWarnings("unchecked")
+			F[] fileArray = (F[]) leftOver.toArray();
+			leftOver = fileFilter.filterFiles(fileArray);
+		}
+		return leftOver;
+	}
+
+}

spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2016 the original author or authors.
+ * Copyright 2002-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,17 +32,18 @@ import org.springframework.util.Assert;
 /**
  * Simple {@link FileListFilter} that predicates its matches against <b>all</b> of the
  * configured {@link FileListFilter}.
+ * @param <F> The type that will be filtered.
  *
  * @author Iwein Fuld
  * @author Josh Long
  * @author Gary Russell
  * @author Artem Bilan
  *
- * @param <F> The type that will be filtered.
+ *
  */
 public class CompositeFileListFilter<F> implements ReversibleFileListFilter<F>, ResettableFileListFilter<F>, Closeable {
 
-	private final Set<FileListFilter<F>> fileFilters;
+	protected final Set<FileListFilter<F>> fileFilters; // NOSONAR
 
 
 	public CompositeFileListFilter() {

spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java

@@ -16,26 +16,33 @@
 
 package org.springframework.integration.file;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Matchers.isA;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 
 import org.junit.Test;
+import org.mockito.ArgumentCaptor;
 
+import org.springframework.integration.file.filters.ChainFileListFilter;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 
 /**
  * @author Iwein Fuld
  * @author Gary Russell
+ * @author Aaron Grant
+ * @author Artem Bilan
  */
 public class CompositeFileListFilterTests {
 
@@ -52,7 +59,7 @@ public class CompositeFileListFilterTests {
 		CompositeFileListFilter<File> compositeFileFilter = new CompositeFileListFilter<File>();
 		compositeFileFilter.addFilter(fileFilterMock1);
 		compositeFileFilter.addFilter(fileFilterMock2);
-		List<File> returnedFiles = Arrays.asList(fileMock);
+		List<File> returnedFiles = Collections.singletonList(fileMock);
 		when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles);
 		when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles);
 		assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock }));
@@ -66,7 +73,7 @@ public class CompositeFileListFilterTests {
 		CompositeFileListFilter<File> compositeFileFilter = new CompositeFileListFilter<File>();
 		compositeFileFilter.addFilter(fileFilterMock1);
 		compositeFileFilter.addFilter(fileFilterMock2);
-		List<File> returnedFiles = Arrays.asList(fileMock);
+		List<File> returnedFiles = Collections.singletonList(fileMock);
 		when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles);
 		when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles);
 		assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock }));
@@ -86,4 +93,22 @@ public class CompositeFileListFilterTests {
 		assertTrue(compositeFileFilter.filterFiles(new File[] { fileMock }).isEmpty());
 		compositeFileFilter.close();
 	}
+
+	@Test
+	public void excludeFromLaterFilters() throws Exception {
+		CompositeFileListFilter<File> compositeFileFilter = new ChainFileListFilter<>();
+		compositeFileFilter.addFilter(this.fileFilterMock1);
+		compositeFileFilter.addFilter(this.fileFilterMock2);
+		List<File> noFiles = new ArrayList<>();
+		when(this.fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(noFiles);
+		assertEquals(noFiles, compositeFileFilter.filterFiles(new File[] { this.fileMock }));
+
+		ArgumentCaptor<File[]> captor = ArgumentCaptor.forClass(File[].class);
+		verify(fileFilterMock1).filterFiles(captor.capture());
+		assertThat(captor.getValue().length, equalTo(1));
+		verify(fileFilterMock2, never()).filterFiles(isA(File[].class));
+
+		compositeFileFilter.close();
+	}
+
 }

src/reference/asciidoc/file.adoc

@@ -105,6 +105,20 @@ to, say, network glitches.
 </bean>
 ----
 
+Starting with _version 4.3.7_ a `ChainFileListFilter` (an extension of `CompositeFileListFilter`) has been introduced to allow scenarios when subsequent filters should only see the result of the previous filter.
+(With the `CompositeFileListFilter`, all filters see all the files, but only files that pass all filters are passed by the `CompositeFileListFilter`).
+An example of where the new behavior is required is a combination of `LastModifiedFileListFilter` and `AcceptOnceFileListFilter`, when we do not wish to accept the file until some amount of time has elapsed.
+With the `CompositeFileListFilter`, since the `AcceptOnceFileListFilter` sees all the files on the first pass, it won't pass it later when the other filter does.
+The `CompositeFileListFilter` approach is useful when a pattern filter is combined with a custom filter that looks for a secondary indicating file transfer is complete.
+The pattern filter might only pass the primary file (e.g. `foo.txt`) but the "done" filter needs to see if, say `foo.done` is present.
+
+Say we have files `a.txt`, `a.done`, and `b.txt`.
+
+The pattern filter only passes `a.txt` and `b.txt`, the "done" filter will see all three files and only pass `a.txt`.
+The final result of the composite filter is only `a.txt` is released.
+
+NOTE: With the `ChainFileListFilter`, if any filter in the chain returns an empty list, the remaining filters are not invoked.
+
 *Directory scanning and polling*
 
 The `FileReadingMessageSource` doesn't produce messages for files from the directory immediately.

src/reference/asciidoc/whats-new.adoc

@@ -169,6 +169,11 @@ This applies when the outbound gateway returns an `InputStream` or the new (S)FT
 Also a new `markers-json` options has been introduced to convert `FileSplitter.FileMarker` to JSON `String` for relaxed downstream network interaction.
 See <<file-splitter>> for more information.
 
+===== File Filters
+
+A new `ChainFileListFilter` is provided as an alternative to `CompositeFileListFilter`.
+See <<file-reading>> for more information.
+
 ==== AMQP Changes
 
 ===== Content Type Message Converter