From d47bca48cc50dfa79735839d6ea51a31840cf977 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Wed, 25 Jan 2017 13:00:19 -0500 Subject: [PATCH] INT-4215: Add `ChainFileListFilter` JIRA: https://jira.spring.io/browse/INT-4215 Fixes GH-1998 (https://github.com/spring-projects/spring-integration/issues/1998) Just make an `CompositeFileListFilter` extension which chains result of the previous filter to the next **Cherry-pick 4.3.x** --- .../file/filters/ChainFileListFilter.java | 48 +++++++++++++++++++ .../file/filters/CompositeFileListFilter.java | 7 +-- .../file/CompositeFileListFilterTests.java | 27 +++++++++-- src/reference/asciidoc/changes-4.2-4.3.adoc | 6 +++ src/reference/asciidoc/file.adoc | 13 +++++ 5 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java new file mode 100644 index 0000000000..cc9c968aa3 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java @@ -0,0 +1,48 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import java.util.Arrays; +import java.util.List; + +import org.springframework.util.Assert; + +/** + * The {@link CompositeFileListFilter} extension which chains the result + * of the previous filter to the next one. + * + * @author Artem Bilan + * + * @since 4.3.7 + * + * @param The type that will be filtered. + */ +public class ChainFileListFilter extends CompositeFileListFilter { + + @Override + public List filterFiles(F[] files) { + Assert.notNull(files, "'files' should not be null"); + List leftOver = Arrays.asList(files); + for (FileListFilter fileFilter : this.fileFilters) { + @SuppressWarnings("unchecked") + F[] fileArray = (F[]) leftOver.toArray(); + leftOver = fileFilter.filterFiles(fileArray); + } + return leftOver; + } + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java index ba737ce3a0..2af226e57a 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,17 +32,18 @@ import org.springframework.util.Assert; /** * Simple {@link FileListFilter} that predicates its matches against all of the * configured {@link FileListFilter}. + * @param The type that will be filtered. * * @author Iwein Fuld * @author Josh Long * @author Gary Russell * @author Artem Bilan * - * @param The type that will be filtered. + * */ public class CompositeFileListFilter implements ReversibleFileListFilter, ResettableFileListFilter, Closeable { - private final Set> fileFilters; + protected final Set> fileFilters; // NOSONAR public CompositeFileListFilter() { diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java index da3857d21a..ad11db8e33 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java @@ -16,26 +16,31 @@ package org.springframework.integration.file; +import static org.hamcrest.Matchers.arrayWithSize; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.isA; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.hamcrest.MockitoHamcrest.argThat; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.Test; +import org.springframework.integration.file.filters.ChainFileListFilter; import org.springframework.integration.file.filters.CompositeFileListFilter; import org.springframework.integration.file.filters.FileListFilter; /** * @author Iwein Fuld * @author Gary Russell + * @author Aaron Grant + * @author Artem Bilan */ public class CompositeFileListFilterTests { @@ -52,7 +57,7 @@ public class CompositeFileListFilterTests { CompositeFileListFilter compositeFileFilter = new CompositeFileListFilter(); compositeFileFilter.addFilter(fileFilterMock1); compositeFileFilter.addFilter(fileFilterMock2); - List returnedFiles = Arrays.asList(fileMock); + List returnedFiles = Collections.singletonList(fileMock); when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles); when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles); assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock })); @@ -66,7 +71,7 @@ public class CompositeFileListFilterTests { CompositeFileListFilter compositeFileFilter = new CompositeFileListFilter(); compositeFileFilter.addFilter(fileFilterMock1); compositeFileFilter.addFilter(fileFilterMock2); - List returnedFiles = Arrays.asList(fileMock); + List returnedFiles = Collections.singletonList(fileMock); when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles); when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles); assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock })); @@ -86,4 +91,20 @@ public class CompositeFileListFilterTests { assertTrue(compositeFileFilter.filterFiles(new File[] { fileMock }).isEmpty()); compositeFileFilter.close(); } + + @Test + public void excludeFromLaterFilters() throws Exception { + CompositeFileListFilter compositeFileFilter = new ChainFileListFilter<>(); + compositeFileFilter.addFilter(this.fileFilterMock1); + compositeFileFilter.addFilter(this.fileFilterMock2); + List noFiles = new ArrayList<>(); + when(this.fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(noFiles); + assertEquals(noFiles, compositeFileFilter.filterFiles(new File[] { this.fileMock })); + + verify(fileFilterMock1).filterFiles(argThat(arrayWithSize(1))); + verify(fileFilterMock2).filterFiles(argThat(arrayWithSize(0))); + + compositeFileFilter.close(); + } + } diff --git a/src/reference/asciidoc/changes-4.2-4.3.adoc b/src/reference/asciidoc/changes-4.2-4.3.adoc index 392f9fcd08..8f058da274 100644 --- a/src/reference/asciidoc/changes-4.2-4.3.adoc +++ b/src/reference/asciidoc/changes-4.2-4.3.adoc @@ -168,6 +168,12 @@ This applies when the outbound gateway returns an `InputStream` or the new (S)FT Also a new `markers-json` options has been introduced to convert `FileSplitter.FileMarker` to JSON `String` for relaxed downstream network interaction. See <> for more information. +===== File Filters + +A new `ChainFileListFilter` is provided as an alternative to `CompositeFileListFilter`. +See <> for more information. + + ==== AMQP Changes ===== Content Type Message Converter diff --git a/src/reference/asciidoc/file.adoc b/src/reference/asciidoc/file.adoc index 52287238ba..8f916896ed 100644 --- a/src/reference/asciidoc/file.adoc +++ b/src/reference/asciidoc/file.adoc @@ -106,6 +106,19 @@ to, say, network glitches. ---- +Starting with _version 4.3.7_ a `ChainFileListFilter` (an extension of `CompositeFileListFilter`) has been introduced to allow scenarios when subsequent filters should only see the result of the previous filter. +(With the `CompositeFileListFilter`, all filters see all the files, but only files that pass all filters are passed by the `CompositeFileListFilter`). +An example of where the new behavior is required is a combination of `LastModifiedFileListFilter` and `AcceptOnceFileListFilter`, when we do not wish to accept the file until some amount of time has elapsed. +With the `CompositeFileListFilter`, since the `AcceptOnceFileListFilter` sees all the files on the first pass, it won't pass it later when the other filter does. +The `CompositeFileListFilter` approach is useful when a pattern filter is combined with a custom filter that looks for a secondary indicating file transfer is complete. +The pattern filter might only pass the primary file (e.g. `foo.txt`) but the "done" filter needs to see if, say `foo.done` is present. + +Say we have files `a.txt`, `a.done`, and `b.txt`. + +The pattern filter only passes `a.txt` and `b.txt`, the "done" filter will see all three files and only pass `a.txt`. +The final result of the composite filter is only `a.txt` is released. + + Starting with _version 5.0_ an `ExpressionFileListFilter` has been introduced to allow to execute SpEL expression against file as a context evaluation root object. For this purpose all the XML components for file handling (local and remote), alongside with an existing `filter` attribute, have been supplied with the `filter-expression` option: [source, xml]