From 35a633d3049ec1667069efea9671686677f951b9 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Wed, 25 Jan 2017 13:00:19 -0500 Subject: [PATCH] INT-4215: Add `ChainFileListFilter` JIRA: https://jira.spring.io/browse/INT-4215 Fixes GH-1998 (https://github.com/spring-projects/spring-integration/issues/1998) Just make an `CompositeFileListFilter` extension which chains result of the previous filter to the next **Cherry-pick 4.3.x** --- build.gradle | 2 +- .../file/filters/ChainFileListFilter.java | 53 +++++++++++++++++++ .../file/filters/CompositeFileListFilter.java | 7 +-- .../file/CompositeFileListFilterTests.java | 31 +++++++++-- src/reference/asciidoc/file.adoc | 14 +++++ src/reference/asciidoc/whats-new.adoc | 5 ++ 6 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java diff --git a/build.gradle b/build.gradle index 725640cbcb..1b0ce20a31 100644 --- a/build.gradle +++ b/build.gradle @@ -135,7 +135,7 @@ subprojects { subproject -> springSecurityVersion = '4.1.0.RELEASE' springSocialTwitterVersion = '1.1.2.RELEASE' springRetryVersion = '1.1.3.RELEASE' - springVersion = project.hasProperty('springVersion') ? project.springVersion : '4.3.6.BUILD-SNAPSHOT' + springVersion = project.hasProperty('springVersion') ? project.springVersion : '4.3.6.RELEASE' springWsVersion = '2.3.0.RELEASE' xmlUnitVersion = '1.6' xstreamVersion = '1.4.7' diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java new file mode 100644 index 0000000000..aba8233529 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ChainFileListFilter.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import java.util.Arrays; +import java.util.List; + +import org.springframework.util.Assert; + +/** + * The {@link CompositeFileListFilter} extension which chains the result + * of the previous filter to the next one. If a filter in the chain returns + * an empty list, the remaining filters are not invoked. + * @param The type that will be filtered. + * + * @author Artem Bilan + * @author Gary Russell + * + * @since 4.3.7 + * + */ +public class ChainFileListFilter extends CompositeFileListFilter { + + @Override + public List filterFiles(F[] files) { + Assert.notNull(files, "'files' should not be null"); + List leftOver = Arrays.asList(files); + for (FileListFilter fileFilter : this.fileFilters) { + if (leftOver.size() == 0) { + break; + } + @SuppressWarnings("unchecked") + F[] fileArray = (F[]) leftOver.toArray(); + leftOver = fileFilter.filterFiles(fileArray); + } + return leftOver; + } + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java index ba737ce3a0..2af226e57a 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,17 +32,18 @@ import org.springframework.util.Assert; /** * Simple {@link FileListFilter} that predicates its matches against all of the * configured {@link FileListFilter}. + * @param The type that will be filtered. * * @author Iwein Fuld * @author Josh Long * @author Gary Russell * @author Artem Bilan * - * @param The type that will be filtered. + * */ public class CompositeFileListFilter implements ReversibleFileListFilter, ResettableFileListFilter, Closeable { - private final Set> fileFilters; + protected final Set> fileFilters; // NOSONAR public CompositeFileListFilter() { diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java index cd9ded3030..34e09cc510 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/CompositeFileListFilterTests.java @@ -16,26 +16,33 @@ package org.springframework.integration.file; +import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.isA; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.File; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.integration.file.filters.ChainFileListFilter; import org.springframework.integration.file.filters.CompositeFileListFilter; import org.springframework.integration.file.filters.FileListFilter; /** * @author Iwein Fuld * @author Gary Russell + * @author Aaron Grant + * @author Artem Bilan */ public class CompositeFileListFilterTests { @@ -52,7 +59,7 @@ public class CompositeFileListFilterTests { CompositeFileListFilter compositeFileFilter = new CompositeFileListFilter(); compositeFileFilter.addFilter(fileFilterMock1); compositeFileFilter.addFilter(fileFilterMock2); - List returnedFiles = Arrays.asList(fileMock); + List returnedFiles = Collections.singletonList(fileMock); when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles); when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles); assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock })); @@ -66,7 +73,7 @@ public class CompositeFileListFilterTests { CompositeFileListFilter compositeFileFilter = new CompositeFileListFilter(); compositeFileFilter.addFilter(fileFilterMock1); compositeFileFilter.addFilter(fileFilterMock2); - List returnedFiles = Arrays.asList(fileMock); + List returnedFiles = Collections.singletonList(fileMock); when(fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(returnedFiles); when(fileFilterMock2.filterFiles(isA(File[].class))).thenReturn(returnedFiles); assertEquals(returnedFiles, compositeFileFilter.filterFiles(new File[] { fileMock })); @@ -86,4 +93,22 @@ public class CompositeFileListFilterTests { assertTrue(compositeFileFilter.filterFiles(new File[] { fileMock }).isEmpty()); compositeFileFilter.close(); } + + @Test + public void excludeFromLaterFilters() throws Exception { + CompositeFileListFilter compositeFileFilter = new ChainFileListFilter<>(); + compositeFileFilter.addFilter(this.fileFilterMock1); + compositeFileFilter.addFilter(this.fileFilterMock2); + List noFiles = new ArrayList<>(); + when(this.fileFilterMock1.filterFiles(isA(File[].class))).thenReturn(noFiles); + assertEquals(noFiles, compositeFileFilter.filterFiles(new File[] { this.fileMock })); + + ArgumentCaptor captor = ArgumentCaptor.forClass(File[].class); + verify(fileFilterMock1).filterFiles(captor.capture()); + assertThat(captor.getValue().length, equalTo(1)); + verify(fileFilterMock2, never()).filterFiles(isA(File[].class)); + + compositeFileFilter.close(); + } + } diff --git a/src/reference/asciidoc/file.adoc b/src/reference/asciidoc/file.adoc index bf8ba31fb1..c7beba87e1 100644 --- a/src/reference/asciidoc/file.adoc +++ b/src/reference/asciidoc/file.adoc @@ -105,6 +105,20 @@ to, say, network glitches. ---- +Starting with _version 4.3.7_ a `ChainFileListFilter` (an extension of `CompositeFileListFilter`) has been introduced to allow scenarios when subsequent filters should only see the result of the previous filter. +(With the `CompositeFileListFilter`, all filters see all the files, but only files that pass all filters are passed by the `CompositeFileListFilter`). +An example of where the new behavior is required is a combination of `LastModifiedFileListFilter` and `AcceptOnceFileListFilter`, when we do not wish to accept the file until some amount of time has elapsed. +With the `CompositeFileListFilter`, since the `AcceptOnceFileListFilter` sees all the files on the first pass, it won't pass it later when the other filter does. +The `CompositeFileListFilter` approach is useful when a pattern filter is combined with a custom filter that looks for a secondary indicating file transfer is complete. +The pattern filter might only pass the primary file (e.g. `foo.txt`) but the "done" filter needs to see if, say `foo.done` is present. + +Say we have files `a.txt`, `a.done`, and `b.txt`. + +The pattern filter only passes `a.txt` and `b.txt`, the "done" filter will see all three files and only pass `a.txt`. +The final result of the composite filter is only `a.txt` is released. + +NOTE: With the `ChainFileListFilter`, if any filter in the chain returns an empty list, the remaining filters are not invoked. + *Directory scanning and polling* The `FileReadingMessageSource` doesn't produce messages for files from the directory immediately. diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 72c376537d..21997671e4 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -169,6 +169,11 @@ This applies when the outbound gateway returns an `InputStream` or the new (S)FT Also a new `markers-json` options has been introduced to convert `FileSplitter.FileMarker` to JSON `String` for relaxed downstream network interaction. See <> for more information. +===== File Filters + +A new `ChainFileListFilter` is provided as an alternative to `CompositeFileListFilter`. +See <> for more information. + ==== AMQP Changes ===== Content Type Message Converter