From 9836828f54def6e3276b3370d8e2dab9f9788020 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Mon, 16 Jan 2017 09:54:45 -0500 Subject: [PATCH] INT-4204: Introduce `ExpressionFileListFilter` JIRA: https://jira.spring.io/browse/INT-4204 Fix error message in the `AbstractRemoteFileOutboundGatewayParser` --- ...RemoteFileInboundChannelAdapterParser.java | 2 +- ...stractRemoteFileOutboundGatewayParser.java | 20 ++++- .../FileInboundChannelAdapterParser.java | 27 +++++-- .../file/config/FileParserUtils.java | 21 ++++- .../dsl/FileInboundChannelAdapterSpec.java | 47 ++++++++++- .../RemoteFileInboundChannelAdapterSpec.java | 47 +++++++++-- .../dsl/RemoteFileOutboundGatewaySpec.java | 79 ++++++++++++++++++- ...ileStreamingInboundChannelAdapterSpec.java | 50 +++++++++++- .../filters/ExpressionFileListFilter.java | 74 +++++++++++++++++ .../config/spring-integration-file-5.0.xsd | 8 ++ ...WithClasspathInPropertiesTests-context.xml | 23 +++--- ...AdapterWithClasspathInPropertiesTests.java | 46 +++++++++-- .../integration/file/dsl/FileTests.java | 4 +- .../ftp/config/spring-integration-ftp-5.0.xsd | 24 ++++++ .../FtpOutboundGatewayParserTests-context.xml | 2 +- .../config/FtpOutboundGatewayParserTests.java | 5 +- .../integration/ftp/dsl/FtpTests.java | 4 +- .../config/spring-integration-sftp-5.0.xsd | 24 ++++++ ...boundChannelAdapterParserTests-context.xml | 2 +- ...amingInboundChannelAdapterParserTests.java | 11 ++- src/reference/asciidoc/file.adoc | 12 ++- src/reference/asciidoc/ftp.adoc | 5 +- src/reference/asciidoc/sftp.adoc | 5 +- 23 files changed, 482 insertions(+), 60 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java index 6f8e7b99e1..836998b2e6 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java index 3fc713a41f..3a1807316b 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileOutboundGatewayParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.xml.AbstractConsumerEndpointParser; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.RegexPatternFileListFilter; import org.springframework.integration.file.filters.SimplePatternFileListFilter; import org.springframework.integration.file.remote.RemoteFileOperations; @@ -92,21 +93,34 @@ public abstract class AbstractRemoteFileOutboundGatewayParser extends AbstractCo protected void configureFilter(BeanDefinitionBuilder builder, Element element, ParserContext parserContext, String filterAttribute, String patternPrefix, String propertyName) { String filter = element.getAttribute(filterAttribute); + String filterExpression = element.getAttribute(filterAttribute + "-expression"); String fileNamePattern = element.getAttribute(patternPrefix + "-pattern"); String fileNameRegex = element.getAttribute(patternPrefix + "-regex"); boolean hasFilter = StringUtils.hasText(filter); + boolean hasFilterExpression = StringUtils.hasText(filterExpression); boolean hasFileNamePattern = StringUtils.hasText(fileNamePattern); boolean hasFileNameRegex = StringUtils.hasText(fileNameRegex); int count = hasFilter ? 1 : 0; + count += hasFilterExpression ? 1 : 0; count += hasFileNamePattern ? 1 : 0; count += hasFileNameRegex ? 1 : 0; if (count > 1) { - parserContext.getReaderContext().error("at most one of '" + patternPrefix + "-pattern', " + - "'" + patternPrefix + "-regex', or '" + filterAttribute + "' is allowed on a remote file outbound gateway", element); + parserContext.getReaderContext() + .error("at most one of '" + patternPrefix + "-pattern', " + + "'" + patternPrefix + "-regex', '" + filterAttribute + + "' or '" + filterAttribute + "-expression' is allowed on a remote file outbound gateway", + element); } else if (hasFilter) { builder.addPropertyReference(propertyName, filter); } + else if (hasFilterExpression) { + BeanDefinition expressionFilterBeanDefinition = + BeanDefinitionBuilder.genericBeanDefinition(ExpressionFileListFilter.class) + .addConstructorArgValue(filterExpression) + .getBeanDefinition(); + builder.addPropertyValue(propertyName, expressionFilterBeanDefinition); + } else if (hasFileNamePattern) { BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition( "filter".equals(filterAttribute) ? diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java index 7c5ffd6053..e3aebf8670 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ import org.springframework.beans.factory.support.BeanDefinitionReaderUtils; import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.locking.NioFileLocker; import org.springframework.util.StringUtils; import org.springframework.util.xml.DomUtils; @@ -87,8 +88,13 @@ public class FileInboundChannelAdapterParser extends AbstractPollingInboundChann String preventDuplicates = element.getAttribute("prevent-duplicates"); String ignoreHidden = element.getAttribute("ignore-hidden"); String filter = element.getAttribute("filter"); - if (!StringUtils.hasText(filter) && !StringUtils.hasText(filenamePattern) && !StringUtils.hasText(filenameRegex) - && !StringUtils.hasText(preventDuplicates) && !StringUtils.hasText(ignoreHidden)) { + String filterExpression = element.getAttribute("filter-expression"); + if (!StringUtils.hasText(filter) + && !StringUtils.hasText(filenamePattern) + && !StringUtils.hasText(filenameRegex) + && !StringUtils.hasText(preventDuplicates) + && !StringUtils.hasText(ignoreHidden) + && !StringUtils.hasText(filterExpression)) { return null; } BeanDefinitionBuilder factoryBeanBuilder = @@ -97,6 +103,17 @@ public class FileInboundChannelAdapterParser extends AbstractPollingInboundChann if (StringUtils.hasText(filter)) { factoryBeanBuilder.addPropertyReference("filter", filter); } + if (StringUtils.hasText(filterExpression)) { + if (StringUtils.hasText(filter)) { + parserContext.getReaderContext() + .error("At most one of 'filter' or 'filter-expression' can be provided.", element); + } + BeanDefinition expressionFilterBeanDefinition = + BeanDefinitionBuilder.genericBeanDefinition(ExpressionFileListFilter.class) + .addConstructorArgValue(filterExpression) + .getBeanDefinition(); + factoryBeanBuilder.addPropertyValue("filter", expressionFilterBeanDefinition); + } if (StringUtils.hasText(filenamePattern)) { if (StringUtils.hasText(filter)) { parserContext.getReaderContext().error( @@ -113,8 +130,8 @@ public class FileInboundChannelAdapterParser extends AbstractPollingInboundChann } IntegrationNamespaceUtils.setValueIfAttributeDefined(factoryBeanBuilder, element, "prevent-duplicates"); IntegrationNamespaceUtils.setValueIfAttributeDefined(factoryBeanBuilder, element, "ignore-hidden"); - return BeanDefinitionReaderUtils.registerWithGeneratedName( - factoryBeanBuilder.getBeanDefinition(), parserContext.getRegistry()); + return BeanDefinitionReaderUtils.registerWithGeneratedName(factoryBeanBuilder.getBeanDefinition(), + parserContext.getRegistry()); } } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java index 4699999bcf..1e622457a7 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2016 the original author or authors. + * Copyright 2013-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,7 @@ import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; import org.springframework.integration.file.DefaultFileNameGenerator; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.RemoteFileOperations; import org.springframework.util.StringUtils; @@ -32,6 +33,7 @@ import org.springframework.util.StringUtils; * @author Mark Fisher * @author David Turanski * @author Gary Russell + * @author Artem Bilan * @since 3.0 * */ @@ -93,12 +95,14 @@ public final class FileParserUtils { static void configureFilter(BeanDefinitionBuilder synchronizerBuilder, Element element, ParserContext parserContext, Class> patternClass, Class> regexClass) { String filter = element.getAttribute("filter"); + String filterExpression = element.getAttribute("filter-expression"); String fileNamePattern = element.getAttribute("filename-pattern"); String fileNameRegex = element.getAttribute("filename-regex"); boolean hasFilter = StringUtils.hasText(filter); + boolean hasFilterExpression = StringUtils.hasText(filterExpression); boolean hasFileNamePattern = StringUtils.hasText(fileNamePattern); boolean hasFileNameRegex = StringUtils.hasText(fileNameRegex); - if (hasFilter || hasFileNamePattern || hasFileNameRegex) { + if (hasFilter || hasFilterExpression || hasFileNamePattern || hasFileNameRegex) { int count = 0; if (hasFilter) { count++; @@ -109,13 +113,24 @@ public final class FileParserUtils { if (hasFileNameRegex) { count++; } + if (hasFilterExpression) { + count++; + } if (count != 1) { parserContext.getReaderContext().error("at most one of 'filename-pattern', " + - "'filename-regex', or 'filter' is allowed on remote file inbound adapter", element); + "'filename-regex', 'filter' or 'filter-expression' is allowed on remote file inbound adapter", + element); } if (hasFilter) { synchronizerBuilder.addPropertyReference("filter", filter); } + else if (hasFilterExpression) { + BeanDefinition expressionFilterBeanDefinition = + BeanDefinitionBuilder.genericBeanDefinition(ExpressionFileListFilter.class) + .addConstructorArgValue(filterExpression) + .getBeanDefinition(); + synchronizerBuilder.addPropertyValue("filter", expressionFilterBeanDefinition); + } else if (hasFileNamePattern) { BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition(patternClass); filterBuilder.addConstructorArgValue(fileNamePattern); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/FileInboundChannelAdapterSpec.java b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/FileInboundChannelAdapterSpec.java index 05a1c8fbf4..3356c171f0 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/FileInboundChannelAdapterSpec.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/FileInboundChannelAdapterSpec.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,16 +17,22 @@ package org.springframework.integration.file.dsl; import java.io.File; +import java.util.Collection; +import java.util.Collections; import java.util.Comparator; +import java.util.function.Function; import org.springframework.beans.factory.BeanCreationException; +import org.springframework.integration.dsl.ComponentsRegistration; import org.springframework.integration.dsl.MessageSourceSpec; +import org.springframework.integration.expression.FunctionExpression; import org.springframework.integration.file.DirectoryScanner; import org.springframework.integration.file.FileLocker; import org.springframework.integration.file.FileReadingMessageSource; import org.springframework.integration.file.config.FileListFilterFactoryBean; import org.springframework.integration.file.filters.AcceptAllFileListFilter; import org.springframework.integration.file.filters.AcceptOnceFileListFilter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.filters.IgnoreHiddenFileListFilter; import org.springframework.integration.file.filters.RegexPatternFileListFilter; @@ -42,12 +48,15 @@ import org.springframework.util.Assert; * @since 5.0 */ public class FileInboundChannelAdapterSpec - extends MessageSourceSpec { + extends MessageSourceSpec + implements ComponentsRegistration { private final FileListFilterFactoryBean fileListFilterFactoryBean = new FileListFilterFactoryBean(); private FileLocker locker; + private ExpressionFileListFilter expressionFileListFilter; + FileInboundChannelAdapterSpec() { this.target = new FileReadingMessageSource(); } @@ -117,6 +126,30 @@ public class FileInboundChannelAdapterSpec return _this(); } + /** + * Configure the {@link ExpressionFileListFilter}. + * @param expression the SpEL expression for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public FileInboundChannelAdapterSpec filterExpression(String expression) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(expression); + return filter(this.expressionFileListFilter); + } + + /** + * Configure the {@link ExpressionFileListFilter}. + * @param filterFunction the {@link Function} for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public FileInboundChannelAdapterSpec filterFunction(Function filterFunction) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(new FunctionExpression<>(filterFunction)); + return filter(this.expressionFileListFilter); + } + /** * Configure an {@link AcceptOnceFileListFilter} if {@code preventDuplicates == true}, * otherwise - {@link AcceptAllFileListFilter}. @@ -226,4 +259,14 @@ public class FileInboundChannelAdapterSpec return this; } + @Override + public Collection getComponentsToRegister() { + if (this.expressionFileListFilter != null) { + return Collections.singleton(this.expressionFileListFilter); + } + else { + return null; + } + } + } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java index b927e49b50..5a8811cfc3 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,15 +17,18 @@ package org.springframework.integration.file.dsl; import java.io.File; +import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; +import java.util.List; import java.util.function.Function; import org.springframework.expression.Expression; import org.springframework.integration.dsl.ComponentsRegistration; import org.springframework.integration.dsl.MessageSourceSpec; import org.springframework.integration.expression.FunctionExpression; +import org.springframework.integration.file.FileReadingMessageSource; import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer; import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizingMessageSource; @@ -43,12 +46,15 @@ import org.springframework.integration.file.remote.synchronizer.AbstractInboundF */ public abstract class RemoteFileInboundChannelAdapterSpec, MS extends AbstractInboundFileSynchronizingMessageSource> - extends MessageSourceSpec implements ComponentsRegistration { + extends MessageSourceSpec + implements ComponentsRegistration { protected final AbstractInboundFileSynchronizer synchronizer; private CompositeFileListFilter filter; + private ExpressionFileListFilter expressionFileListFilter; + protected RemoteFileInboundChannelAdapterSpec(AbstractInboundFileSynchronizer synchronizer) { this.synchronizer = synchronizer; } @@ -171,7 +177,7 @@ public abstract class RemoteFileInboundChannelAdapterSpec) filter; } else { - this.filter = new CompositeFileListFilter(); + this.filter = new CompositeFileListFilter<>(); this.filter.addFilter(filter); } this.synchronizer.setFilter(this.filter); @@ -182,6 +188,30 @@ public abstract class RemoteFileInboundChannelAdapterSpec(expression); + return filter(this.expressionFileListFilter); + } + + /** + * Configure the {@link ExpressionFileListFilter}. + * @param filterFunction the {@link Function} for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public S filterFunction(Function filterFunction) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(new FunctionExpression<>(filterFunction)); + return filter(this.expressionFileListFilter); + } + /** * Configure a simple pattern filter (e.g. '*.txt'). * @param pattern the pattern. @@ -220,7 +250,14 @@ public abstract class RemoteFileInboundChannelAdapterSpec getComponentsToRegister() { - return Collections.singletonList(this.synchronizer); + List componentsToRegister = new ArrayList<>(); + componentsToRegister.add(this.synchronizer); + + if (this.expressionFileListFilter != null) { + componentsToRegister.add(this.expressionFileListFilter); + } + + return componentsToRegister; } } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileOutboundGatewaySpec.java b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileOutboundGatewaySpec.java index 8dc96c7607..7e62c011b8 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileOutboundGatewaySpec.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileOutboundGatewaySpec.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,12 +17,18 @@ package org.springframework.integration.file.dsl; import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.function.Function; import org.springframework.expression.Expression; +import org.springframework.integration.dsl.ComponentsRegistration; import org.springframework.integration.dsl.MessageHandlerSpec; import org.springframework.integration.expression.FunctionExpression; +import org.springframework.integration.file.FileReadingMessageSource; import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.filters.RegexPatternFileListFilter; import org.springframework.integration.file.filters.SimplePatternFileListFilter; @@ -40,10 +46,15 @@ import org.springframework.messaging.Message; * @since 5.0 */ public abstract class RemoteFileOutboundGatewaySpec> - extends MessageHandlerSpec> { + extends MessageHandlerSpec> + implements ComponentsRegistration { private CompositeFileListFilter filter; + private ExpressionFileListFilter expressionFileListFilter; + + private ExpressionFileListFilter mputExpressionFileListFilter; + private CompositeFileListFilter mputFilter; protected RemoteFileOutboundGatewaySpec(AbstractRemoteFileOutboundGateway outboundGateway) { @@ -166,6 +177,30 @@ public abstract class RemoteFileOutboundGatewaySpec(expression); + return filter(this.expressionFileListFilter); + } + + /** + * Configure the {@link ExpressionFileListFilter}. + * @param filterFunction the {@link Function} for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public S filterFunction(Function filterFunction) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(new FunctionExpression<>(filterFunction)); + return filter(this.expressionFileListFilter); + } + /** * A {@link FileListFilter} that runs against the local file system view when * using {@code MPUT} command. @@ -205,10 +240,34 @@ public abstract class RemoteFileOutboundGatewaySpec(expression); + return mputFilter(this.mputExpressionFileListFilter); + } + + /** + * Configure the {@link ExpressionFileListFilter}. + * @param filterFunction the {@link Function} for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public S mputFilterFunction(Function filterFunction) { + this.mputExpressionFileListFilter = new ExpressionFileListFilter<>(new FunctionExpression<>(filterFunction)); + return mputFilter(this.mputExpressionFileListFilter); + } + /** * Specify a SpEL expression for files renaming during transfer. * @param expression the String in SpEL syntax. @@ -280,6 +339,20 @@ public abstract class RemoteFileOutboundGatewaySpec getComponentsToRegister() { + List componentsToRegister = new ArrayList<>(); + if (this.expressionFileListFilter != null) { + componentsToRegister.add(this.expressionFileListFilter); + } + if (this.mputExpressionFileListFilter != null) { + componentsToRegister.add(this.expressionFileListFilter); + } + + return componentsToRegister; + } + + /** * Specify a simple pattern to match remote files (e.g. '*.txt'). * @param pattern the pattern. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java index 7494ef654d..be120550f3 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,17 @@ package org.springframework.integration.file.dsl; +import java.util.Collection; +import java.util.Collections; import java.util.function.Function; import org.springframework.expression.Expression; +import org.springframework.integration.dsl.ComponentsRegistration; import org.springframework.integration.dsl.MessageSourceSpec; import org.springframework.integration.expression.FunctionExpression; +import org.springframework.integration.file.FileReadingMessageSource; import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource; import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizingMessageSource; @@ -38,13 +43,16 @@ import org.springframework.messaging.Message; * * @since 5.0 */ -public abstract class - RemoteFileStreamingInboundChannelAdapterSpec, +public abstract class RemoteFileStreamingInboundChannelAdapterSpec, MS extends AbstractRemoteFileStreamingMessageSource> - extends MessageSourceSpec { + extends MessageSourceSpec + implements ComponentsRegistration { private CompositeFileListFilter filter; + private ExpressionFileListFilter expressionFileListFilter; + /** * Configure the file name path separator used by the remote system. Defaults to '/'. * @param remoteFileSeparator the remoteFileSeparator. @@ -109,6 +117,40 @@ public abstract class return _this(); } + /** + * Configure the {@link ExpressionFileListFilter}. + * @param expression the SpEL expression for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public S filterExpression(String expression) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(expression); + return filter(this.expressionFileListFilter); + } + + /** + * Configure the {@link ExpressionFileListFilter}. + * @param filterFunction the {@link Function} for files filtering. + * @return the spec. + * @see FileReadingMessageSource#setFilter(FileListFilter) + * @see ExpressionFileListFilter + */ + public S filterFunction(Function filterFunction) { + this.expressionFileListFilter = new ExpressionFileListFilter<>(new FunctionExpression<>(filterFunction)); + return filter(this.expressionFileListFilter); + } + + @Override + public Collection getComponentsToRegister() { + if (this.expressionFileListFilter != null) { + return Collections.singleton(this.expressionFileListFilter); + } + else { + return null; + } + } + /** * Configure a simple pattern filter (e.g. '*.txt'). * @param pattern the pattern. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java new file mode 100644 index 0000000000..a874add028 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java @@ -0,0 +1,74 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.BeanFactoryAware; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.expression.ExpressionUtils; +import org.springframework.util.Assert; + +/** + * A SpEL expression based {@link AbstractFileListFilter} implementation. + * + * @author Artem Bilan + * + * @since 5.0 + */ +public class ExpressionFileListFilter extends AbstractFileListFilter + implements BeanFactoryAware { + + private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); + + private final Expression expression; + + private BeanFactory beanFactory; + + private EvaluationContext evaluationContext; + + public ExpressionFileListFilter(String expression) { + this(EXPRESSION_PARSER.parseExpression(expression)); + Assert.hasText(expression, "'expression' must not be empty"); + } + + public ExpressionFileListFilter(Expression expression) { + Assert.notNull(expression, "'expression' must not be null"); + this.expression = expression; + } + + @Override + public void setBeanFactory(BeanFactory beanFactory) throws BeansException { + this.beanFactory = beanFactory; + } + + @Override + protected boolean accept(F file) { + return this.expression.getValue(getEvaluationContext(), file, Boolean.class); + } + + private EvaluationContext getEvaluationContext() { + if (this.evaluationContext == null) { + this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(this.beanFactory); + } + return this.evaluationContext; + } + +} diff --git a/spring-integration-file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-5.0.xsd b/spring-integration-file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-5.0.xsd index 777f183660..963fcc4f59 100644 --- a/spring-integration-file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-5.0.xsd +++ b/spring-integration-file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-5.0.xsd @@ -76,6 +76,14 @@ + + + + + + directory="${inputdir}" + filter-expression="true" + auto-startup="false"> - - + + diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/config/FileInboundChannelAdapterWithClasspathInPropertiesTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/config/FileInboundChannelAdapterWithClasspathInPropertiesTests.java index d94965ca5d..54267ea0d1 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/config/FileInboundChannelAdapterWithClasspathInPropertiesTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/config/FileInboundChannelAdapterWithClasspathInPropertiesTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,45 @@ package org.springframework.integration.file.config; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.util.Iterator; +import java.util.Set; + import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; + import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.BeanFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.integration.file.FileReadingMessageSource; +import org.springframework.integration.file.filters.CompositeFileListFilter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; +import org.springframework.integration.test.util.TestUtils; import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import java.io.File; - -import static org.junit.Assert.assertEquals; +import org.springframework.test.context.junit4.SpringRunner; /** * @author Iwein Fuld + * @author Artem Bilan */ @ContextConfiguration -@RunWith(SpringJUnit4ClassRunner.class) +@RunWith(SpringRunner.class) public class FileInboundChannelAdapterWithClasspathInPropertiesTests { - @Autowired(required = true) + @Autowired private FileReadingMessageSource source; + @Autowired + private BeanFactory beanFactory; + private DirectFieldAccessor accessor; @Before @@ -48,10 +63,25 @@ public class FileInboundChannelAdapterWithClasspathInPropertiesTests { } @Test + @SuppressWarnings("unchecked") public void inputDirectory() throws Exception { File expected = new ClassPathResource("").getFile(); File actual = (File) accessor.getPropertyValue("directory"); assertEquals("'directory' should be set", expected, actual); + + FileListFilter fileListFilter = + TestUtils.getPropertyValue(this.source, "scanner.filter", FileListFilter.class); + assertThat(fileListFilter, instanceOf(CompositeFileListFilter.class)); + Set> fileFilters = + TestUtils.getPropertyValue(fileListFilter, "fileFilters", Set.class); + assertEquals(2, fileFilters.size()); + Iterator> iterator = fileFilters.iterator(); + iterator.next(); + FileListFilter expressionFilter = iterator.next(); + assertThat(expressionFilter, instanceOf(ExpressionFileListFilter.class)); + assertEquals("true", + TestUtils.getPropertyValue(expressionFilter, "expression.expression", String.class)); + assertSame(this.beanFactory, TestUtils.getPropertyValue(expressionFilter, "beanFactory")); } } diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/dsl/FileTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/dsl/FileTests.java index 32440979ee..f64b8c8f42 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/dsl/FileTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/dsl/FileTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -391,7 +391,7 @@ public class FileTests { public IntegrationFlow fileSplitterFlow() { return IntegrationFlows .from(Files.inboundAdapter(tmpDir.getRoot()) - .patternFilter("foo.tmp"), + .filterFunction(f -> "foo.tmp".equals(f.getName())), e -> e.poller(p -> p.fixedDelay(100))) .split(Files.splitter() .markers() diff --git a/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-5.0.xsd b/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-5.0.xsd index d7f45240f8..bbb61dbcb7 100644 --- a/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-5.0.xsd +++ b/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-5.0.xsd @@ -337,6 +337,14 @@ + + + + + @@ -376,6 +384,14 @@ + + + + + @@ -546,6 +562,14 @@ + + + + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests-context.xml index d3ea7b97d5..9ffbe7cd0d 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests-context.xml @@ -34,7 +34,7 @@ expression="payload" order="1" mode="APPEND" - mput-regex=".*"> + mput-filter-expression="name matches '.*'"> diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java index 3087cc5a4e..efee4f96cd 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.expression.Expression; import org.springframework.integration.endpoint.AbstractEndpoint; import org.springframework.integration.file.FileNameGenerator; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.filters.RegexPatternFileListFilter; import org.springframework.integration.file.filters.SimplePatternFileListFilter; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway.Command; @@ -108,7 +109,7 @@ public class FtpOutboundGatewayParserTests { assertEquals(Long.valueOf(777), sendTimeout); assertTrue(TestUtils.getPropertyValue(gateway, "requiresReply", Boolean.class)); assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), - Matchers.instanceOf(RegexPatternFileListFilter.class)); + Matchers.instanceOf(ExpressionFileListFilter.class)); assertEquals(FileExistsMode.APPEND, TestUtils.getPropertyValue(gateway, "fileExistsMode")); } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/dsl/FtpTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/dsl/FtpTests.java index c61c4160a9..e44fcd0109 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/dsl/FtpTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/dsl/FtpTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2016 the original author or authors. + * Copyright 2014-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -153,7 +153,7 @@ public class FtpTests extends FtpTestSupport { .handle(Ftp.outboundGateway(sessionFactory(), AbstractRemoteFileOutboundGateway.Command.MGET, "payload") .options(AbstractRemoteFileOutboundGateway.Option.RECURSIVE) - .regexFileNameFilter("(subFtpSource|.*1.txt)") + .filterExpression("name matches 'subFtpSource|.*1.txt'") .localDirectoryExpression("'" + getTargetLocalDirectoryName() + "' + #remoteDirectory") .localFilenameExpression("#remoteFileName.replaceFirst('ftpSource', 'localTarget')")) .channel(out); diff --git a/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-5.0.xsd b/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-5.0.xsd index d02ec6cc53..f975d10090 100644 --- a/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-5.0.xsd +++ b/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-5.0.xsd @@ -339,6 +339,14 @@ + + + + + @@ -378,6 +386,14 @@ + + + + + @@ -547,6 +563,14 @@ + + + + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests-context.xml index 91d10c657f..8023cb22a2 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests-context.xml @@ -40,7 +40,7 @@ session-factory="csf" auto-startup="false" phase="23" - filename-pattern="*.txt" + filter-expression="new org.springframework.util.AntPathMatcher().match('*.txt', filename)" remote-file-separator="X" remote-directory-expression="'foo/bar'"> diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests.java index aedacf743a..48cc6dec0a 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/SftpStreamingInboundChannelAdapterParserTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ import org.junit.runner.RunWith; import org.springframework.beans.factory.FactoryBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.endpoint.SourcePollingChannelAdapter; +import org.springframework.integration.file.filters.ExpressionFileListFilter; import org.springframework.integration.file.remote.session.CachingSessionFactory; import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter; import org.springframework.integration.sftp.inbound.SftpStreamingMessageSource; @@ -45,6 +46,7 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; /** * @author Gary Russell + * @author Artem Bilan */ @ContextConfiguration @RunWith(SpringJUnit4ClassRunner.class) @@ -54,6 +56,9 @@ public class SftpStreamingInboundChannelAdapterParserTests { @Autowired private SourcePollingChannelAdapter sftpInbound; + @Autowired + private SourcePollingChannelAdapter contextLoadsWithNoComparator; + @Autowired private MessageChannel sftpChannel; @@ -74,6 +79,10 @@ public class SftpStreamingInboundChannelAdapterParserTests { assertThat(TestUtils.getPropertyValue(source, "filter"), instanceOf(SftpSimplePatternFileListFilter.class)); assertSame(this.csf, TestUtils.getPropertyValue(source, "remoteFileTemplate.sessionFactory")); assertEquals(31, TestUtils.getPropertyValue(source, "maxFetchSize")); + + source = TestUtils.getPropertyValue(this.contextLoadsWithNoComparator, "source", + SftpStreamingMessageSource.class); + assertThat(TestUtils.getPropertyValue(source, "filter"), instanceOf(ExpressionFileListFilter.class)); } public static class TestSessionFactoryBean implements FactoryBean { diff --git a/src/reference/asciidoc/file.adoc b/src/reference/asciidoc/file.adoc index 22d9f4f3e9..3a97d78f51 100644 --- a/src/reference/asciidoc/file.adoc +++ b/src/reference/asciidoc/file.adoc @@ -69,7 +69,7 @@ metadata store on every update (if the store implements `Flushable`). A common problem with reading files is that a file may be detected before it is ready. The default `AcceptOnceFileListFilter` does not prevent this. In most cases, this can be prevented if the file-writing process renames each file as soon as it is ready for reading. -A filename-pattern or filename-regex filter that accepts only files that are ready (e.g. +A `filename-pattern` or `filename-regex` filter that accepts only files that are ready (e.g. based on a known suffix), composed with the default `AcceptOnceFileListFilter` allows for this. The `CompositeFileListFilter` enables the composition. [source,xml] @@ -106,6 +106,16 @@ to, say, network glitches. ---- +Starting with _version 5.0_ an `ExpressionFileListFilter` has been introduced to allow to execute SpEL expression against file as a context evaluation root object. +For this purpose all the XML components for file handling (local and remote), alongside with an existing `filter` attribute, have been supplied with the `filter-expression` option: +[source, xml] +---- + +---- + *Message Headers* Starting with _version 5.0_ the `FileReadingMessageSource`, in addition to the `payload` as a polled `File`, populates these headers to the outbound `Message`: diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 9d929cb9ef..fa7646f2c6 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -516,6 +516,7 @@ See <> and <> for more information about thes filename-pattern="*.txt" filename-regex=".*\.txt" filter="filter" + filter-expression="@myFilterBean.check(#root)" remote-file-separator="/" comparator="comparator" max-fetch-size="1" @@ -524,7 +525,7 @@ See <> and <> for more information about thes ---- -Only one of `filename-pattern`, `filename-regex` or `filter` is allowed. +Only one of `filename-pattern`, `filename-regex`, `filter` or `filter-expression` is allowed. IMPORTANT: Unlike the non-streaming inbound channel adapter, this adapter does not prevent duplicates by default. If you do not delete the remote file (e.g. using an outbound gateway with an rm command) and you wish to prevent the @@ -911,7 +912,7 @@ _mput_ sends multiple files to the server and supports the following option: The message payload must be a `java.io.File` representing a local directory. The same attributes as the `put` command are supported. -In addition, files in the local directory can be filtered with one of `mput-pattern`, `mput-regex` or `mput-filter`. +In addition, files in the local directory can be filtered with one of `mput-pattern`, `mput-regex`, `mput-filter` or `mput-filter-expression`. The filter works with recursion, as long as the subdirectories themselves pass the filter. Subdirectories that do not pass the filter are not recursed. diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index 08e27ab6e4..cee2d96fd4 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -555,6 +555,7 @@ See <> and <> for more information about thes filename-pattern="*.txt" filename-regex=".*\.txt" filter="filter" + filter-expression="@myFilterBean.check(#root)" remote-file-separator="/" comparator="comparator" max-fetch-size="1" @@ -563,7 +564,7 @@ See <> and <> for more information about thes ---- -Only one of `filename-pattern`, `filename-regex` or `filter` is allowed. +Only one of `filename-pattern`, `filename-regex`, `filter` or `filter-expression` is allowed. IMPORTANT: Unlike the non-streaming inbound channel adapter, this adapter does not prevent duplicates by default. If you do not delete the remote file (e.g. using an outbound gateway with an rm command) and you wish to prevent the @@ -935,7 +936,7 @@ _mput_ sends multiple files to the server and supports the following option: The message payload must be a `java.io.File` representing a local directory. The same attributes as the `put` command are supported. -In addition, files in the local directory can be filtered with one of `mput-pattern`, `mput-regex` or `mput-filter`. +In addition, files in the local directory can be filtered with one of `mput-pattern`, `mput-regex`, `mput-filter` or `mput-filter-expression`. The filter works with recursion, as long as the subdirectories themselves pass the filter. Subdirectories that do not pass the filter are not recursed.