INT-3718: Add ignore-hidden to int-file:i-c-a

JIRA: https://jira.spring.io/browse/INT-3718

* If not set to `false` explicitly, add `IgnoreHiddenFileListFilter` as a default in `FileListFilterFactoryBean`
* Update XML Schema XSD and add `ignore-hidden` attribute
* Update + Add tests
* Add documentation
* Add default `IgnoreHiddenFileListFilter` also to `DefaultDirectoryScanner`
This commit is contained in:
Gunnar Hillert
2015-05-21 16:43:29 -04:00
committed by Artem Bilan
parent 5b0b3fc159
commit a0160e50d4
11 changed files with 191 additions and 88 deletions

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2013 the original author or authors.
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,27 +17,30 @@
package org.springframework.integration.file;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.springframework.messaging.MessagingException;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.filters.IgnoreHiddenFileListFilter;
/**
* Default directory scanner and base class for other directory scanners.
* Manages the default interrelations between filtering, scanning and locking.
*
* @author Iwein Fuld
* @author Gunnar Hillert
* @since 2.0
*/
public class DefaultDirectoryScanner implements DirectoryScanner {
private volatile FileListFilter<File> filter = new AcceptOnceFileListFilter<File>();
private volatile FileListFilter<File> filter;
private volatile FileLocker locker;
public void setFilter(FileListFilter<File> filter) {
this.filter = filter;
}
@@ -49,6 +52,20 @@ public class DefaultDirectoryScanner implements DirectoryScanner {
this.locker = locker;
}
/**
* Initializes {@link DefaultDirectoryScanner#filter} with a default list of
* {@link FileListFilter}s using a {@link CompositeFileListFilter}:
* <ul>
* <li>{@link IgnoreHiddenFileListFilter}</li>
* <li>{@link AcceptOnceFileListFilter}</li>
* </ul>
*/
public DefaultDirectoryScanner() {
final List<FileListFilter<File>> defaultFilters = new ArrayList<FileListFilter<File>>(2);
defaultFilters.add(new IgnoreHiddenFileListFilter());
defaultFilters.add(new AcceptOnceFileListFilter<File>());
this.filter = new CompositeFileListFilter<File>(defaultFilters);
}
/**
* {@inheritDoc}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2013 the original author or authors.
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@ import org.springframework.util.xml.DomUtils;
* @author Iwein Fuld
* @author Mark Fisher
* @author Gary Russell
* @author Gunnar Hillert
*/
public class FileInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@@ -97,6 +98,7 @@ public class FileInboundChannelAdapterParser extends AbstractPollingInboundChann
factoryBeanBuilder.addPropertyValue("filenameRegex", filenameRegex);
}
IntegrationNamespaceUtils.setValueIfAttributeDefined(factoryBeanBuilder, element, "prevent-duplicates");
IntegrationNamespaceUtils.setValueIfAttributeDefined(factoryBeanBuilder, element, "ignore-hidden");
return BeanDefinitionReaderUtils.registerWithGeneratedName(
factoryBeanBuilder.getBeanDefinition(), parserContext.getRegistry());
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2010 the original author or authors.
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,10 +21,17 @@ import java.util.ArrayList;
import java.util.List;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.integration.file.filters.*;
import org.springframework.integration.file.filters.AcceptAllFileListFilter;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.filters.IgnoreHiddenFileListFilter;
import org.springframework.integration.file.filters.RegexPatternFileListFilter;
import org.springframework.integration.file.filters.SimplePatternFileListFilter;
/**
* @author Mark Fisher
* @author Gunnar Hillert
* @since 1.0.3
*/
public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<File>> {
@@ -37,11 +44,12 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
private volatile String filenameRegex;
private volatile Boolean ignoreHidden = Boolean.TRUE;
private volatile Boolean preventDuplicates;
private final Object monitor = new Object();
public void setFilter(FileListFilter<File> filter) {
this.filter = filter;
}
@@ -54,6 +62,16 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
this.filenameRegex = filenameRegex;
}
/**
* Specify whether hidden files shall be ignored.
* This is {@code true} by default.
* @param ignoreHidden Can be null, which triggers default behavior.
* @since 4.2
*/
public void setIgnoreHidden(Boolean ignoreHidden) {
this.ignoreHidden = ignoreHidden;
}
public void setPreventDuplicates(Boolean preventDuplicates) {
this.preventDuplicates = preventDuplicates;
}
@@ -89,19 +107,26 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
throw new IllegalArgumentException("The 'filename-pattern' and 'filename-regex' attributes are mutually exclusive.");
}
final List<FileListFilter<File>> filtersNeeded = new ArrayList<FileListFilter<File>>();
if (!Boolean.FALSE.equals(this.ignoreHidden)) {
filtersNeeded.add(new IgnoreHiddenFileListFilter());
}
//'filter' is set
if (this.filter != null) {
if (Boolean.TRUE.equals(this.preventDuplicates)) {
createdFilter = this.createCompositeWithAcceptOnceFilter(this.filter);
filtersNeeded.add(new AcceptOnceFileListFilter<File>());
filtersNeeded.add(this.filter);
}
else { // preventDuplicates is either FALSE or NULL
createdFilter = this.filter;
filtersNeeded.add(this.filter);
}
}
// 'file-pattern' or 'file-regex' is set
else if (this.filenamePattern != null || this.filenameRegex != null) {
List<FileListFilter<File>> filtersNeeded = new ArrayList<FileListFilter<File>>();
if (!Boolean.FALSE.equals(this.preventDuplicates)) {
//preventDuplicates is either null or true
filtersNeeded.add(new AcceptOnceFileListFilter<File>());
@@ -112,30 +137,24 @@ public class FileListFilterFactoryBean implements FactoryBean<FileListFilter<Fil
if (this.filenameRegex != null) {
filtersNeeded.add(new RegexPatternFileListFilter(this.filenameRegex));
}
if (filtersNeeded.size() == 1) {
createdFilter = filtersNeeded.get(0);
}
else {
createdFilter = new CompositeFileListFilter<File>(filtersNeeded);
}
}
// no filters are provided
else if (Boolean.FALSE.equals(this.preventDuplicates)) {
createdFilter = new AcceptAllFileListFilter<File>();
filtersNeeded.add(new AcceptAllFileListFilter<File>());
}
else { // preventDuplicates is either TRUE or NULL
createdFilter = new AcceptOnceFileListFilter<File>();
filtersNeeded.add(new AcceptOnceFileListFilter<File>());
}
if (filtersNeeded.size() == 1) {
createdFilter = filtersNeeded.get(0);
}
else {
createdFilter = new CompositeFileListFilter<File>(filtersNeeded);
}
this.result = createdFilter;
}
private CompositeFileListFilter<File> createCompositeWithAcceptOnceFilter(FileListFilter<File> otherFilter) {
CompositeFileListFilter<File> compositeFilter = new CompositeFileListFilter<File>();
compositeFilter.addFilter(new AcceptOnceFileListFilter<File>());
compositeFilter.addFilter(otherFilter);
return compositeFilter;
}
}

View File

@@ -101,6 +101,17 @@ Only files matching this regular expression will be picked up by this adapter.
</xsd:appinfo>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="ignore-hidden">
<xsd:annotation><xsd:documentation><![CDATA[
A boolean flag indicating whether hidden files shall be ignored.
If set to 'false', hidden files will be processed. If not specified,
this value will default to 'true' and an 'IgnoreHiddenFileListFilter'
will be added.
]]></xsd:documentation></xsd:annotation>
<xsd:simpleType>
<xsd:union memberTypes="xsd:boolean xsd:string"/>
</xsd:simpleType>
</xsd:attribute>
<xsd:attribute name="prevent-duplicates" type="xsd:string">
<xsd:annotation>
<xsd:documentation><![CDATA[

View File

@@ -14,6 +14,7 @@
directory="${java.io.tmpdir}"
filter="filter"
comparator="testComparator"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="5000">
<integration:transactional synchronization-factory="syncFactory"/>
@@ -22,7 +23,9 @@
<integration:channel id="successChannel" />
<beans:bean id="filter" class="org.springframework.integration.file.config.FileListFilterFactoryBean"/>
<beans:bean id="filter" class="org.springframework.integration.file.config.FileListFilterFactoryBean">
<beans:property name="ignoreHidden" value="false"/>
</beans:bean>
<beans:bean id="compositeFilter" class="org.springframework.integration.file.filters.CompositeFileListFilter">
<beans:constructor-arg>
@@ -34,9 +37,9 @@
<beans:bean id="testComparator"
class="org.springframework.integration.file.config.FileInboundChannelAdapterParserTests$TestComparator"/>
<beans:bean id="transactionManager" class="org.springframework.integration.transaction.PseudoTransactionManager"/>
<beans:bean id="syncFactory" class="org.springframework.integration.transaction.DefaultTransactionSynchronizationFactory">
<beans:constructor-arg>
<beans:bean class="org.springframework.integration.transaction.ExpressionEvaluatingTransactionSynchronizationProcessor">

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2013 the original author or authors.
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -41,69 +41,70 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
* @author Iwein Fuld
* @author Mark Fisher
* @author Gary Russell
* @author Gunnar Hillert
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class FileInboundChannelAdapterParserTests {
@Autowired(required = true)
private ApplicationContext context;
@Autowired(required = true)
private ApplicationContext context;
@Autowired
private FileReadingMessageSource source;
@Autowired
private FileReadingMessageSource source;
private DirectFieldAccessor accessor;
private DirectFieldAccessor accessor;
@Before
public void init() {
accessor = new DirectFieldAccessor(source);
}
@Before
public void init() {
accessor = new DirectFieldAccessor(source);
}
@Test
public void channelName() throws Exception {
context.getBean("inputDirPoller");
AbstractMessageChannel channel = context.getBean("inputDirPoller", AbstractMessageChannel.class);
assertEquals("Channel should be available under specified id", "inputDirPoller", channel.getComponentName());
}
@Test
public void channelName() throws Exception {
context.getBean("inputDirPoller");
AbstractMessageChannel channel = context.getBean("inputDirPoller", AbstractMessageChannel.class);
assertEquals("Channel should be available under specified id", "inputDirPoller", channel.getComponentName());
}
@Test
public void inputDirectory() {
File expected = new File(System.getProperty("java.io.tmpdir"));
File actual = (File) accessor.getPropertyValue("directory");
assertEquals("'directory' should be set", expected, actual);
}
@Test
public void inputDirectory() {
File expected = new File(System.getProperty("java.io.tmpdir"));
File actual = (File) accessor.getPropertyValue("directory");
assertEquals("'directory' should be set", expected, actual);
}
@Test
public void filter() throws Exception {
DefaultDirectoryScanner scanner = (DefaultDirectoryScanner) accessor.getPropertyValue("scanner");
DirectFieldAccessor scannerAccessor = new DirectFieldAccessor(scanner);
Object filter = scannerAccessor.getPropertyValue("filter");
assertTrue("'filter' should be set",
filter instanceof AcceptOnceFileListFilter);
}
@Test
public void filter() throws Exception {
DefaultDirectoryScanner scanner = (DefaultDirectoryScanner) accessor.getPropertyValue("scanner");
DirectFieldAccessor scannerAccessor = new DirectFieldAccessor(scanner);
Object filter = scannerAccessor.getPropertyValue("filter");
assertTrue("'filter' should be set and be of instance AcceptOnceFileListFilter but got "
+ filter.getClass().getSimpleName(), filter instanceof AcceptOnceFileListFilter);
}
@Test
public void comparator() throws Exception {
Object priorityQueue = accessor.getPropertyValue("toBeReceived");
assertEquals(PriorityBlockingQueue.class, priorityQueue.getClass());
Object expected = context.getBean("testComparator");
DirectFieldAccessor queueAccessor = new DirectFieldAccessor(priorityQueue);
Object innerQueue = queueAccessor.getPropertyValue("q");
Object actual;
if (innerQueue != null) {
actual = new DirectFieldAccessor(innerQueue).getPropertyValue("comparator");
}
else {
// probably running under JDK 7
actual = queueAccessor.getPropertyValue("comparator");
}
assertSame("comparator reference not set, ", expected, actual);
}
@Test
public void comparator() throws Exception {
Object priorityQueue = accessor.getPropertyValue("toBeReceived");
assertEquals(PriorityBlockingQueue.class, priorityQueue.getClass());
Object expected = context.getBean("testComparator");
DirectFieldAccessor queueAccessor = new DirectFieldAccessor(priorityQueue);
Object innerQueue = queueAccessor.getPropertyValue("q");
Object actual;
if (innerQueue != null) {
actual = new DirectFieldAccessor(innerQueue).getPropertyValue("comparator");
}
else {
// probably running under JDK 7
actual = queueAccessor.getPropertyValue("comparator");
}
assertSame("comparator reference not set, ", expected, actual);
}
static class TestComparator implements Comparator<File> {
static class TestComparator implements Comparator<File> {
public int compare(File f1, File f2) {
return 0;
}
public int compare(File f1, File f2) {
return 0;
}
}
}

View File

@@ -14,6 +14,7 @@
<inbound-channel-adapter id="adapterWithPattern"
directory="file:${java.io.tmpdir}"
ignore-hidden="false"
filename-pattern="*.txt" auto-startup="false">
<integration:poller fixed-rate="1000"/>
</inbound-channel-adapter>

View File

@@ -20,6 +20,7 @@
directory="file:${java.io.tmpdir}"
filter="testFilter"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
</inbound-channel-adapter>
@@ -29,6 +30,7 @@
filter="testFilter"
prevent-duplicates="true"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
</inbound-channel-adapter>
@@ -38,6 +40,7 @@
filter="testFilter"
prevent-duplicates="false"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -47,6 +50,7 @@
directory="file:${java.io.tmpdir}"
filename-pattern="test"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -57,6 +61,7 @@
filename-pattern="test"
prevent-duplicates="true"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -67,6 +72,7 @@
filename-pattern="test"
prevent-duplicates="false"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -75,6 +81,7 @@
<inbound-channel-adapter id="defaultAndNull"
directory="file:${java.io.tmpdir}"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -84,6 +91,7 @@
directory="file:${java.io.tmpdir}"
prevent-duplicates="true"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>
@@ -93,6 +101,7 @@
directory="file:${java.io.tmpdir}"
prevent-duplicates="false"
channel="channel"
ignore-hidden="false"
auto-startup="false">
<integration:poller fixed-rate="10000"/>

View File

@@ -46,6 +46,7 @@ public class FileListFilterFactoryBeanTests {
@Test(expected = IllegalArgumentException.class)
public void customFilterAndFilenamePatternAreMutuallyExclusive() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
factory.setFilter(new TestFilter());
factory.setFilenamePattern("foo");
factory.getObject();
@@ -54,6 +55,7 @@ public class FileListFilterFactoryBeanTests {
@Test
public void customFilterAndPreventDuplicatesNull() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
TestFilter testFilter = new TestFilter();
factory.setFilter(testFilter);
FileListFilter<File> result = factory.getObject();
@@ -64,6 +66,7 @@ public class FileListFilterFactoryBeanTests {
@Test
public void customFilterAndPreventDuplicatesTrue() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
TestFilter testFilter = new TestFilter();
factory.setFilter(testFilter);
factory.setPreventDuplicates(Boolean.TRUE);
@@ -77,6 +80,7 @@ public class FileListFilterFactoryBeanTests {
@Test
public void customFilterAndPreventDuplicatesFalse() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
TestFilter testFilter = new TestFilter();
factory.setFilter(testFilter);
factory.setPreventDuplicates(Boolean.FALSE);
@@ -89,6 +93,7 @@ public class FileListFilterFactoryBeanTests {
@SuppressWarnings("unchecked")
public void filenamePatternAndPreventDuplicatesNull() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
factory.setFilenamePattern("foo");
FileListFilter<File> result = factory.getObject();
assertTrue(result instanceof CompositeFileListFilter);
@@ -103,6 +108,7 @@ public class FileListFilterFactoryBeanTests {
@SuppressWarnings("unchecked")
public void filenamePatternAndPreventDuplicatesTrue() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
factory.setFilenamePattern(("foo"));
factory.setPreventDuplicates(Boolean.TRUE);
FileListFilter<File> result = factory.getObject();
@@ -117,6 +123,7 @@ public class FileListFilterFactoryBeanTests {
@Test
public void filenamePatternAndPreventDuplicatesFalse() throws Exception {
FileListFilterFactoryBean factory = new FileListFilterFactoryBean();
factory.setIgnoreHidden(false);
factory.setFilenamePattern(("foo"));
factory.setPreventDuplicates(Boolean.FALSE);
FileListFilter<File> result = factory.getObject();
@@ -124,7 +131,6 @@ public class FileListFilterFactoryBeanTests {
assertThat(result, is(instanceOf(SimplePatternFileListFilter.class)));
}
private static class TestFilter extends AbstractFileListFilter<File> {
@Override
public boolean accept(File file) {

View File

@@ -24,9 +24,24 @@ This is an implementation of `MessageSource` that creates messages from a file s
p:directory="${input.directory}"/>
----
To prevent creating messages for certain files, you may supply a `FileListFilter`.
By default, an `AcceptOnceFileListFilter` is used.
This filter ensures files are picked up only once from the directory.
To prevent creating messages for certain files, you may supply a `FileListFilter`. By default the following 2 filters are used:
* `IgnoreHiddenFileListFilter`
* `AcceptOnceFileListFilter`
The `IgnoreHiddenFileListFilter` ensures that *hidden* files are not being processed.
Please keep in mind that the exact definition of *hidden* is system-dependent. For example,
on _UNIX_-based systems, a file beginning with a period character is considered to be hidden.
_Microsoft Windows_, on the other hand, has a dedicated file attribute to indicate
hidden files.
[IMPORTANT]
=====
The `IgnoreHiddenFileListFilter` was introduced with _version 4.2_. In prior versions hidden files *were being picked up*.
With the default configuration, the `IgnoreHiddenFileListFilter` will be triggered first, then the `AcceptOnceFileListFilter`.
=====
The `AcceptOnceFileListFilter` ensures files are picked up only once from the directory.
[NOTE]
=====
@@ -93,11 +108,11 @@ To do this use the following template.
</beans>
----
Within this namespace you can reduce the FileReadingMessageSource and wrap it in an inbound Channel Adapter like this:
Within this namespace you can reduce the `FileReadingMessageSource` and wrap it in an inbound Channel Adapter like this:
[source,xml]
----
<int-file:inbound-channel-adapter id="filesIn1"
directory="file:${input.directory}" prevent-duplicates="true"/>
directory="file:${input.directory}" prevent-duplicates="true" ignore-hidden="true"/>
<int-file:inbound-channel-adapter id="filesIn2"
directory="file:${input.directory}"
@@ -112,7 +127,20 @@ Within this namespace you can reduce the FileReadingMessageSource and wrap it in
filename-regex="test[0-9]+\.txt" />
----
The first channel adapter is relying on the default filter that just prevents duplication, the second is using a custom filter, the third is using the_filename-pattern_ attribute to add an `AntPathMatcher` based filter, and the fourth is using the _filename-regex_ attribute to add a regular expression Pattern based filter to the `FileReadingMessageSource`.
The first channel adapter example is relying on the default `FileListFilter`s:
* `IgnoreHiddenFileListFilter` (Do not process hidden files)
* `AcceptOnceFileListFilter` (Prevents duplication)
Therefore, you can also leave off the 2 attributes `prevent-duplicates` and `ignore-hidden` as they are `true` by default.
[IMPORTANT]
=====
The `ignore-hidden` attribute was introduced with _Spring Integration 4.2_. In prior versions hidden files *were being picked up*.
=====
The second channel adapter example is using a custom filter, the third is using the _filename-pattern_ attribute to
add an `AntPathMatcher` based filter, and the fourth is using the _filename-regex_ attribute to add a regular expression Pattern based filter to the `FileReadingMessageSource`.
The _filename-pattern_ and _filename-regex_ attributes are each mutually exclusive with the regular _filter_ reference attribute.
However, you can use the _filter_ attribute to reference an instance of `CompositeFileListFilter` that combines any number of filters, including one or more pattern based filters to fit your particular needs.

View File

@@ -37,13 +37,19 @@ For more information, see <<security>>.
As an alternative to the existing `selector` attribute, the `<wire-tap/>` now supports the `selector-expression` attribute.
[[x4.2-file-outbound-channel-adapter]]
==== File Outbound Channel Adapter
[[x4.2-file-changes]]
==== File Changes
The `<int-file:outbound-channel-adapter>` and `<int-file:outbound-gateway>` now support an `append-new-line` attribute.
If set to `true`, a new line is appended to the file after a message is written.
The default attribute value is `false`.
The `ignore-hidden` attribute has been introduced for the `<int-file:inbound-channel-adapter>` to pick up or not
the _hidden_ files from the source directory.
It is `true` by default.
See <<files>> for more information.
[[x4.2-class-package-change]]
==== Class Package Change