diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java index 0de8aebcc0..bcae8b9279 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ import org.w3c.dom.Element; * * @author Oleg Zhurakousky * @author Mark Fisher + * @author Gary Russell * @since 2.0 */ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser { @@ -62,6 +63,7 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst if (StringUtils.hasText(comparator)) { messageSourceBuilder.addConstructorArgReference(comparator); } + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(messageSourceBuilder, element, "local-filter"); IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, "local-directory"); IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, "auto-create-local-directory"); String localFileGeneratorExpression = element.getAttribute("local-filename-generator-expression"); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java index 28485c2bea..74f023b208 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java @@ -79,6 +79,8 @@ public abstract class AbstractInboundFileSynchronizingMessageSource extends M */ private final FileReadingMessageSource fileSource; + private volatile FileListFilter localFileListFilter = new AcceptOnceFileListFilter(); + public AbstractInboundFileSynchronizingMessageSource(AbstractInboundFileSynchronizer synchronizer) { this(synchronizer, null); @@ -104,6 +106,20 @@ public abstract class AbstractInboundFileSynchronizingMessageSource extends M this.localDirectory = localDirectory; } + /** + * A {@link FileListFilter} used to determine which files will generate messages + * after they have been synchronized. It will be combined with a filter that + * will prevent accessing files that are in the process of being synchronized + * (files having the {@link AbstractInboundFileSynchronizer#getTemporaryFileSuffix()}). + *

+ * The default is an {@link AcceptOnceFileListFilter} which filters duplicate file + * names (processed during the current execution). + * @param localFileListFilter + */ + public void setLocalFilter(FileListFilter localFileListFilter) { + this.localFileListFilter = localFileListFilter; + } + @Override protected void onInit() { Assert.notNull(this.localDirectory, "localDirectory must not be null"); @@ -153,7 +169,7 @@ public abstract class AbstractInboundFileSynchronizingMessageSource extends M private FileListFilter buildFilter() { Pattern completePattern = Pattern.compile("^.*(?(Arrays.asList( - new AcceptOnceFileListFilter(), + this.localFileListFilter, new RegexPatternFileListFilter(completePattern))); } diff --git a/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd b/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd index 4443e15c81..6cbb5ceb88 100644 --- a/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd +++ b/spring-integration-ftp/src/main/resources/org/springframework/integration/ftp/config/spring-integration-ftp-3.0.xsd @@ -190,7 +190,28 @@ Allows you to specify a reference to a [org.springframework.integration.file.filters.FileListFilter] - bean. + bean. This filter is applied to files on the remote server and + only files that pass the filter are retrieved. + + + + + + + + + + + + Allows you to specify a reference to a + [org.springframework.integration.file.filters.FileListFilter] + bean. This filter is applied to files after they have been + retrieved. The default is an AcceptOnceFileListFilter which means that, + even if a new instance of a file is retrieved from the remote server, + a message won't be generated. The filter provided here is combined + with a filter that prevents the message source from processing + files that are currently being downloaded. diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml index 96a5762125..4dc42a1f70 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests-context.xml @@ -24,11 +24,14 @@ local-filename-generator-expression="#this.toUpperCase() + '.a'" comparator="comparator" temporary-file-suffix=".foo" + local-filter="acceptAllFilter" remote-directory="foo/bar"> + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java index 33e2c03667..a046da08cb 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParserTests.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Collection; import java.util.Comparator; import java.util.Map; import java.util.concurrent.PriorityBlockingQueue; @@ -34,6 +35,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.integration.MessageChannel; import org.springframework.integration.endpoint.SourcePollingChannelAdapter; +import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.remote.session.CachingSessionFactory; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter; @@ -77,6 +79,8 @@ public class FtpInboundChannelAdapterParserTests { assertNotNull(filter); Object sessionFactory = TestUtils.getPropertyValue(fisync, "sessionFactory"); assertTrue(DefaultFtpSessionFactory.class.isAssignableFrom(sessionFactory.getClass())); + FileListFilter acceptAllFilter = ac.getBean("acceptAllFilter", FileListFilter.class); + assertTrue(TestUtils.getPropertyValue(inbound, "fileSource.scanner.filter.fileFilters", Collection.class).contains(acceptAllFilter)); } @Test diff --git a/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-3.0.xsd b/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-3.0.xsd index b71272951d..f6a1942485 100644 --- a/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-3.0.xsd +++ b/spring-integration-sftp/src/main/resources/org/springframework/integration/sftp/config/spring-integration-sftp-3.0.xsd @@ -151,7 +151,8 @@ Allows you to specify a reference to a [org.springframework.integration.file.filters.FileListFilter] - bean. + bean. This filter is applied to files on the remote server and + only files that pass the filter are retrieved. @@ -195,6 +196,26 @@ + + + + + + + + + Allows you to specify a reference to a + [org.springframework.integration.file.filters.FileListFilter] + bean. This filter is applied to files after they have been + retrieved. The default is an AcceptOnceFileListFilter which means that, + even if a new instance of a file is retrieved from the remote server, + a message won't be generated. The filter provided here is combined + with a filter that prevents the message source from processing + files that are currently being downloaded. + + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml index 8e4a85fae8..3e461e1db5 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests-context.xml @@ -48,11 +48,14 @@ local-filename-generator-expression="#this.toUpperCase() + '.a'" temporary-file-suffix=".bar" comparator="comparator" + local-filter="acceptAllFilter" delete-remote-files="${delete.remote.files}"> + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java index 8c8b3475f1..fb60fafd47 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/config/InboundChannelAdapterParserTests.java @@ -19,10 +19,11 @@ package org.springframework.integration.sftp.config; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.Collection; import java.util.Comparator; import java.util.concurrent.PriorityBlockingQueue; @@ -30,7 +31,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; - import org.springframework.beans.factory.BeanCreationException; import org.springframework.beans.factory.BeanDefinitionStoreException; import org.springframework.context.ApplicationContext; @@ -38,6 +38,7 @@ import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.integration.MessageChannel; import org.springframework.integration.core.PollableChannel; import org.springframework.integration.endpoint.SourcePollingChannelAdapter; +import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizer; import org.springframework.integration.sftp.inbound.SftpInboundFileSynchronizingMessageSource; import org.springframework.integration.test.util.TestUtils; @@ -87,6 +88,8 @@ public class InboundChannelAdapterParserTests { assertEquals(".", remoteFileSeparator); PollableChannel requestChannel = context.getBean("requestChannel", PollableChannel.class); assertNotNull(requestChannel.receive(2000)); + FileListFilter acceptAllFilter = context.getBean("acceptAllFilter", FileListFilter.class); + assertTrue(TestUtils.getPropertyValue(source, "fileSource.scanner.filter.fileFilters", Collection.class).contains(acceptAllFilter)); } @Test diff --git a/src/reference/docbook/ftp.xml b/src/reference/docbook/ftp.xml index f1cc5f75ac..1ebd592aee 100644 --- a/src/reference/docbook/ftp.xml +++ b/src/reference/docbook/ftp.xml @@ -151,6 +151,7 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { remote-directory="some/remote/path" remote-file-separator="/" local-filename-generator-expression="#this.toUpperCase() + '.a'" + local-filter="myFilter" local-directory="."> ]]> @@ -174,13 +175,20 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { (e.g. filename-regex=".*\.test$"). And of course if you need complete control you can use filter attribute and provide a reference to any custom implementation of the org.springframework.integration.file.filters.FileListFilter, a strategy interface for filtering a - list of files. + list of files. This filter determines which remote files are retrieved. - - As of Spring Integration 2.0.2, we have added a 'remote-file-separator' attribute. That allows you to configure a - file separator character to use if the default '/' is not applicable for your particular environment. + + Beginning with 3.0, you can also specify a filter used to filter the files locally, once they have + been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing + files with the same name multiple times in the same JVM execution; this can now be overridden + (for example with an AcceptAllFileListFilter), using the local-filter attribute. + Previously, the default AcceptOnceFileListFilter could not be overridden. - + + The 'remote-file-separator' attribute allows you to configure a + file separator character to use if the default '/' is not applicable for your particular environment. + + Please refer to the schema for more details on these attributes. diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 4830a2eeb4..debf56db43 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -227,6 +227,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp local-directory="file:target/foo" auto-create-local-directory="true" local-filename-generator-expression="#this.toUpperCase() + '.a'" + local-filter="myFilter" delete-remote-files="false"> ]]> @@ -251,8 +252,15 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp (e.g. filename-regex=".*\.test$"). And of course if you need complete control you can use the filter attribute to provide a reference to a custom implementation of the org.springframework.integration.file.filters.FileListFilter - a strategy interface for filtering a - list of files. + list of files. This filter determines which remote files are retrieved. + + Beginning with 3.0, you can also specify a filter used to filter the files locally, once they have + been retrieved. The default filter is an AcceptOnceFileListFilter which prevents processing + files with the same name multiple times in the same JVM execution; this can now be overridden + (for example with an AcceptAllFileListFilter), using the local-filter attribute. + Previously, the default AcceptOnceFileListFilter could not be overridden. + Please refer to the schema for more detail on these attributes. diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index 211d82d9c4..7430c0e9ed 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -88,6 +88,27 @@ URI-schemes supported by Spring Web Services. For more information see . +

+ (S)FTP(S) Inbound Adapters + + Previously, there was no way to override the default filter used to process files retrieved + from a remote server. The filter attribute determines which files are retrieved + but the FileReadingMessageSource uses an + AcceptOnceFileListFilter. This means that if a new copy of a file + is retrieved, with the same name as a previously copied file, no message was sent from the + adapter. + + + With this release, a new attribute local-filter allows you to override the + default filter, for example with an AcceptAllFileListFilter, or some + other custom filter. + + + For users that wish the behavior of the AcceptOnceFileListFilter + to be maintained across JVM executions, a custom filter that retains state, perhaps on + the file system, can now be configured. + +
(S)FTP(S) Gateways