diff --git a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/DirectoryScanner.java b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/DirectoryScanner.java new file mode 100644 index 0000000000..d1c503f319 --- /dev/null +++ b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/DirectoryScanner.java @@ -0,0 +1,39 @@ +/* + * Copyright 2002-2009 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file; + +import java.io.File; + +/** + * Strategy for scanning directories. Implementations may select all children and grandchildren of the scanned directory + * in any order. This interface is intended to enable the selection and ordering of files in a directory like + * RecursiveDirectoryScanner. If the only requirement is to ignore certain files a FileListFilter implementation should + * suffice. + * + * @author Iwein Fuld + */ +public interface DirectoryScanner { + + /** + * Scans the directory according to the strategy particular to this implementation and returns the selected files + * as a File array. + * + * @param directory the directory to scan for files + * @return a list of files representing the content of the directory + */ + File[] listFiles(File directory) throws IllegalArgumentException; +} diff --git a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java index daa8cce496..ae00e8c966 100644 --- a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java +++ b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java @@ -67,6 +67,8 @@ public class FileReadingMessageSource implements MessageSource, private volatile File directory; + private volatile DirectoryScanner scanner = new ToplevelDirectoryScanner(); + private volatile boolean autoCreateDirectory = true; /** @@ -109,6 +111,14 @@ public class FileReadingMessageSource implements MessageSource, this.directory = directory; } + /** + * Optionally specify a custom scanner, for example the + * {@link org.springframework.integration.file.RecursiveLeafOnlyDirectoryScanner} + */ + public void setScanner(DirectoryScanner scanner) { + this.scanner = scanner; + } + /** * Specify whether to create the source directory automatically if it does * not yet exist upon initialization. By default, this value is @@ -202,7 +212,7 @@ public class FileReadingMessageSource implements MessageSource, } private void scanInputDirectory() { - File[] fileArray = directory.listFiles(); + File[] fileArray = scanner.listFiles(directory); if (fileArray == null) { throw new MessagingException("The path [" + this.directory + "] does not denote a properly accessible directory."); @@ -241,7 +251,7 @@ public class FileReadingMessageSource implements MessageSource, * Implementation of FileLocker that doesn't provide any protection against * duplicate listing. */ - class NoopFileLocker implements FileLocker { + private static class NoopFileLocker implements FileLocker { public boolean lock(File fileToLock) { return true; @@ -251,4 +261,10 @@ public class FileReadingMessageSource implements MessageSource, // noop } } + + private static class ToplevelDirectoryScanner implements DirectoryScanner { + public File[] listFiles(File directory) throws IllegalArgumentException { + return directory.listFiles(); + } + } } diff --git a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/RecursiveLeafOnlyDirectoryScanner.java b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/RecursiveLeafOnlyDirectoryScanner.java new file mode 100644 index 0000000000..411fe49a73 --- /dev/null +++ b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/RecursiveLeafOnlyDirectoryScanner.java @@ -0,0 +1,28 @@ +package org.springframework.integration.file; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * DirectoryScanner that lists all files inside a directory and subdirectories, without limit. This scanner should not + * be used with directories that contain a vast number of files, as all the file names will be read into memory and the + * scanning will be done recursively. + * + * @author Iwein Fuld + */ +public class RecursiveLeafOnlyDirectoryScanner implements DirectoryScanner { + public File[] listFiles(File directory) throws IllegalArgumentException { + File[] rootFiles = directory.listFiles(); + List files = new ArrayList(rootFiles.length); + for (File rootFile : rootFiles) { + if (rootFile.isDirectory()){ + files.addAll(Arrays.asList(listFiles(rootFile))); + } else { + files.add(rootFile); + } + } + return files.toArray(new File[files.size()]); + } +} diff --git a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java index 0b6250ffae..88f1e4b437 100644 --- a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java +++ b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileInboundChannelAdapterParser.java @@ -42,6 +42,7 @@ public class FileInboundChannelAdapterParser extends AbstractPollingInboundChann BeanDefinitionBuilder builder = BeanDefinitionBuilder.genericBeanDefinition( PACKAGE_NAME + ".config.FileReadingMessageSourceFactoryBean"); IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "comparator"); + IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "scanner"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "directory"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "auto-create-directory"); String filterBeanName = this.registerFileListFilter(element, parserContext); diff --git a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileReadingMessageSourceFactoryBean.java b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileReadingMessageSourceFactoryBean.java index 1d21c08dc8..0ec6668ba1 100644 --- a/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileReadingMessageSourceFactoryBean.java +++ b/org.springframework.integration.file/src/main/java/org/springframework/integration/file/config/FileReadingMessageSourceFactoryBean.java @@ -19,6 +19,7 @@ package org.springframework.integration.file.config; import org.springframework.beans.factory.FactoryBean; import org.springframework.context.ResourceLoaderAware; import org.springframework.core.io.ResourceLoader; +import org.springframework.integration.file.DirectoryScanner; import org.springframework.integration.file.FileListFilter; import org.springframework.integration.file.FileReadingMessageSource; @@ -40,7 +41,9 @@ public class FileReadingMessageSourceFactoryBean implements FactoryBean, Resourc private volatile FileListFilter filter; - private volatile Comparator comparator; + private volatile Comparator comparator; + + private volatile DirectoryScanner scanner; private volatile Boolean scanEachPoll; @@ -61,7 +64,11 @@ public class FileReadingMessageSourceFactoryBean implements FactoryBean, Resourc this.comparator = comparator; } - public void setFilter(FileListFilter filter) { + public void setScanner(DirectoryScanner scanner) { + this.scanner = scanner; + } + + public void setFilter(FileListFilter filter) { this.filter = filter; } @@ -105,6 +112,9 @@ public class FileReadingMessageSourceFactoryBean implements FactoryBean, Resourc if (this.autoCreateDirectory != null) { this.source.setAutoCreateDirectory(this.autoCreateDirectory); } + if (this.scanner != null) { + this.source.setScanner(this.scanner); + } this.source.afterPropertiesSet(); } } diff --git a/org.springframework.integration.file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-2.0.xsd b/org.springframework.integration.file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-2.0.xsd index 042602e910..f2d65bcb35 100644 --- a/org.springframework.integration.file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-2.0.xsd +++ b/org.springframework.integration.file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-2.0.xsd @@ -1,66 +1,79 @@ + xmlns:xsd="http://www.w3.org/2001/XMLSchema" + xmlns:beans="http://www.springframework.org/schema/beans" + xmlns:tool="http://www.springframework.org/schema/tool" + xmlns:integration="http://www.springframework.org/schema/integration" + targetNamespace="http://www.springframework.org/schema/integration/file" + elementFormDefault="qualified" + attributeFormDefault="unqualified"> - - + + - - + - + - - - - Configures an inbound Channel Adapter that polls a directory and sends - Messages whose payloads are instances of java.io.File. - - - - - - - - - - - - - - - - - - - - + + + Configures an inbound Channel Adapter that polls a directory and sends + Messages whose payloads are instances of java.io.File. + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + Reference to a custom DirectoryScanner implementation. + + + + + + + - - - - - - - Specify whether to automatically create the source directory if it does not yet exist when this - adapter is being initialized. The default value is 'true'. If set to 'false' and the directory - does not exist upon initialization, an Exception will be thrown. - - - - - + + + + + + + Specify whether to automatically create the source directory if it does not yet exist when this + adapter is being initialized. The default value is 'true'. If set to 'false' and the directory + does not exist upon initialization, an Exception will be thrown. + + + + + - - - - - Configures an outbound Channel Adapter that writes Message payloads to a File. - - - - - - - - - - - - - - - - - + + + + + Configures an outbound Channel Adapter that writes Message payloads to a File. + + + + + + + + + + + + + + + + + - - - - - Configures an outbound Gateway that writes request Message payloads to a File and - then generates a reply Message containing the newly written File as its payload. - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + Configures an outbound Gateway that writes request Message payloads to a File and + then generates a reply Message containing the newly written File as its payload. + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - + + + + - - - - - - Specify whether to automatically create the destination directory if it does not yet exist - when this adapter is being initialized. The default value is 'true'. If set to 'false' and - the directory does not exist upon initialization, an Exception will be thrown. - - - - - + + + + + + Specify whether to automatically create the destination directory if it does not yet exist + when this adapter is being initialized. The default value is 'true'. If set to 'false' and + the directory does not exist upon initialization, an Exception will be thrown. + + + + + - - - - - Creates a Transformer that converts a File payload to a String. - - - - - - - - - + + + + + Creates a Transformer that converts a File payload to a String. + + + + + + + + + - - - - - Creates a Transformer that converts a File payload to an array of bytes. - - - - - - - + + + + + Creates a Transformer that converts a File payload to an array of bytes. + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests-context.xml b/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests-context.xml new file mode 100644 index 0000000000..780e38f940 --- /dev/null +++ b/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests-context.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests.java b/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests.java new file mode 100644 index 0000000000..641d85c1af --- /dev/null +++ b/org.springframework.integration.file/src/test/java/org/springframework/integration/file/recursive/FileInboundChannelAdapterWithRecursiveDirectoryTests.java @@ -0,0 +1,76 @@ +/* + * Copyright 2002-2009 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.recursive; + +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.integration.channel.PollableChannel; +import org.springframework.integration.core.Message; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertThat; +import static org.junit.matchers.JUnitMatchers.hasItem; +import static org.junit.matchers.JUnitMatchers.hasItems; +import static org.springframework.integration.test.matcher.PayloadMatcher.hasPayload; + +/** + * @author Iwein Fuld + */ +@ContextConfiguration +@RunWith(SpringJUnit4ClassRunner.class) +public class FileInboundChannelAdapterWithRecursiveDirectoryTests { + + @Autowired + private TemporaryFolder directory; + + @Autowired + private PollableChannel files; + + @Test(timeout = 2000) + public void shouldScanDirectoriesRecursively() throws IOException { + + //when + File folder = directory.newFolder("foo"); + File file = new File(folder, "bar"); + file.createNewFile(); + + //verify + assertThat(files.receive(), hasPayload(file)); + } + + @Test(timeout = 2000) + public void shouldReturnFilesMultipleLevels() throws IOException { + + //when + File folder = directory.newFolder("foo"); + File siblingFile = directory.newFile("bar"); + File childFile = new File(folder, "baz"); + childFile.createNewFile(); + + List> received = Arrays.asList(files.receive(), files.receive()); + //verify + assertThat(received, hasItems(hasPayload(siblingFile), hasPayload(childFile))); + } +} diff --git a/org.springframework.integration/src/test/java/log4j.properties b/org.springframework.integration/src/test/resources/log4j.properties similarity index 65% rename from org.springframework.integration/src/test/java/log4j.properties rename to org.springframework.integration/src/test/resources/log4j.properties index c5000b932a..c916dc5d9a 100644 --- a/org.springframework.integration/src/test/java/log4j.properties +++ b/org.springframework.integration/src/test/resources/log4j.properties @@ -4,4 +4,5 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%c{1}: %m%n -log4j.category.org.springframework.integration=INFO \ No newline at end of file +log4j.category.org.springframework.integration=INFO +log4j.category.org.springframework.integration.file=DEBUG \ No newline at end of file