From 96ca9647a4f2b8bcd4bdfd47b8420652826f9486 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Tue, 9 Aug 2016 17:52:17 -0400 Subject: [PATCH] INT-4086: (S)FTP: Add Local File Tree Support JIRA: https://jira.spring.io/browse/INT-4086 The remote files may be carried with sub-path, not just file name. The same sub-dirs logic can be achieve with the `localFilenameGeneratorExpression` * Add the logic into `AbstractInboundFileSynchronizer` to build sub-dirs for local files * Switch on `WatchService` for the internal `FileReadingMessageSource` in the `AbstractInboundFileSynchronizingMessageSource` to react for all the changes in the `localDirectory` Introduce `RecursiveDirectoryScanner` and use it for `AbstractInboundFileSynchronizingMessageSource` by default * Add `maxDepth` and `fileVisitOptions` options to the `RecursiveDirectoryScanner` * Polishing JavaDocs, tests and docs --- .../file/DefaultDirectoryScanner.java | 7 +- .../file/FileReadingMessageSource.java | 4 + .../file/RecursiveDirectoryScanner.java | 90 +++++++++++++++++++ .../file/filters/AbstractFileListFilter.java | 4 +- ...actPersistentAcceptOnceFileListFilter.java | 2 +- .../filters/ExpressionFileListFilter.java | 2 +- .../filters/IgnoreHiddenFileListFilter.java | 4 +- .../AbstractInboundFileSynchronizer.java | 10 ++- ...InboundFileSynchronizingMessageSource.java | 29 +++++- .../file/RecursiveDirectoryScannerTests.java | 87 ++++++++++++++++++ .../AbstractRemoteFileSynchronizerTests.java | 3 +- ...oundRemoteFileSystemSynchronizerTests.java | 38 +++++--- ...oundRemoteFileSystemSynchronizerTests.java | 2 + src/reference/asciidoc/file.adoc | 7 ++ src/reference/asciidoc/ftp.adoc | 14 ++- src/reference/asciidoc/sftp.adoc | 13 ++- src/reference/asciidoc/whats-new.adoc | 4 + 17 files changed, 289 insertions(+), 31 deletions(-) create mode 100644 spring-integration-file/src/main/java/org/springframework/integration/file/RecursiveDirectoryScanner.java create mode 100644 spring-integration-file/src/test/java/org/springframework/integration/file/RecursiveDirectoryScannerTests.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/DefaultDirectoryScanner.java b/spring-integration-file/src/main/java/org/springframework/integration/file/DefaultDirectoryScanner.java index c75ef55c2e..8b3b7e67f7 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/DefaultDirectoryScanner.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/DefaultDirectoryScanner.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,6 +34,7 @@ import org.springframework.messaging.MessagingException; * @author Iwein Fuld * @author Gunnar Hillert * @author Artem Bilan + * * @since 2.0 */ public class DefaultDirectoryScanner implements DirectoryScanner { @@ -80,12 +81,12 @@ public class DefaultDirectoryScanner implements DirectoryScanner { * @param file the file to try to claim. */ @Override - public final boolean tryClaim(File file) { + public boolean tryClaim(File file) { return (this.locker == null) || this.locker.lock(file); } @Override - public final List listFiles(File directory) throws IllegalArgumentException { + public List listFiles(File directory) throws IllegalArgumentException { File[] files = listEligibleFiles(directory); if (files == null) { throw new MessagingException("The path [" + directory diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java index d438b999c9..0c15fd8a1a 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java @@ -273,6 +273,10 @@ public class FileReadingMessageSource extends IntegrationObjectSupport implement this.useWatchService = useWatchService; } + public boolean isUseWatchService() { + return this.useWatchService; + } + /** * The {@link WatchService} event types. * If {@link #setUseWatchService} isn't {@code true}, this option is ignored. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/RecursiveDirectoryScanner.java b/spring-integration-file/src/main/java/org/springframework/integration/file/RecursiveDirectoryScanner.java new file mode 100644 index 0000000000..709513eaf4 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/RecursiveDirectoryScanner.java @@ -0,0 +1,90 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.springframework.integration.file.filters.AbstractFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; +import org.springframework.util.Assert; + +/** + * The {@link DefaultDirectoryScanner} extension which walks through the directory tree + * using {@link Files#walk(Path, int, FileVisitOption...)}. + *

+ * By default this class visits all levels of the file tree without any {@link FileVisitOption}s. + * + * @author Artem Bilan + * + * @since 5.0 + * + * @see Files#walk + */ +public class RecursiveDirectoryScanner extends DefaultDirectoryScanner { + + private int maxDepth = Integer.MAX_VALUE; + + private FileVisitOption[] fileVisitOptions = new FileVisitOption[0]; + + /** + * The maximum number of directory levels to visit. + * @param maxDepth the maximum number of directory levels to visit + */ + public void setMaxDepth(int maxDepth) { + this.maxDepth = maxDepth; + } + + /** + * The options to configure the traversal. + * @param fileVisitOptions options to configure the traversal + */ + public void setFileVisitOptions(FileVisitOption... fileVisitOptions) { + Assert.notNull(fileVisitOptions, "'fileVisitOptions' must not be null"); + this.fileVisitOptions = fileVisitOptions; + } + + @Override + public List listFiles(File directory) throws IllegalArgumentException { + FileListFilter filter = getFilter(); + boolean supportAcceptFilter = filter instanceof AbstractFileListFilter; + try { + Stream fileStream = Files.walk(directory.toPath(), this.maxDepth, this.fileVisitOptions) + .skip(1) + .map(Path::toFile) + .filter(file -> !supportAcceptFilter + || ((AbstractFileListFilter) filter).accept(file)); + + if (supportAcceptFilter) { + return fileStream.collect(Collectors.toList()); + } + else { + return filter.filterFiles(fileStream.toArray(File[]::new)); + } + } + catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractFileListFilter.java index 745d940c08..f8924a0bbd 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,6 +47,6 @@ public abstract class AbstractFileListFilter implements FileListFilter { * @param file The file. * @return true if the file passes the filter. */ - protected abstract boolean accept(F file); + public abstract boolean accept(F file); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java index 2fa5c7c6bf..406c20d234 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java @@ -70,7 +70,7 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter extends Abst } @Override - protected boolean accept(F file) { + public boolean accept(F file) { String key = buildKey(file); synchronized (this.monitor) { String newValue = value(file); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java index a874add028..5a96b10c45 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/ExpressionFileListFilter.java @@ -60,7 +60,7 @@ public class ExpressionFileListFilter extends AbstractFileListFilter } @Override - protected boolean accept(F file) { + public boolean accept(F file) { return this.expression.getValue(getEvaluationContext(), file, Boolean.class); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/IgnoreHiddenFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/IgnoreHiddenFileListFilter.java index bbc1aa4aa7..85db50e2c1 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/IgnoreHiddenFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/IgnoreHiddenFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2016 the original author or authors. + * Copyright 2015-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ public class IgnoreHiddenFileListFilter extends AbstractFileListFilter { * @return Returns {@code true} for any non-hidden files. */ @Override - protected boolean accept(File file) { + public boolean accept(File file) { return !file.isHidden(); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java index efbe25de10..a8c80db46e 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java @@ -25,6 +25,7 @@ import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.regex.Matcher; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -100,7 +101,7 @@ public abstract class AbstractInboundFileSynchronizer * Should we transfer the remote file timestamp * to the local file? By default this is false. */ - private volatile boolean preserveTimestamp; + private volatile boolean preserveTimestamp; private BeanFactory beanFactory; @@ -313,7 +314,12 @@ public abstract class AbstractInboundFileSynchronizer long modified = getModified(remoteFile); File localFile = new File(localDirectory, localFileName); - if (!localFile.exists() || (this.preserveTimestamp && modified != localFile.lastModified())) { + boolean exists = localFile.exists(); + if (!exists || (this.preserveTimestamp && modified != localFile.lastModified())) { + if (!exists && + localFileName.replaceAll("/", Matcher.quoteReplacement(File.separator)).contains(File.separator)) { + localFile.getParentFile().mkdirs(); //NOSONAR - will fail on the writing below + } String tempFileName = localFile.getAbsolutePath() + this.temporaryFileSuffix; File tempFile = new File(tempFileName); OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile)); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java index 80505be641..6e128c3bec 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java @@ -27,6 +27,7 @@ import org.springframework.beans.factory.BeanInitializationException; import org.springframework.context.Lifecycle; import org.springframework.integration.endpoint.AbstractFetchLimitingMessageSource; import org.springframework.integration.file.FileReadingMessageSource; +import org.springframework.integration.file.RecursiveDirectoryScanner; import org.springframework.integration.file.filters.AcceptOnceFileListFilter; import org.springframework.integration.file.filters.CompositeFileListFilter; import org.springframework.integration.file.filters.FileListFilter; @@ -128,6 +129,22 @@ public abstract class AbstractInboundFileSynchronizingMessageSource this.localFileListFilter = localFileListFilter; } + /** + * Switch the local {@link FileReadingMessageSource} to use its internal + * {@code FileReadingMessageSource.WatchServiceDirectoryScanner}. + * @param useWatchService the {@code boolean} flag to switch to + * {@code FileReadingMessageSource.WatchServiceDirectoryScanner} on {@code true}. + * @since 5.0 + */ + public void setUseWatchService(boolean useWatchService) { + this.fileSource.setUseWatchService(useWatchService); + if (useWatchService) { + this.fileSource.setWatchEvents(FileReadingMessageSource.WatchEventType.CREATE, + FileReadingMessageSource.WatchEventType.MODIFY, + FileReadingMessageSource.WatchEventType.DELETE); + } + } + @Override public void afterPropertiesSet() throws Exception { super.afterPropertiesSet(); @@ -149,7 +166,15 @@ public abstract class AbstractInboundFileSynchronizingMessageSource this.localFileListFilter = new FileSystemPersistentAcceptOnceFileListFilter( new SimpleMetadataStore(), getComponentName()); } - this.fileSource.setFilter(this.buildFilter()); + FileListFilter filter = buildFilter(); + if (!this.fileSource.isUseWatchService()) { + RecursiveDirectoryScanner directoryScanner = new RecursiveDirectoryScanner(); + directoryScanner.setFilter(filter); + this.fileSource.setScanner(directoryScanner); + } + else { + this.fileSource.setFilter(filter); + } if (this.getBeanFactory() != null) { this.fileSource.setBeanFactory(this.getBeanFactory()); } @@ -168,12 +193,14 @@ public abstract class AbstractInboundFileSynchronizingMessageSource @Override public void start() { this.running = true; + this.fileSource.start(); } @Override public void stop() { this.running = false; try { + this.fileSource.stop(); this.synchronizer.close(); } catch (IOException e) { diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/RecursiveDirectoryScannerTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/RecursiveDirectoryScannerTests.java new file mode 100644 index 0000000000..6dfe5c3e54 --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/RecursiveDirectoryScannerTests.java @@ -0,0 +1,87 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file; + +import static org.hamcrest.Matchers.hasItem; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import org.springframework.integration.file.filters.AcceptOnceFileListFilter; + +/** + * @author Iwein Fuld + * @author Gunnar Hillert + * @author Gary Russell + * @author Artem Bilan + * + * @since 5.0 + */ +public class RecursiveDirectoryScannerTests { + + private File subFolder; + + private File subSubFolder; + + private File topLevelFile; + + private File subLevelFile; + + private File subSubLevelFile; + + @Rule + public TemporaryFolder recursivePath = new TemporaryFolder(); + + @Before + public void setup() throws IOException { + this.subFolder = this.recursivePath.newFolder("subFolder"); + this.subSubFolder = new File(this.subFolder, "subSubFolder"); + this.subSubFolder.mkdir(); + this.topLevelFile = this.recursivePath.newFile("file1"); + this.subLevelFile = new File(this.subFolder, "file2"); + this.subLevelFile.createNewFile(); + this.subSubLevelFile = new File(this.subSubFolder, "file3"); + this.subSubLevelFile.createNewFile(); + } + + @Test + public void shouldReturnAllFilesIncludingDirs() throws IOException { + RecursiveDirectoryScanner scanner = new RecursiveDirectoryScanner(); + scanner.setFilter(new AcceptOnceFileListFilter<>()); + List files = scanner.listFiles(this.recursivePath.getRoot()); + assertEquals(5, files.size()); + assertThat(files, hasItem(this.topLevelFile)); + assertThat(files, hasItem(this.subLevelFile)); + assertThat(files, hasItem(this.subSubLevelFile)); + assertThat(files, hasItem(this.subFolder)); + assertThat(files, hasItem(this.subSubFolder)); + File file = new File(this.subSubFolder, "file4"); + file.createNewFile(); + files = scanner.listFiles(this.recursivePath.getRoot()); + assertEquals(1, files.size()); + assertThat(files, hasItem(file)); + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java index 173304dd3c..92ec8a150f 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java @@ -131,6 +131,7 @@ public class AbstractRemoteFileSynchronizerTests { source.setMaxFetchSize(1); source.setBeanName("maxFetchSizeSource"); source.afterPropertiesSet(); + source.start(); source.receive(); assertEquals(1, count.get()); @@ -138,7 +139,7 @@ public class AbstractRemoteFileSynchronizerTests { source.receive(); sync.synchronizeToLocalDirectory(mock(File.class), 1); source.receive(); - sync.close(); + source.stop(); } private AbstractInboundFileSynchronizer createLimitingSynchronizer(final AtomicInteger count) { diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java index 918f797182..abdf77a7f0 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpInboundRemoteFileSystemSynchronizerTests.java @@ -74,13 +74,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { @Before @After public void cleanup() { - File file = new File("test"); - if (file.exists()) { - for (File f : file.listFiles()) { - f.delete(); - } - file.delete(); - } + recursiveDelete(new File("test")); } @Test @@ -109,7 +103,7 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { synchronizer.setFilter(filter); ExpressionParser expressionParser = new SpelExpressionParser(new SpelParserConfiguration(true, true)); - Expression expression = expressionParser.parseExpression("#this.toUpperCase() + '.a'"); + Expression expression = expressionParser.parseExpression("'subdir/' + #this.toUpperCase() + '.a'"); synchronizer.setLocalFilenameGeneratorExpression(expression); synchronizer.setBeanFactory(mock(BeanFactory.class)); synchronizer.afterPropertiesSet(); @@ -126,6 +120,8 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { localFileListFilter.addFilter(localAcceptOnceFilter); ms.setLocalFilter(localFileListFilter); ms.afterPropertiesSet(); + ms.start(); + Message atestFile = ms.receive(); assertNotNull(atestFile); assertEquals("A.TEST.a", atestFile.getPayload().getName()); @@ -146,13 +142,13 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { // two times because on the third receive (above) the internal queue will be empty, so it will attempt verify(synchronizer, times(2)).synchronizeToLocalDirectory(localDirectory, Integer.MIN_VALUE); - assertTrue(new File("test/A.TEST.a").exists()); - assertTrue(new File("test/B.TEST.a").exists()); + assertTrue(new File("test/subdir/A.TEST.a").exists()); + assertTrue(new File("test/subdir/B.TEST.a").exists()); TestUtils.getPropertyValue(localAcceptOnceFilter, "seenSet", Collection.class).clear(); - new File("test/A.TEST.a").delete(); - new File("test/B.TEST.a").delete(); + new File("test/subdir/A.TEST.a").delete(); + new File("test/subdir/B.TEST.a").delete(); // the remote filter should prevent a re-fetch nothing = ms.receive(); assertNull(nothing); @@ -192,6 +188,24 @@ public class FtpInboundRemoteFileSystemSynchronizerTests { assertEquals(0, localDirectory.list().length); } + private static void recursiveDelete(File file) { + if (file != null && file.exists()) { + File[] files = file.listFiles(); + if (files != null) { + for (File f : files) { + if (f.isDirectory()) { + recursiveDelete(f); + } + else { + f.delete(); + } + } + } + file.delete(); + } + } + + public static class TestFtpSessionFactory extends AbstractFtpSessionFactory { private final Collection ftpFiles = new ArrayList(); diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java index a475c6ce73..3a7245666d 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpInboundRemoteFileSystemSynchronizerTests.java @@ -121,6 +121,8 @@ public class SftpInboundRemoteFileSystemSynchronizerTests { localFileListFilter.addFilter(localAcceptOnceFilter); ms.setLocalFilter(localFileListFilter); ms.afterPropertiesSet(); + ms.start(); + Message atestFile = ms.receive(); assertNotNull(atestFile); assertEquals("a.test", atestFile.getPayload().getName()); diff --git a/src/reference/asciidoc/file.adoc b/src/reference/asciidoc/file.adoc index 95b83f9cd4..b777c2bfc5 100644 --- a/src/reference/asciidoc/file.adoc +++ b/src/reference/asciidoc/file.adoc @@ -162,6 +162,13 @@ It is used by the internal (`PriorityBlockingQueue`) to reorder its content acco Therefore, to process files in a specific order, you should provide a comparator to the `FileReadingMessageSource`, rather than ordering the list produced by a custom `DirectoryScanner`. +Starting with _version 5.0_, a new `RecursiveDirectoryScanner` is presented to perform file tree visiting. +The implementation is based on the `Files.walk(Path start, int maxDepth, FileVisitOption... options)` functionality. +The root directory (`DirectoryScanner.listFiles(File)` argument) is excluded from the result. +All other sub-directories includes/excludes are based on the target `FileListFilter` implementation. +For example the `SimplePatternFileListFilter` filters directories by default. +See `AbstractDirectoryAwareFileListFilter` and its implementations for more information. + [[file-namespace-support]] ==== Namespace Support diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 58967df2ab..4c7190301c 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -389,8 +389,7 @@ This will work for any `ResettableFileListFilter`. remote-directory-expression="'/sftpSource'" local-directory="file:myLocalDir" auto-create-local-directory="true" - filename-pattern="*.txt" - local-filter="acceptOnceFilter"> + filename-pattern="*.txt"> @@ -400,13 +399,22 @@ This will work for any `ResettableFileListFilter`. class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" /> - + ---- +Starting with _version 5.0_, the Inbound Channel Adapter can build sub-directories locally according the generated local file name. +That can be a remote sub-path as well. +To be able to read local directory recursively for modification according the hierarchy support, an internal `FileReadingMessageSource` now have been switched to use a new `RecursiveDirectoryScanner` based on the `Files.walk()` algorithm. +Also the `AbstractInboundFileSynchronizingMessageSource` can now be switched to the `WatchService` -based `DirectoryScanner` via `setUseWatchService()` option. +It is also configured for all the `WatchEventType` s to react for any modifications in local directory. +The reprocessing sample above is based on the build-in functionality of the `FileReadingMessageSource.WatchServiceDirectoryScanner` to perform `ResettableFileListFilter.remove()` when the file is deleted (`StandardWatchEventKinds.ENTRY_DELETE`) from the local directory. +See <> for more information. + + ==== Configuring with Java Configuration The following Spring Boot application provides an example of configuring the inbound adapter using Java configuration: diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index 59386e9fc6..2b4afa4c8c 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -427,8 +427,7 @@ This will work for any `ResettableFileListFilter`. remote-directory-expression="'/sftpSource'" local-directory="file:myLocalDir" auto-create-local-directory="true" - filename-pattern="*.txt" - local-filter="acceptOnceFilter"> + filename-pattern="*.txt"> @@ -438,13 +437,21 @@ This will work for any `ResettableFileListFilter`. class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" /> - + ---- +Starting with _version 5.0_, the Inbound Channel Adapter can build sub-directories locally according the generated local file name. +That can be a remote sub-path as well. +To be able to read local directory recursively for modification according the hierarchy support, an internal `FileReadingMessageSource` now have been switched to use a new `RecursiveDirectoryScanner` based on the `Files.walk()` algorithm. +Also the `AbstractInboundFileSynchronizingMessageSource` can now be switched to the `WatchService` -based `DirectoryScanner` via `setUseWatchService()` option. +It is also configured for all the `WatchEventType` s to react for any modifications in local directory. +The reprocessing sample above is based on the build-in functionality of the `FileReadingMessageSource.WatchServiceDirectoryScanner` to perform `ResettableFileListFilter.remove()` when the file is deleted (`StandardWatchEventKinds.ENTRY_DELETE`) from the local directory. +See <> for more information. + ==== Configuring with Java Configuration The following Spring Boot application provides an example of configuring the inbound adapter using Java configuration: diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 1fb3285d99..1f9faa2107 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -125,8 +125,12 @@ The (S)FTP streaming inbound channel adapters now add remote file information in The FTP and SFTP outbound channel adapters, as well as `PUT` command of the outbound gateways, now support `InputStream` as `payload`, too. +The inbound channel adapters now can build file tree locally and use a new `RecursiveDirectoryScanner` by default for local directory. +Also these adapters can now be switched to the `WatchService` instead. + See <> and <> for more information. + ==== Integration Properties Since _version 4.3.2_ a new `spring.integration.readOnly.headers` global property has been added to customize the list of headers which should not be copied to a newly created `Message` by the `MessageBuilder`.