Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3557: Add maxDepth, dirPredicate to FileReadMS (#8596)

Browse Source 
* GH-3557: Add maxDepth, dirPredicate to FileReadMS

Fixes https://github.com/spring-projects/spring-integration/issues/3557

* Expose a `watchMaxDepth` on the `FileReadingMessageSource` for its `Files.walkFileTree()` API usage
* Add `watchDirPredicate` option ot the `FileReadingMessageSource` to skip sub-tree for `Files.walkFileTree()`
scanning according to some condition against directory `Path`

* Fix language in docs

Co-authored-by: Gary Russell <grussell@vmware.com>

---------

Co-authored-by: Gary Russell <grussell@vmware.com>
This commit is contained in:
Artem Bilan
2023-04-13 17:03:39 -04:00
committed by GitHub
parent aaaa489f0c
commit 053cc00484
5 changed files with 156 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
spring-integration-file/src/main/java/org/springframework/integration/file/FileReadingMessageSource.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
@@ -29,6 +30,7 @@ import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedHashSet;
@@ -39,6 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.regex.Matcher;
import org.springframework.context.Lifecycle;
@@ -122,6 +125,10 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
private WatchEventType[] watchEvents = {WatchEventType.CREATE};
private int watchMaxDepth = Integer.MAX_VALUE;
private Predicate<Path> watchDirPredicate = path -> true;
/**
* Create a FileReadingMessageSource with a naturally ordered queue of unbounded capacity.
*/
@@ -237,15 +244,14 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
* Set this flag if you want to make sure the internal queue is
* refreshed with the latest content of the input directory on each poll.
* <p>
* By default this implementation will empty its queue before looking at the
* By default, this implementation will empty its queue before looking at the
* directory again. In cases where order is relevant it is important to
* consider the effects of setting this flag. The internal
* {@link java.util.concurrent.BlockingQueue} that this class is keeping
* will more likely be out of sync with the file system if this flag is set
* to false, but it will change more often (causing expensive reordering) if it is set to true.
* @param scanEachPoll
* whether or not the component should re-scan (as opposed to not
* rescanning until the entire backlog has been delivered)
* @param scanEachPoll whether the component should re-scan (as opposed to not
* rescanning until the entire backlog has been delivered)
*/
public void setScanEachPoll(boolean scanEachPoll) {
this.scanEachPoll = scanEachPoll;
@@ -282,6 +288,28 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
this.watchEvents = Arrays.copyOf(watchEvents, watchEvents.length);
}
/**
* Set a max depth for the {@link Files#walkFileTree(Path, Set, int, FileVisitor)} API when
* {@link #useWatchService} is enabled.
* Defaults to {@link Integer#MAX_VALUE} - walk the whole tree.
* @param watchMaxDepth the depth for {@link Files#walkFileTree(Path, Set, int, FileVisitor)}.
* @since 6.1
*/
public void setWatchMaxDepth(int watchMaxDepth) {
this.watchMaxDepth = watchMaxDepth;
}
/**
* Set a {@link Predicate} to check a directory in the {@link Files#walkFileTree(Path, Set, int, FileVisitor)} call
* if it is eligible for {@link WatchService}.
* @param watchDirPredicate the {@link Predicate} to check dirs for walking.
* @since 6.1
*/
public void setWatchDirPredicate(Predicate<Path> watchDirPredicate) {
Assert.notNull(watchDirPredicate, "'watchDirPredicate' must not be null.");
this.watchDirPredicate = watchDirPredicate;
}
@Override
public String getComponentType() {
return "file:inbound-channel-adapter";
@@ -299,16 +327,16 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
() -> "Source path [" + this.directory + "] does not point to a directory.");
Assert.isTrue(this.directory.canRead(),
() -> "Source directory [" + this.directory + "] is not readable.");
if (this.scanner instanceof Lifecycle) {
((Lifecycle) this.scanner).start();
if (this.scanner instanceof Lifecycle lifecycle) {
lifecycle.start();
}
}
}
@Override
public void stop() {
if (this.running.getAndSet(false) && this.scanner instanceof Lifecycle) {
((Lifecycle) this.scanner).stop();
if (this.running.getAndSet(false) && this.scanner instanceof Lifecycle lifecycle) {
lifecycle.stop();
}
}
@@ -418,8 +446,8 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
@Override
public void setFilter(FileListFilter<File> filter) {
if (filter instanceof DiscardAwareFileListFilter) {
((DiscardAwareFileListFilter<File>) filter).addDiscardCallback(this.filesToPoll::add);
if (filter instanceof DiscardAwareFileListFilter<File> discardAwareFileListFilter) {
discardAwareFileListFilter.addDiscardCallback(this.filesToPoll::add);
}
super.setFilter(filter);
}
@@ -505,8 +533,8 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
logger.debug(() -> "Watch event [" + event.kind() + "] for file [" + file + "]");
if (StandardWatchEventKinds.ENTRY_DELETE.equals(event.kind())) {
if (getFilter() instanceof ResettableFileListFilter) {
((ResettableFileListFilter<File>) getFilter()).remove(file);
if (getFilter() instanceof ResettableFileListFilter<File> resettableFileListFilter) {
resettableFileListFilter.remove(file);
}
boolean fileRemoved = files.remove(file);
if (fileRemoved) {
@@ -540,8 +568,8 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
}
this.pathKeys.clear();
if (event.context() != null && event.context() instanceof Path) {
files.addAll(walkDirectory((Path) event.context(), event.kind()));
if (event.context() != null && event.context() instanceof Path path) {
files.addAll(walkDirectory(path, event.kind()));
}
else {
files.addAll(walkDirectory(FileReadingMessageSource.this.directory.toPath(), event.kind()));
@@ -552,25 +580,32 @@ public class FileReadingMessageSource extends AbstractMessageSource<File> implem
final Set<File> walkedFiles = new LinkedHashSet<>();
try {
registerWatch(directory);
Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
Files.walkFileTree(directory, Collections.emptySet(), FileReadingMessageSource.this.watchMaxDepth,
new SimpleFileVisitor<>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
FileVisitResult fileVisitResult = super.preVisitDirectory(dir, attrs);
registerWatch(dir);
return fileVisitResult;
}
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
FileVisitResult fileVisitResult = super.visitFile(file, attrs);
if (!StandardWatchEventKinds.ENTRY_MODIFY.equals(kind)) {
walkedFiles.add(file.toFile());
}
return fileVisitResult;
}
if (FileReadingMessageSource.this.watchDirPredicate.test(dir)) {
registerWatch(dir);
return FileVisitResult.CONTINUE;
}
else {
return FileVisitResult.SKIP_SUBTREE;
}
}
});
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
FileVisitResult fileVisitResult = super.visitFile(file, attrs);
if (!StandardWatchEventKinds.ENTRY_MODIFY.equals(kind)) {
walkedFiles.add(file.toFile());
}
return fileVisitResult;
}
});
}
catch (IOException ex) {
logger.error(ex, () -> "Failed to walk directory: " + directory.toString());

30
spring-integration-file/src/main/java/org/springframework/integration/file/dsl/FileInboundChannelAdapterSpec.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2016-2021 the original author or authors.
* Copyright 2016-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,10 +17,12 @@
package org.springframework.integration.file.dsl;
import java.io.File;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Comparator;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Predicate;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.integration.dsl.ComponentsRegistration;
@@ -276,6 +278,32 @@ public class FileInboundChannelAdapterSpec
return this;
}
/**
* Set a depth for files walk API.
* @param watchMaxDepth the depth for files walk.
* @return the spec.
* @since 6.1
* @see #useWatchService
* @see FileReadingMessageSource#setWatchMaxDepth(int)
*/
public FileInboundChannelAdapterSpec watchMaxDepth(int watchMaxDepth) {
target.setWatchMaxDepth(watchMaxDepth);
return this;
}
/**
* Set a {@link Predicate} to check if it is eligible for {@link java.nio.file.WatchService}.
* @param watchDirPredicate the {@link Predicate} to check dirs for walking.
* @return the spec.
* @since 6.1
* @see #useWatchService
* @see FileReadingMessageSource#setWatchDirPredicate(Predicate)
*/
public FileInboundChannelAdapterSpec watchDirPredicate(Predicate<Path> watchDirPredicate) {
target.setWatchDirPredicate(watchDirPredicate);
return this;
}
@Override
public Map<Object, String> getComponentsToRegister() {
if (this.scanner == null || this.filtersSet) {

92
spring-integration-file/src/test/java/org/springframework/integration/file/WatchServiceDirectoryScannerTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2015-2022 the original author or authors.
* Copyright 2015-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,10 +25,9 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.integration.file.filters.ChainFileListFilter;
@@ -44,13 +43,14 @@ import static org.mockito.Mockito.mock;
/**
* @author Gary Russell
* @author Artem Bilan
*
* @since 4.2
*
*/
public class WatchServiceDirectoryScannerTests {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@TempDir
public File rootDir;
private File foo;
@@ -62,24 +62,33 @@ public class WatchServiceDirectoryScannerTests {
private File bar1;
@Before
private File skipped;
private File skippedFile;
@BeforeEach
public void setUp() throws IOException {
this.foo = this.folder.newFolder("foo");
this.bar = this.folder.newFolder("bar");
this.top1 = this.folder.newFile();
this.foo = new File(rootDir, "foo");
this.foo.mkdir();
this.bar = new File(rootDir, "bar");
this.bar.mkdir();
this.top1 = File.createTempFile("tmp", null, this.rootDir);
this.foo1 = File.createTempFile("foo", ".txt", this.foo);
this.bar1 = File.createTempFile("bar", ".txt", this.bar);
this.skipped = new File(rootDir, "skipped");
this.skipped.mkdir();
this.skippedFile = File.createTempFile("skippedFile", null, this.skipped);
}
@Test
@SuppressWarnings("unchecked")
public void testWatchServiceDirectoryScanner() throws Exception {
FileReadingMessageSource fileReadingMessageSource = new FileReadingMessageSource();
fileReadingMessageSource.setDirectory(folder.getRoot());
fileReadingMessageSource.setDirectory(this.rootDir);
fileReadingMessageSource.setUseWatchService(true);
fileReadingMessageSource.setWatchEvents(FileReadingMessageSource.WatchEventType.CREATE,
FileReadingMessageSource.WatchEventType.MODIFY,
FileReadingMessageSource.WatchEventType.DELETE);
fileReadingMessageSource.setWatchDirPredicate(path -> !path.getFileName().toString().equals("skipped"));
fileReadingMessageSource.setBeanFactory(mock(BeanFactory.class));
final CountDownLatch removeFileLatch = new CountDownLatch(1);
@@ -107,35 +116,36 @@ public class WatchServiceDirectoryScannerTests {
assertThat(scanner.getClass().getName()).contains("FileReadingMessageSource$WatchServiceDirectoryScanner");
// Files are skipped by the LastModifiedFileListFilter
List<File> files = scanner.listFiles(folder.getRoot());
assertThat(files.size()).isEqualTo(0);
List<File> files = scanner.listFiles(this.rootDir);
assertThat(files).hasSize(0);
// Consider all the files as one day old
fileLastModifiedFileListFilter.setAge(-60 * 60 * 24);
files = scanner.listFiles(folder.getRoot());
assertThat(files.size()).isEqualTo(3);
assertThat(files.contains(top1)).isTrue();
assertThat(files.contains(foo1)).isTrue();
assertThat(files.contains(bar1)).isTrue();
files = scanner.listFiles(this.rootDir);
assertThat(files).hasSize(3);
assertThat(files).contains(top1);
assertThat(files).contains(foo1);
assertThat(files).contains(bar1);
assertThat(files).doesNotContain(this.skippedFile);
fileReadingMessageSource.start();
File top2 = this.folder.newFile();
File top2 = File.createTempFile("tmp", null, this.rootDir);
File foo2 = File.createTempFile("foo", ".txt", this.foo);
File bar2 = File.createTempFile("bar", ".txt", this.bar);
File baz = new File(this.foo, "baz");
baz.mkdir();
File baz1 = File.createTempFile("baz", ".txt", baz);
files = scanner.listFiles(folder.getRoot());
files = scanner.listFiles(this.rootDir);
int n = 0;
Set<File> accum = new HashSet<File>(files);
Set<File> accum = new HashSet<>(files);
while (n++ < 300 && accum.size() != 4) {
Thread.sleep(100);
files = scanner.listFiles(folder.getRoot());
files = scanner.listFiles(this.rootDir);
accum.addAll(files);
}
assertThat(accum.size()).isEqualTo(4);
assertThat(accum.contains(top2)).isTrue();
assertThat(accum.contains(foo2)).isTrue();
assertThat(accum.contains(bar2)).isTrue();
assertThat(accum.contains(baz1)).isTrue();
assertThat(accum).hasSize(4);
assertThat(accum).contains(top2);
assertThat(accum).contains(foo2);
assertThat(accum).contains(bar2);
assertThat(accum).contains(baz1);
/*See AbstractWatchKey#signalEvent source code:
if(var5 >= 512) {
@@ -143,35 +153,33 @@ public class WatchServiceDirectoryScannerTests {
}
*/
fileReadingMessageSource.start();
List<File> filesForOverflow = new ArrayList<File>(600);
List<File> filesForOverflow = new ArrayList<>(600);
for (int i = 0; i < 600; i++) {
filesForOverflow.add(this.folder.newFile("" + i));
filesForOverflow.add(File.createTempFile("tmp" + i, null, this.rootDir));
}
n = 0;
while (n++ < 300 && accum.size() < 604) {
Thread.sleep(100);
files = scanner.listFiles(folder.getRoot());
files = scanner.listFiles(this.rootDir);
accum.addAll(files);
}
assertThat(accum.size()).isEqualTo(604);
assertThat(accum).hasSize(604);
for (File fileForOverFlow : filesForOverflow) {
accum.contains(fileForOverFlow);
}
assertThat(accum).containsAll(filesForOverflow);
File baz2 = File.createTempFile("baz2", ".txt", baz);
n = 0;
while (n++ < 300 && accum.size() < 605) {
Thread.sleep(100);
files = scanner.listFiles(folder.getRoot());
files = scanner.listFiles(this.rootDir);
accum.addAll(files);
}
assertThat(accum.contains(baz2)).isTrue();
assertThat(accum).contains(baz2);
File baz2Copy = new File(baz2.getAbsolutePath());
@@ -181,12 +189,12 @@ public class WatchServiceDirectoryScannerTests {
files.clear();
while (n++ < 300 && files.size() < 1) {
Thread.sleep(100);
files = scanner.listFiles(folder.getRoot());
files = scanner.listFiles(this.rootDir);
accum.addAll(files);
}
assertThat(files.size()).isEqualTo(1);
assertThat(files.contains(baz2)).isTrue();
assertThat(files).hasSize(1);
assertThat(files).contains(baz2);
baz2.delete();
@@ -194,7 +202,7 @@ public class WatchServiceDirectoryScannerTests {
n = 0;
while (n++ < 300 && removeFileLatch.getCount() > 0) {
Thread.sleep(100);
scanner.listFiles(folder.getRoot());
scanner.listFiles(this.rootDir);
}
assertThat(removeFileLatch.await(10, TimeUnit.SECONDS)).isTrue();

5
src/reference/asciidoc/file.adoc
View File

@@ -386,6 +386,11 @@ The following example shows how to configure different logic for create and modi
----
====
Starting with version 6.1, the `FileReadingMessageSource` exposes two new `WatchService`-related options:
* `watchMaxDepth` - an argument for the `Files.walkFileTree(Path root, Set attributes, int maxDepth, FileVisitor visitor)` API;
* `watchDirPredicate` - a `Predicate<Path>` to test if a directory in the scanned tree should be walked and registered with the `WatchService` and the configured watch event kinds.
==== Limiting Memory Consumption
You can use a `HeadDirectoryScanner` to limit the number of files retained in memory.

6
src/reference/asciidoc/whats-new.adoc
View File

@@ -63,3 +63,9 @@ See <<./jms.adoc#jms-inbound-gateway, JMS Inbound Gateway>> for more information
The (previously deprecated) `ImapIdleChannelAdapter.sendingTaskExecutor` property has been removed in favor of an asynchronous message process downstream in the flow.
See <<./mail.adoc#mail-inbound, Mail-receiving Channel Adapter>> for more information.
[[x6.1-file]]
=== Files Changes
The `FileReadingMessageSource` now exposes `watchMaxDepth` and `watchDirPredicate` options for the `WatchService`.
See <<./file.adoc#watch-service-directory-scanner, `WatchServiceDirectoryScanner`>> for more information.