Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-2777: Remote File Filter Improvements

Browse Source 
Resolves https://github.com/spring-projects/spring-integration/issues/2777

If the filter supports it, defer filtering until the last possible moment.
Then, the worst case scenario after a catastrophic failure (e.g. power loss),
would be that at most one file will be incorrectly filtered on restart.

Polishing and add more tests.

Polishing Javadocs

More Polishing

Final polishing

More polishing.

Polishing and docPolishing and docs.

* Fix typos in Docs
This commit is contained in:
Gary Russell
2019-03-01 17:29:32 -05:00
committed by Artem Bilan
parent bb62cb8471
commit 931df86274
21 changed files with 449 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
spring-integration-file/src/test/java/org/springframework/integration/file/filters/CompositeFileListFilterTests.java
View File

@@ -17,6 +17,7 @@
package org.springframework.integration.file.filters;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -24,6 +25,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -101,4 +103,59 @@ public class CompositeFileListFilterTests {
compositeFileFilter.close();
}
@Test
public void singleFileCapableUO() throws IOException {
CompositeFileListFilter<String> compo =
new CompositeFileListFilter<>(Collections.singletonList(new FileListFilter<String>() {
@Override
public List<String> filterFiles(String[] files) {
return Collections.emptyList();
}
@Override
public boolean supportsSingleFileFiltering() {
return true;
}
}));
assertThatExceptionOfType(UnsupportedOperationException.class).isThrownBy(() -> compo.accept("foo"));
compo.close();
}
@Test
public void singleFileCapable() throws IOException {
CompositeFileListFilter<String> compo =
new CompositeFileListFilter<>(Collections.singletonList(new FileListFilter<String>() {
@Override
public List<String> filterFiles(String[] files) {
return Collections.emptyList();
}
@Override
public boolean supportsSingleFileFiltering() {
return true;
}
@Override
public boolean accept(String file) {
return true;
}
}));
assertThat(compo.accept("foo")).isTrue();
compo.addFilter(s -> null);
assertThat(compo.supportsSingleFileFiltering()).isFalse();
compo.close();
}
@Test
public void notSingleFileCapable() throws IOException {
CompositeFileListFilter<String> compo =
new CompositeFileListFilter<>(Collections.singletonList(s -> null));
assertThat(compo.supportsSingleFileFiltering()).isFalse();
compo.close();
}
}

6
spring-integration-file/src/test/java/org/springframework/integration/file/filters/LastModifiedFileListFilterTests.java
View File

@@ -45,10 +45,12 @@ public class LastModifiedFileListFilterTests {
FileOutputStream fileOutputStream = new FileOutputStream(foo);
fileOutputStream.write("x".getBytes());
fileOutputStream.close();
assertThat(filter.filterFiles(new File[] { foo }).size()).isEqualTo(0);
assertThat(filter.filterFiles(new File[] { foo })).hasSize(0);
assertThat(filter.accept(foo)).isFalse();
// Make a file as of yesterday's
foo.setLastModified(System.currentTimeMillis() - 1000 * 60 * 60 * 24);
assertThat(filter.filterFiles(new File[] { foo }).size()).isEqualTo(1);
assertThat(filter.filterFiles(new File[] { foo })).hasSize(1);
assertThat(filter.accept(foo)).isTrue();
}
}

44
spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java
View File

@@ -35,6 +35,8 @@ import java.util.Comparator;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.Test;
@@ -45,6 +47,7 @@ import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.file.splitter.FileSplitter;
@@ -65,13 +68,33 @@ public class StreamingInboundTests {
private final StreamTransformer transformer = new StreamTransformer();
@SuppressWarnings("unchecked")
@Test
public void testAllData() throws Exception {
public void testAllDataNoFilter() throws IOException {
testAllData(null, true);
}
@Test
public void testAllDataSingleCapableFilter() throws IOException {
testAllData(null, false);
}
@Test
public void testAllDataBulkOnlyFilter() throws IOException {
testAllData(fs -> Stream.of(fs).collect(Collectors.toList()), false);
}
@SuppressWarnings("unchecked")
private void testAllData(FileListFilter<String> filter, boolean nullFilter) throws IOException {
StringSessionFactory sessionFactory = new StringSessionFactory();
Streamer streamer = new Streamer(new StringRemoteFileTemplate(sessionFactory), null);
streamer.setBeanFactory(mock(BeanFactory.class));
streamer.setRemoteDirectory("/foo");
if (filter != null) {
streamer.setFilter(filter);
}
if (nullFilter) {
streamer.setFilter(null);
}
streamer.afterPropertiesSet();
streamer.start();
Message<byte[]> received = (Message<byte[]>) this.transformer.transform(streamer.receive());
@@ -116,7 +139,6 @@ public class StreamingInboundTests {
Streamer streamer = new Streamer(new StringRemoteFileTemplate(sessionFactory), null);
streamer.setBeanFactory(mock(BeanFactory.class));
streamer.setRemoteDirectory("/foo");
streamer.setMaxFetchSize(1);
streamer.setFilter(new AcceptOnceFileListFilter<>());
streamer.afterPropertiesSet();
streamer.start();
@@ -133,10 +155,10 @@ public class StreamingInboundTests {
assertThat(received.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)).isEqualTo("/foo");
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE)).isEqualTo("bar");
// close after list, transform
verify(new IntegrationMessageHeaderAccessor(received).getCloseableResource(), times(4)).close();
// close after transform
verify(new IntegrationMessageHeaderAccessor(received).getCloseableResource(), times(3)).close();
verify(sessionFactory.getSession(), times(2)).list("/foo");
verify(sessionFactory.getSession()).list("/foo");
}
@Test
@@ -204,10 +226,9 @@ public class StreamingInboundTests {
streamer.start();
assertThat(streamer.receive()).isNotNull();
assertThat(TestUtils.getPropertyValue(streamer, "toBeReceived", BlockingQueue.class)).hasSize(1);
assertThat(streamer.metadataMap).hasSize(2);
assertThat(streamer.metadataMap).hasSize(1);
streamer.stop();
assertThat(TestUtils.getPropertyValue(streamer, "toBeReceived", BlockingQueue.class)).hasSize(0);
assertThat(streamer.metadataMap).hasSize(1);
}
@SuppressWarnings("unchecked")
@@ -221,13 +242,18 @@ public class StreamingInboundTests {
assertThatExceptionOfType(UncheckedIOException.class)
.isThrownBy(streamer::receive);
assertThat(TestUtils.getPropertyValue(streamer, "toBeReceived", BlockingQueue.class)).hasSize(1);
assertThat(streamer.metadataMap).hasSize(1);
assertThat(streamer.metadataMap).hasSize(0);
}
public static class Streamer extends AbstractRemoteFileStreamingMessageSource<String> {
ConcurrentHashMap<String, String> metadataMap = new ConcurrentHashMap<>();
protected Streamer(RemoteFileTemplate<String> template) {
super(template, null);
doSetFilter(null);
}
protected Streamer(RemoteFileTemplate<String> template, Comparator<String> comparator) {
super(template, comparator);
doSetFilter(new StringPersistentFileListFilter(new SimpleMetadataStore(this.metadataMap), "streamer"));

97
spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java
View File

@@ -27,12 +27,15 @@ import java.io.OutputStream;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.junit.Test;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.integration.file.HeadDirectoryScanner;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.integration.file.filters.ChainFileListFilter;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.messaging.MessagingException;
@@ -113,7 +116,7 @@ public class AbstractRemoteFileSynchronizerTests {
}
@Test
public void testMaxFetchSizeSource() throws Exception {
public void testMaxFetchSizeSource() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizer<String> sync = createLimitingSynchronizer(count);
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(sync);
@@ -130,7 +133,62 @@ public class AbstractRemoteFileSynchronizerTests {
}
@Test
public void testExclusiveScanner() throws Exception {
public void testDefaultFilter() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(count);
source.afterPropertiesSet();
source.start();
source.receive();
assertThat(count.get()).isEqualTo(1);
source.receive();
assertThat(count.get()).isEqualTo(2);
source.receive();
assertThat(count.get()).isEqualTo(3);
source.receive();
assertThat(count.get()).isEqualTo(3);
}
@Test
public void testNoFilter() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizer<String> sync = createLimitingSynchronizer(count);
sync.setFilter(null);
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(sync);
source.afterPropertiesSet();
source.start();
source.receive();
assertThat(count.get()).isEqualTo(1);
source.receive();
assertThat(count.get()).isEqualTo(2);
source.receive();
assertThat(count.get()).isEqualTo(3);
source.receive();
assertThat(count.get()).isEqualTo(4);
}
@Test
public void testBulkOnlyFilter() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizer<String> sync = createLimitingSynchronizer(count);
ChainFileListFilter<String> cflf = new ChainFileListFilter<>();
cflf.addFilter(new AcceptOnceFileListFilter<>());
cflf.addFilter(fs -> Stream.of(fs).collect(Collectors.toList()));
sync.setFilter(cflf);
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(sync);
source.afterPropertiesSet();
source.start();
source.receive();
assertThat(count.get()).isEqualTo(1);
source.receive();
assertThat(count.get()).isEqualTo(2);
source.receive();
assertThat(count.get()).isEqualTo(3);
source.receive();
assertThat(count.get()).isEqualTo(3);
}
@Test
public void testExclusiveScanner() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(count);
source.setScanner(new HeadDirectoryScanner(1));
@@ -141,7 +199,7 @@ public class AbstractRemoteFileSynchronizerTests {
}
@Test
public void testExclusiveWatchService() throws Exception {
public void testExclusiveWatchService() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(count);
source.setUseWatchService(true);
@@ -152,7 +210,7 @@ public class AbstractRemoteFileSynchronizerTests {
}
@Test(expected = IllegalStateException.class)
public void testScannerAndWatchServiceConflict() throws Exception {
public void testScannerAndWatchServiceConflict() {
final AtomicInteger count = new AtomicInteger();
AbstractInboundFileSynchronizingMessageSource<String> source = createSource(count);
source.setUseWatchService(true);
@@ -166,6 +224,7 @@ public class AbstractRemoteFileSynchronizerTests {
private AbstractInboundFileSynchronizingMessageSource<String> createSource(
AbstractInboundFileSynchronizer<String> sync) {
AbstractInboundFileSynchronizingMessageSource<String> source =
new AbstractInboundFileSynchronizingMessageSource<String>(sync) {
@@ -204,7 +263,7 @@ public class AbstractRemoteFileSynchronizerTests {
@Override
protected boolean copyFileToLocalDirectory(String remoteDirectoryPath, String remoteFile,
File localDirectory, Session<String> session) throws IOException {
File localDirectory, Session<String> session) {
count.incrementAndGet();
return true;
}
@@ -227,40 +286,44 @@ public class AbstractRemoteFileSynchronizerTests {
private class StringSession implements Session<String> {
StringSession() {
super();
}
@Override
public boolean remove(String path) throws IOException {
public boolean remove(String path) {
return true;
}
@Override
public String[] list(String path) throws IOException {
public String[] list(String path) {
return new String[] { "foo", "bar", "baz" };
}
@Override
public void read(String source, OutputStream outputStream) throws IOException {
public void read(String source, OutputStream outputStream) {
}
@Override
public void write(InputStream inputStream, String destination) throws IOException {
public void write(InputStream inputStream, String destination) {
}
@Override
public void append(InputStream inputStream, String destination) throws IOException {
public void append(InputStream inputStream, String destination) {
}
@Override
public boolean mkdir(String directory) throws IOException {
public boolean mkdir(String directory) {
return true;
}
@Override
public boolean rmdir(String directory) throws IOException {
public boolean rmdir(String directory) {
return true;
}
@Override
public void rename(String pathFrom, String pathTo) throws IOException {
public void rename(String pathFrom, String pathTo) {
}
@Override
@@ -273,22 +336,22 @@ public class AbstractRemoteFileSynchronizerTests {
}
@Override
public boolean exists(String path) throws IOException {
public boolean exists(String path) {
return true;
}
@Override
public String[] listNames(String path) throws IOException {
public String[] listNames(String path) {
return new String[0];
}
@Override
public InputStream readRaw(String source) throws IOException {
public InputStream readRaw(String source) {
return null;
}
@Override
public boolean finalizeRaw() throws IOException {
public boolean finalizeRaw() {
return true;
}