Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4115: (S)FtpPersistentFileFilter by Default

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4115

Apply `(S)FtpPersistentAcceptOnceFileListFilter` for the `(S)FtpInboundFileSynchronizer` by default to avoid cases to sync the same remote files to the local directory again.
Especially when `localFileName` strategy is applied and we end up with new local files, but with the same remote content

Accept `(S)FtpPersistentAcceptOnceFileListFilter` for streaming adapters

Make `doSetFilter()` as `protected final`

Fix tests after rebase

Fix "What's New" after rebase

Compose `PersistentAcceptOnceFileListFilter` together with the regex or pattern filters
Document such a behavior

Address PR comments for formatting and typo
This commit is contained in:
Artem Bilan
2017-02-06 16:04:47 -05:00
committed by Gary Russell
parent d3fb8b8f9e
commit 10ce68d3e3
29 changed files with 328 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2012 the original author or authors.
* Copyright 2002-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,8 +17,10 @@
package org.springframework.integration.ftp.config;
import org.springframework.integration.file.config.AbstractRemoteFileInboundChannelAdapterParser;
import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizer;
@@ -29,6 +31,8 @@ import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizingMe
*
* @author Mark Fisher
* @author Gary Russell
* @author Artem Bilan
*
* @since 2.0
*/
public class FtpInboundChannelAdapterParser extends AbstractRemoteFileInboundChannelAdapterParser {
@@ -53,4 +57,9 @@ public class FtpInboundChannelAdapterParser extends AbstractRemoteFileInboundCha
return FtpRegexPatternFileListFilter.class;
}
@Override
protected Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass() {
return FtpPersistentAcceptOnceFileListFilter.class;
}
}

11
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpStreamingInboundChannelAdapterParser.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2016 the original author or authors.
* Copyright 2016-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,8 +18,10 @@ package org.springframework.integration.ftp.config;
import org.springframework.integration.core.MessageSource;
import org.springframework.integration.file.config.AbstractRemoteFileStreamingInboundChannelAdapterParser;
import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.RemoteFileOperations;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
import org.springframework.integration.ftp.inbound.FtpStreamingMessageSource;
@@ -27,6 +29,8 @@ import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
/**
* @author Gary Russell
* @author Artem Bilan
*
* @since 4.3
*
*/
@@ -52,4 +56,9 @@ public class FtpStreamingInboundChannelAdapterParser extends AbstractRemoteFileS
return FtpRegexPatternFileListFilter.class;
}
@Override
protected Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass() {
return FtpPersistentAcceptOnceFileListFilter.class;
}
}

19
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpInboundChannelAdapterSpec.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2014-2016 the original author or authors.
* Copyright 2014-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,17 +22,22 @@ import java.util.Comparator;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.dsl.RemoteFileInboundChannelAdapterSpec;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizer;
import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizingMessageSource;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
* A {@link RemoteFileInboundChannelAdapterSpec} for a
* {@link FtpInboundFileSynchronizingMessageSource}.
*
* @author Artem Bilan
*
* @since 5.0
*/
public class FtpInboundChannelAdapterSpec
@@ -52,7 +57,7 @@ public class FtpInboundChannelAdapterSpec
*/
@Override
public FtpInboundChannelAdapterSpec patternFilter(String pattern) {
return filter(new FtpSimplePatternFileListFilter(pattern));
return filter(composeFilters(new FtpSimplePatternFileListFilter(pattern)));
}
/**
@@ -63,7 +68,15 @@ public class FtpInboundChannelAdapterSpec
*/
@Override
public FtpInboundChannelAdapterSpec regexFilter(String regex) {
return filter(new FtpRegexPatternFileListFilter(regex));
return filter(composeFilters(new FtpRegexPatternFileListFilter(regex)));
}
@SuppressWarnings("unchecked")
private CompositeFileListFilter<FTPFile> composeFilters(FileListFilter<FTPFile> fileListFilter) {
CompositeFileListFilter<FTPFile> compositeFileListFilter = new CompositeFileListFilter<>();
compositeFileListFilter.addFilters(fileListFilter,
new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "ftpMessageSource"));
return compositeFileListFilter;
}
}

18
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpStreamingInboundChannelAdapterSpec.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2014-2016 the original author or authors.
* Copyright 2014-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,11 +21,15 @@ import java.util.Comparator;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.dsl.RemoteFileStreamingInboundChannelAdapterSpec;
import org.springframework.integration.file.filters.CompositeFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
import org.springframework.integration.ftp.inbound.FtpStreamingMessageSource;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
* A {@link RemoteFileStreamingInboundChannelAdapterSpec} for a
@@ -51,7 +55,7 @@ public class FtpStreamingInboundChannelAdapterSpec
*/
@Override
public FtpStreamingInboundChannelAdapterSpec patternFilter(String pattern) {
return filter(new FtpSimplePatternFileListFilter(pattern));
return filter(composeFilters(new FtpSimplePatternFileListFilter(pattern)));
}
/**
@@ -62,7 +66,15 @@ public class FtpStreamingInboundChannelAdapterSpec
*/
@Override
public FtpStreamingInboundChannelAdapterSpec regexFilter(String regex) {
return filter(new FtpRegexPatternFileListFilter(regex));
return filter(composeFilters(new FtpRegexPatternFileListFilter(regex)));
}
@SuppressWarnings("unchecked")
private CompositeFileListFilter<FTPFile> composeFilters(FileListFilter<FTPFile> fileListFilter) {
CompositeFileListFilter<FTPFile> compositeFileListFilter = new CompositeFileListFilter<>();
compositeFileListFilter.addFilters(fileListFilter,
new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "ftpStreamingMessageSource"));
return compositeFileListFilter;
}
}

8
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpInboundFileSynchronizer.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2016 the original author or authors.
* Copyright 2002-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,6 +22,8 @@ import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.file.remote.session.Session;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
* An implementation of {@link AbstractInboundFileSynchronizer} for FTP.
@@ -36,12 +38,12 @@ public class FtpInboundFileSynchronizer extends AbstractInboundFileSynchronizer<
/**
* Create a synchronizer with the {@link SessionFactory} used to acquire {@link Session} instances.
*
* @param sessionFactory The session factory.
*/
public FtpInboundFileSynchronizer(SessionFactory<FTPFile> sessionFactory) {
super(sessionFactory);
setRemoteDirectoryExpression(new LiteralExpression(null));
doSetRemoteDirectoryExpression(new LiteralExpression(null));
doSetFilter(new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "ftpMessageSource"));
}
@Override

9
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2016 the original author or authors.
* Copyright 2016-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -26,12 +26,16 @@ import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.remote.AbstractFileInfo;
import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
import org.springframework.integration.file.remote.RemoteFileTemplate;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.session.FtpFileInfo;
import org.springframework.integration.metadata.SimpleMetadataStore;
/**
* Message source for streaming FTP remote file contents.
*
* @author Gary Russell
* @author Artem Bilan
*
* @since 4.3
*
*/
@@ -42,7 +46,7 @@ public class FtpStreamingMessageSource extends AbstractRemoteFileStreamingMessag
* @param template the template.
*/
public FtpStreamingMessageSource(RemoteFileTemplate<FTPFile> template) {
super(template, null);
this(template, null);
}
/**
@@ -55,6 +59,7 @@ public class FtpStreamingMessageSource extends AbstractRemoteFileStreamingMessag
public FtpStreamingMessageSource(RemoteFileTemplate<FTPFile> template,
Comparator<AbstractFileInfo<FTPFile>> comparator) {
super(template, comparator);
doSetFilter(new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "ftpStreamingMessageSource"));
}
@Override