Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4283: MarkerFilePresentFileListFilters

Browse Source 
JIRA: https://jira.spring.io/browse/INT-4283

Filter that only passes files where a corresponding "marker" file is also present.

Implementations for file and(S)FTP.

* Polishing - PR Comments

* Missed one
This commit is contained in:
Gary Russell
2017-06-02 19:31:05 -04:00
committed by Artem Bilan
parent c3031a2c7c
commit 2c7bf9271c
15 changed files with 722 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
spring-integration-core/src/test/java/org/springframework/integration/gateway/InnerGatewayWithChainTests-context.xml
View File

@@ -6,64 +6,64 @@
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<int:channel id="errorChannelA"/>
<int:service-activator input-channel="errorChannelA" expression="'ERROR from errorChannelA'"/>
<int:channel id="errorChannelB"/>
<int:service-activator input-channel="errorChannelB" expression="'ERROR from errorChannelB'"/>
<int:gateway id="testGatewayWithErrorChannelA"
<int:service-activator input-channel="errorChannelA" expression="'ERROR from errorChannelA'"/>
<int:channel id="errorChannelB"/>
<int:service-activator input-channel="errorChannelB" expression="'ERROR from errorChannelB'"/>
<int:gateway id="testGatewayWithErrorChannelA"
service-interface="org.springframework.integration.gateway.InnerGatewayWithChainTests.TestGateway"
default-request-channel="requestChannelA"
error-channel="errorChannelA"/>
<int:chain input-channel="requestChannelA">
<int:service-activator expression="1/(payload-5)"/>
<int:gateway request-channel="requestChannelB" error-channel="errorChannelB"/>
</int:chain>
<int:gateway id="testGatewayWithErrorChannelAA"
<int:gateway id="testGatewayWithErrorChannelAA"
service-interface="org.springframework.integration.gateway.InnerGatewayWithChainTests.TestGateway"
default-request-channel="requestChannelAA"
error-channel="errorChannelA"/>
<int:chain input-channel="requestChannelAA">
<int:service-activator expression="1/(payload-5)"/>
<int:gateway request-channel="requestChannelB"/>
</int:chain>
<int:gateway id="testGatewayWithNoErrorChannelAAA"
<int:gateway id="testGatewayWithNoErrorChannelAAA"
service-interface="org.springframework.integration.gateway.InnerGatewayWithChainTests.TestGateway"
default-request-channel="requestChannelAAA"/>
<int:chain input-channel="requestChannelAAA">
<int:service-activator expression="1/(payload-5)"/>
<int:gateway request-channel="requestChannelB"/>
</int:chain>
<int:inbound-channel-adapter id="inboundAdapterDefaultErrorChannel" auto-startup="false" expression="5" channel="requestChannelAAAA">
<int:poller fixed-rate="2000"/>
</int:inbound-channel-adapter>
<int:chain input-channel="requestChannelAAAA">
<int:service-activator expression="1/(payload-5)"/>
<int:gateway request-channel="requestChannelB"/>
</int:chain>
<int:inbound-channel-adapter id="inboundAdapterAssignedErrorChannel" auto-startup="false" expression="5" channel="requestChannelAAAA">
<int:poller fixed-rate="2000" error-channel="assignedErrorChannel"/>
</int:inbound-channel-adapter>
<int:chain input-channel="requestChannelAAAA">
<int:service-activator expression="1/(payload-5)"/>
<int:gateway request-channel="requestChannelB"/>
</int:chain>
<int:publish-subscribe-channel id="assignedErrorChannel"/>
<int:service-activator input-channel="requestChannelB" expression="10/payload"/>
</beans>

142
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilter.java Normal file
View File

@@ -0,0 +1,142 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* A FileListFilter that only passes files matched by one or more {@link FileListFilter}
* if a corresponding marker file is also present to indicate a file transfer is complete.
*
* @author Gary Russell
* @since 5.0
*
*/
public abstract class AbstractMarkerFilePresentFileListFilter<F> implements FileListFilter<F> {
private final Map<FileListFilter<F>, Function<String, String>> filtersAndFunctions = new HashMap<>();
/**
* Construct an instance with a single {@link FileListFilter} and ".complete"
* will be appended to the name of a matched file when looking for the marker file.
* i.e. if a file {@code foo.txt} is matched by the filter this filter will only pass
* "foo.txt" if "foo.txt.complete" is present.
* @param filter the file name filter.
*/
public AbstractMarkerFilePresentFileListFilter(FileListFilter<F> filter) {
this(filter, defaultFileNameFunction(".complete"));
}
/**
* Construct an instance with a single {@link FileListFilter} and a suffix
* that will will be appended to the name of a matched file when looking for the marker
* file. i.e. if a file {@code foo.txt} is matched by the filter and the suffix is
* ".complete", this filter will only pass "foo.txt" if "foo.txt.complete" is present.
* @param filter the file name filter.
* @param suffix the replacement suffix.
*/
public AbstractMarkerFilePresentFileListFilter(FileListFilter<F> filter, String suffix) {
this(filter, defaultFileNameFunction(suffix));
}
/**
* Construct an instance with a single {@link FileListFilter} and a function
* that will be applied to the name of a matched file when looking for the marker
* file. The function returns the name of the marker file to match, or {@code null}
* for never match. If a file {@code foo.txt} is matched by the filter and the
* function returns "foo.txt.complete", this filter will only pass "foo.txt" if
* "foo.txt.complete" is present.
* @param filter the file name filter.
* @param function the function to create the marker file name from the file name.
*/
public AbstractMarkerFilePresentFileListFilter(FileListFilter<F> filter,
Function<String, String> function) {
this(Collections.singletonMap(filter, function));
}
/**
* Construct an instance with a map of {@link FileListFilter} and functions be
* applied to the name of a matched file when looking for the marker file. i.e. if a
* file {@code foo.txt} is matched by one of the filters and the corresponding
* function returns "foo.txt.complete", this filter will only pass "foo.txt" if
* "foo.txt.complete" is present. The function returns the name of the marker file to
* match, or {@code null} for never match. Due to type erasure, we cannot provide a
* constructor taking {@code Map<Filter, suffix}. For convenience, you can use
* {@link #defaultFileNameFunction(String)} to use the default function used by the
* {@link #AbstractMarkerFilePresentFileListFilter(FileListFilter, String)}
* constructor.
* @param filtersAndFunctions the filters and functions.
*/
public AbstractMarkerFilePresentFileListFilter(
Map<FileListFilter<F>, Function<String, String>> filtersAndFunctions) {
this.filtersAndFunctions.putAll(filtersAndFunctions);
}
/**
* The default function used to create the file name for the corresponding marker file.
* Appends a suffix to the file name.
* @param suffix the suffix to append.
* @return the function.
*/
public static Function<String, String> defaultFileNameFunction(final String suffix) {
return s -> s + suffix;
}
@SuppressWarnings("unchecked")
@Override
public List<F> filterFiles(F[] files) {
if (files.length < 2) {
return Collections.emptyList();
}
final Set<String> candidates = Arrays.stream(files)
.map(this::getFilename)
.collect(Collectors.toSet());
List<F> results = new ArrayList<>();
for (F file : files) {
boolean anyMatch = this.filtersAndFunctions.entrySet().stream().anyMatch(entry -> {
F[] fileToCheck = (F[]) Array.newInstance(file.getClass(), 1);
fileToCheck[0] = file;
if (entry.getKey().filterFiles(fileToCheck).size() > 0) {
String markerName = entry.getValue().apply(getFilename(file));
return markerName != null && candidates.contains(markerName);
}
return false;
});
if (anyMatch) {
results.add(file);
}
}
return results;
}
/**
* Return the name of the file represented by this F.
* @param file the file.
* @return the name.
*/
protected abstract String getFilename(F file);
}

2
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2014 the original author or authors.
* Copyright 2002-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

2
spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2014 the original author or authors.
* Copyright 2002-2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

55
spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilter.java Normal file
View File

@@ -0,0 +1,55 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import java.io.File;
import java.util.Map;
import java.util.function.Function;
/**
* File system implementation of {@link AbstractMarkerFilePresentFileListFilter}.
*
* @author Gary Russell
* @since 5.0
*
*/
public class FileSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter<File> {
FileSystemMarkerFilePresentFileListFilter(FileListFilter<File> filter) {
super(filter);
}
FileSystemMarkerFilePresentFileListFilter(FileListFilter<File> filter, String suffix) {
super(filter, suffix);
}
FileSystemMarkerFilePresentFileListFilter(FileListFilter<File> filter,
Function<String, String> function) {
super(filter, function);
}
FileSystemMarkerFilePresentFileListFilter(
Map<FileListFilter<File>, Function<String, String>> filtersAndFunctions) {
super(filtersAndFunctions);
}
@Override
protected String getFilename(File file) {
return file.getName();
}
}

125
spring-integration-file/src/test/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilterTests.java Normal file
View File

@@ -0,0 +1,125 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import org.junit.Test;
/**
* @author Gary Russell
* @since 5.0
*
*/
public class AbstractMarkerFilePresentFileListFilterTests {
@Test
public void testDefault() {
StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter(
new StringSimplePatternFilter("*.txt"));
List<String> filtered = filter.filterFiles(new String[] { "foo.txt", "foo.txt.complete", "bar.txt" });
assertThat(filtered.size(), equalTo(1));
assertThat(filtered.get(0), equalTo("foo.txt"));
}
@Test
public void testSimple() {
StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter(
new StringSimplePatternFilter("*.txt"), ".done");
List<String> filtered = filter.filterFiles(new String[] { "foo.txt", "foo.txt.done", "bar.txt", "baz.txt" });
assertThat(filtered.size(), equalTo(1));
assertThat(filtered.get(0), equalTo("foo.txt"));
}
@Test
public void testCustomFunction() {
StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter(
new StringSimplePatternFilter("*.txt"), s -> "allFilesDone");
List<String> filtered = filter.filterFiles(new String[] { "foo.txt", "bar.txt" });
assertThat(filtered.size(), equalTo(0));
filtered = filter.filterFiles(new String[] { "foo.txt", "bar.txt", "allFilesDone" });
assertThat(filtered.get(0), equalTo("foo.txt"));
assertThat(filtered.get(1), equalTo("bar.txt"));
}
@Test
public void testMulti() {
Map<FileListFilter<String>, Function<String, String>> map = new HashMap<>();
map.put(new StringSimplePatternFilter("*.txt"),
AbstractMarkerFilePresentFileListFilter.defaultFileNameFunction(".done"));
map.put(new StringSimplePatternFilter("*.xml"),
AbstractMarkerFilePresentFileListFilter.defaultFileNameFunction(".complete"));
StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter(map);
List<String> filtered = filter
.filterFiles(new String[] { "foo.txt", "foo.txt.done", "bar.xml", "bar.xml.complete", "baz.txt" });
assertThat(filtered.size(), equalTo(2));
assertThat(filtered.get(0), equalTo("foo.txt"));
assertThat(filtered.get(1), equalTo("bar.xml"));
}
private static class StringSimplePatternFilter extends AbstractSimplePatternFileListFilter<String> {
StringSimplePatternFilter(String pattern) {
super(pattern);
}
@Override
protected String getFilename(String file) {
return file;
}
@Override
protected boolean isDirectory(String file) {
return false;
}
}
private static class StringMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter<String> {
StringMarkerFilePresentFileListFilter(FileListFilter<String> filter) {
super(filter);
}
StringMarkerFilePresentFileListFilter(FileListFilter<String> filter, String suffix) {
super(filter, suffix);
}
StringMarkerFilePresentFileListFilter(FileListFilter<String> filter,
Function<String, String> function) {
super(filter, function);
}
StringMarkerFilePresentFileListFilter(
Map<FileListFilter<String>, Function<String, String>> filtersAndFunctions) {
super(filtersAndFunctions);
}
@Override
protected String getFilename(String file) {
return file;
}
}
}

53
spring-integration-file/src/test/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilterTests.java Normal file
View File

@@ -0,0 +1,53 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.filters;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import java.io.File;
import java.util.List;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/**
* @author Gary Russell
* @since 5.0
*
*/
public class FileSystemMarkerFilePresentFileListFilterTests {
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@Test
public void test() throws Exception {
FileSystemMarkerFilePresentFileListFilter filter = new FileSystemMarkerFilePresentFileListFilter(
new SimplePatternFileListFilter("*.txt"));
File foo = this.folder.newFile("foo.txt");
foo.createNewFile();
assertThat(filter.filterFiles(new File[] { foo }).size(), equalTo(0));
File complete = this.folder.newFile("foo.txt.complete");
complete.createNewFile();
List<File> filtered = filter.filterFiles(new File[] { foo, complete });
assertThat(filtered.size(), equalTo(1));
assertThat(filtered.get(0).getName(), equalTo("foo.txt"));
}
}

59
spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpSystemMarkerFilePresentFileListFilter.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.ftp.filters;
import java.util.Map;
import java.util.function.Function;
import org.apache.commons.net.ftp.FTPFile;
import org.springframework.integration.file.filters.AbstractMarkerFilePresentFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
/**
* FTP implementation of {@link AbstractMarkerFilePresentFileListFilter}.
*
* @author Gary Russell
* @since 5.0
*
*/
public class FtpSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter<FTPFile> {
FtpSystemMarkerFilePresentFileListFilter(FileListFilter<FTPFile> filter) {
super(filter);
}
FtpSystemMarkerFilePresentFileListFilter(FileListFilter<FTPFile> filter, String suffix) {
super(filter, suffix);
}
FtpSystemMarkerFilePresentFileListFilter(FileListFilter<FTPFile> filter,
Function<String, String> function) {
super(filter, function);
}
FtpSystemMarkerFilePresentFileListFilter(
Map<FileListFilter<FTPFile>, Function<String, String>> filtersAndFunctions) {
super(filtersAndFunctions);
}
@Override
protected String getFilename(FTPFile file) {
return file.getName();
}
}

85
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/filters/FtpFileListFilterTests.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.ftp.filters;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertThat;
import java.io.File;
import java.util.List;
import org.apache.commons.net.ftp.FTPFile;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.ftp.FtpTestSupport;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit4.SpringRunner;
/**
* @author Gary Russell
* @since 5.0
*
*/
@RunWith(SpringRunner.class)
@DirtiesContext
public class FtpFileListFilterTests extends FtpTestSupport {
@Autowired
private FtpRemoteFileTemplate template;
@Test
public void testMarkerFile() throws Exception {
FtpSystemMarkerFilePresentFileListFilter filter = new FtpSystemMarkerFilePresentFileListFilter(
new FtpSimplePatternFileListFilter("*.txt"));
FTPFile[] files = template.list("ftpSource");
assertThat(files.length, greaterThan(0));
List<FTPFile> filtered = filter.filterFiles(files);
assertThat(filtered.size(), equalTo(0));
File remoteDir = getSourceRemoteDirectory();
File marker = new File(remoteDir, "ftpSource2.txt.complete");
marker.createNewFile();
files = template.list("ftpSource");
filtered = filter.filterFiles(files);
assertThat(filtered.size(), equalTo(1));
assertThat(filtered.get(0).getName(), equalTo("ftpSource2.txt"));
marker.delete();
}
@Configuration
public static class Config {
@Bean
public SessionFactory<FTPFile> ftpSessionFactory() {
return FtpFileListFilterTests.sessionFactory();
}
@Bean
public FtpRemoteFileTemplate remoteFileTempalte() {
return new FtpRemoteFileTemplate(ftpSessionFactory());
}
}
}

59
spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpSystemMarkerFilePresentFileListFilter.java Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.filters;
import java.util.Map;
import java.util.function.Function;
import org.springframework.integration.file.filters.AbstractMarkerFilePresentFileListFilter;
import org.springframework.integration.file.filters.FileListFilter;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* SFTP implementation of {@link AbstractMarkerFilePresentFileListFilter}.
*
* @author Gary Russell
* @since 5.0
*
*/
public class SftpSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter<LsEntry> {
SftpSystemMarkerFilePresentFileListFilter(FileListFilter<LsEntry> filter) {
super(filter);
}
SftpSystemMarkerFilePresentFileListFilter(FileListFilter<LsEntry> filter, String suffix) {
super(filter, suffix);
}
SftpSystemMarkerFilePresentFileListFilter(FileListFilter<LsEntry> filter,
Function<String, String> function) {
super(filter, function);
}
SftpSystemMarkerFilePresentFileListFilter(
Map<FileListFilter<LsEntry>, Function<String, String>> filtersAndFunctions) {
super(filtersAndFunctions);
}
@Override
protected String getFilename(LsEntry file) {
return file.getFilename();
}
}

85
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/filters/SftpFileListFilterTests.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2017 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.sftp.filters;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertThat;
import java.io.File;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.sftp.SftpTestSupport;
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit4.SpringRunner;
import com.jcraft.jsch.ChannelSftp.LsEntry;
/**
* @author Gary Russell
* @since 5.0
*
*/
@RunWith(SpringRunner.class)
@DirtiesContext
public class SftpFileListFilterTests extends SftpTestSupport {
@Autowired
private SftpRemoteFileTemplate template;
@Test
public void testMarkerFile() throws Exception {
SftpSystemMarkerFilePresentFileListFilter filter = new SftpSystemMarkerFilePresentFileListFilter(
new SftpSimplePatternFileListFilter("*.txt"));
LsEntry[] files = template.list("sftpSource");
assertThat(files.length, greaterThan(0));
List<LsEntry> filtered = filter.filterFiles(files);
assertThat(filtered.size(), equalTo(0));
File remoteDir = getSourceRemoteDirectory();
File marker = new File(remoteDir, "sftpSource2.txt.complete");
marker.createNewFile();
files = template.list("sftpSource");
filtered = filter.filterFiles(files);
assertThat(filtered.size(), equalTo(1));
assertThat(filtered.get(0).getFilename(), equalTo("sftpSource2.txt"));
marker.delete();
}
@Configuration
public static class Config {
@Bean
public SessionFactory<LsEntry> sftpSessionFactory() {
return SftpFileListFilterTests.sessionFactory();
}
@Bean
public SftpRemoteFileTemplate remoteFileTempalte() {
return new SftpRemoteFileTemplate(sftpSessionFactory());
}
}
}

15
src/reference/asciidoc/file.adoc
View File

@@ -523,6 +523,21 @@ The file will be reopened for each chunk (the default is to keep the file open).
IMPORTANT: Specifying the `delay`, `end` or `reopen` attributes, forces the use of the Apache commons-io adapter and the `native-options` attribute is not allowed.
[[file-incomplete]]
==== Dealing With Incomplete Data
A common problem in file transfer scenarios is how to determine that the transfer is complete, so you don't start reading an incomplete file.
A common technique to solve this problem is to write the file with a temporary name and then atomically rename it to the final name.
This, together with a filter that masks the temporary file from being picked up by the consumer provides a robust solution.
This technique is used by Spring Integration components that write files (locally or remotely); by default, they append `.writing` to the file name and remove it when the transfer is complete.
Another common technique is to write a second "marker" file to indicate the file transfer is complete.
In this scenario, say, you should not consider `foo.txt` to be available for use until `foo.txt.complete` is also present.
Spring Integration _version 5.0_ introduces new filters to support this mechanism.
Implementations are provided for the file system (`FileSystemMarkerFilePresentFileListFilter`), <<ftp-incomplete, FTP>> and <<sftp-incomplete, SFTP>>.
They are configurable such that the marker file can have any name, although it will usually be related to the file being transferred.
See the javadocs for more information.
[[file-writing]]
=== Writing files

8
src/reference/asciidoc/ftp.adoc
View File

@@ -509,6 +509,14 @@ public class FtpJavaApplication {
}
----
[[ftp-incomplete]]
==== Dealing With Incomplete Data
See <<file-incomplete>>.
The `FtpSystemMarkerFilePresentFileListFilter` is provided to filter remote files that don't have a corresponding marker file on the remote system.
See the javadocs for configuration information.
[[ftp-streaming]]
=== FTP Streaming Inbound Channel Adapter

8
src/reference/asciidoc/sftp.adoc
View File

@@ -554,6 +554,14 @@ public class SftpJavaApplication {
}
----
[[sftp-incomplete]]
==== Dealing With Incomplete Data
See <<file-incomplete>>.
The `SftpSystemMarkerFilePresentFileListFilter` is provided to filter remote files that don't have the corresponding marker file on the remote system.
See the javadocs for configuration information.
[[sftp-streaming]]
=== SFTP Streaming Inbound Channel Adapter

3
src/reference/asciidoc/whats-new.adoc
View File

@@ -132,6 +132,8 @@ See <<file-writing-destination-exists>> for more information.
They also now support setting file permissions on the newly written file.
See <<file-permissions>> for more information.
A new `FileSystemMarkerFilePresentFileListFilter` is now available; see <<file-incomplete>> for more information.
==== (S)FTP Changes
The inbound channel adapters now have a property `max-fetch-size` which is used to limit the number of files fetched during a poll when there are no files currently in the local directory.
@@ -159,6 +161,7 @@ The `RemoteFileTemplate` is supplied now with the `invoke(OperationsCallback<F,
See <<ftp>> and <<sftp>> for more information.
New filters for detecting incomplete remote files are now provided; see <<ftp-incomplete>> and <<sftp-incomplete>> for more information.
==== Integration Properties