diff --git a/spring-integration-core/src/test/java/org/springframework/integration/gateway/InnerGatewayWithChainTests-context.xml b/spring-integration-core/src/test/java/org/springframework/integration/gateway/InnerGatewayWithChainTests-context.xml index 4c5fd19d1e..b908823cca 100644 --- a/spring-integration-core/src/test/java/org/springframework/integration/gateway/InnerGatewayWithChainTests-context.xml +++ b/spring-integration-core/src/test/java/org/springframework/integration/gateway/InnerGatewayWithChainTests-context.xml @@ -6,64 +6,64 @@ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> - - - - - - - - + + + + + + + - + - - - + - - - + - + - + - + - + - + - - + + - + diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilter.java new file mode 100644 index 0000000000..c32ec5b729 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilter.java @@ -0,0 +1,142 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A FileListFilter that only passes files matched by one or more {@link FileListFilter} + * if a corresponding marker file is also present to indicate a file transfer is complete. + * + * @author Gary Russell + * @since 5.0 + * + */ +public abstract class AbstractMarkerFilePresentFileListFilter implements FileListFilter { + + private final Map, Function> filtersAndFunctions = new HashMap<>(); + + /** + * Construct an instance with a single {@link FileListFilter} and ".complete" + * will be appended to the name of a matched file when looking for the marker file. + * i.e. if a file {@code foo.txt} is matched by the filter this filter will only pass + * "foo.txt" if "foo.txt.complete" is present. + * @param filter the file name filter. + */ + public AbstractMarkerFilePresentFileListFilter(FileListFilter filter) { + this(filter, defaultFileNameFunction(".complete")); + } + + /** + * Construct an instance with a single {@link FileListFilter} and a suffix + * that will will be appended to the name of a matched file when looking for the marker + * file. i.e. if a file {@code foo.txt} is matched by the filter and the suffix is + * ".complete", this filter will only pass "foo.txt" if "foo.txt.complete" is present. + * @param filter the file name filter. + * @param suffix the replacement suffix. + */ + public AbstractMarkerFilePresentFileListFilter(FileListFilter filter, String suffix) { + this(filter, defaultFileNameFunction(suffix)); + } + + /** + * Construct an instance with a single {@link FileListFilter} and a function + * that will be applied to the name of a matched file when looking for the marker + * file. The function returns the name of the marker file to match, or {@code null} + * for never match. If a file {@code foo.txt} is matched by the filter and the + * function returns "foo.txt.complete", this filter will only pass "foo.txt" if + * "foo.txt.complete" is present. + * @param filter the file name filter. + * @param function the function to create the marker file name from the file name. + */ + public AbstractMarkerFilePresentFileListFilter(FileListFilter filter, + Function function) { + this(Collections.singletonMap(filter, function)); + } + + /** + * Construct an instance with a map of {@link FileListFilter} and functions be + * applied to the name of a matched file when looking for the marker file. i.e. if a + * file {@code foo.txt} is matched by one of the filters and the corresponding + * function returns "foo.txt.complete", this filter will only pass "foo.txt" if + * "foo.txt.complete" is present. The function returns the name of the marker file to + * match, or {@code null} for never match. Due to type erasure, we cannot provide a + * constructor taking {@code Map, Function> filtersAndFunctions) { + this.filtersAndFunctions.putAll(filtersAndFunctions); + } + + /** + * The default function used to create the file name for the corresponding marker file. + * Appends a suffix to the file name. + * @param suffix the suffix to append. + * @return the function. + */ + public static Function defaultFileNameFunction(final String suffix) { + return s -> s + suffix; + } + + @SuppressWarnings("unchecked") + @Override + public List filterFiles(F[] files) { + if (files.length < 2) { + return Collections.emptyList(); + } + final Set candidates = Arrays.stream(files) + .map(this::getFilename) + .collect(Collectors.toSet()); + List results = new ArrayList<>(); + for (F file : files) { + boolean anyMatch = this.filtersAndFunctions.entrySet().stream().anyMatch(entry -> { + F[] fileToCheck = (F[]) Array.newInstance(file.getClass(), 1); + fileToCheck[0] = file; + if (entry.getKey().filterFiles(fileToCheck).size() > 0) { + String markerName = entry.getValue().apply(getFilename(file)); + return markerName != null && candidates.contains(markerName); + } + return false; + }); + if (anyMatch) { + results.add(file); + } + } + return results; + } + + /** + * Return the name of the file represented by this F. + * @param file the file. + * @return the name. + */ + protected abstract String getFilename(F file); + +} diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java index 1d846197d2..644cc8fbf4 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractRegexPatternFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java index 757d63f69f..284792efa8 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractSimplePatternFileListFilter.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2014 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilter.java b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilter.java new file mode 100644 index 0000000000..1ddfba46a8 --- /dev/null +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilter.java @@ -0,0 +1,55 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import java.io.File; +import java.util.Map; +import java.util.function.Function; + +/** + * File system implementation of {@link AbstractMarkerFilePresentFileListFilter}. + * + * @author Gary Russell + * @since 5.0 + * + */ +public class FileSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter { + + FileSystemMarkerFilePresentFileListFilter(FileListFilter filter) { + super(filter); + } + + FileSystemMarkerFilePresentFileListFilter(FileListFilter filter, String suffix) { + super(filter, suffix); + } + + FileSystemMarkerFilePresentFileListFilter(FileListFilter filter, + Function function) { + super(filter, function); + } + + FileSystemMarkerFilePresentFileListFilter( + Map, Function> filtersAndFunctions) { + super(filtersAndFunctions); + } + + @Override + protected String getFilename(File file) { + return file.getName(); + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilterTests.java new file mode 100644 index 0000000000..85a230d36a --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/AbstractMarkerFilePresentFileListFilterTests.java @@ -0,0 +1,125 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import org.junit.Test; + +/** + * @author Gary Russell + * @since 5.0 + * + */ +public class AbstractMarkerFilePresentFileListFilterTests { + + @Test + public void testDefault() { + StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter( + new StringSimplePatternFilter("*.txt")); + List filtered = filter.filterFiles(new String[] { "foo.txt", "foo.txt.complete", "bar.txt" }); + assertThat(filtered.size(), equalTo(1)); + assertThat(filtered.get(0), equalTo("foo.txt")); + } + + @Test + public void testSimple() { + StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter( + new StringSimplePatternFilter("*.txt"), ".done"); + List filtered = filter.filterFiles(new String[] { "foo.txt", "foo.txt.done", "bar.txt", "baz.txt" }); + assertThat(filtered.size(), equalTo(1)); + assertThat(filtered.get(0), equalTo("foo.txt")); + } + + @Test + public void testCustomFunction() { + StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter( + new StringSimplePatternFilter("*.txt"), s -> "allFilesDone"); + List filtered = filter.filterFiles(new String[] { "foo.txt", "bar.txt" }); + assertThat(filtered.size(), equalTo(0)); + filtered = filter.filterFiles(new String[] { "foo.txt", "bar.txt", "allFilesDone" }); + assertThat(filtered.get(0), equalTo("foo.txt")); + assertThat(filtered.get(1), equalTo("bar.txt")); + } + + @Test + public void testMulti() { + Map, Function> map = new HashMap<>(); + map.put(new StringSimplePatternFilter("*.txt"), + AbstractMarkerFilePresentFileListFilter.defaultFileNameFunction(".done")); + map.put(new StringSimplePatternFilter("*.xml"), + AbstractMarkerFilePresentFileListFilter.defaultFileNameFunction(".complete")); + StringMarkerFilePresentFileListFilter filter = new StringMarkerFilePresentFileListFilter(map); + List filtered = filter + .filterFiles(new String[] { "foo.txt", "foo.txt.done", "bar.xml", "bar.xml.complete", "baz.txt" }); + assertThat(filtered.size(), equalTo(2)); + assertThat(filtered.get(0), equalTo("foo.txt")); + assertThat(filtered.get(1), equalTo("bar.xml")); + } + + private static class StringSimplePatternFilter extends AbstractSimplePatternFileListFilter { + + StringSimplePatternFilter(String pattern) { + super(pattern); + } + + @Override + protected String getFilename(String file) { + return file; + } + + @Override + protected boolean isDirectory(String file) { + return false; + } + + } + + private static class StringMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter { + + StringMarkerFilePresentFileListFilter(FileListFilter filter) { + super(filter); + } + + StringMarkerFilePresentFileListFilter(FileListFilter filter, String suffix) { + super(filter, suffix); + } + + StringMarkerFilePresentFileListFilter(FileListFilter filter, + Function function) { + super(filter, function); + } + + StringMarkerFilePresentFileListFilter( + Map, Function> filtersAndFunctions) { + super(filtersAndFunctions); + } + + @Override + protected String getFilename(String file) { + return file; + } + + } + +} diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilterTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilterTests.java new file mode 100644 index 0000000000..804566154a --- /dev/null +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/filters/FileSystemMarkerFilePresentFileListFilterTests.java @@ -0,0 +1,53 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.file.filters; + +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.util.List; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * @author Gary Russell + * @since 5.0 + * + */ +public class FileSystemMarkerFilePresentFileListFilterTests { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Test + public void test() throws Exception { + FileSystemMarkerFilePresentFileListFilter filter = new FileSystemMarkerFilePresentFileListFilter( + new SimplePatternFileListFilter("*.txt")); + File foo = this.folder.newFile("foo.txt"); + foo.createNewFile(); + assertThat(filter.filterFiles(new File[] { foo }).size(), equalTo(0)); + File complete = this.folder.newFile("foo.txt.complete"); + complete.createNewFile(); + List filtered = filter.filterFiles(new File[] { foo, complete }); + assertThat(filtered.size(), equalTo(1)); + assertThat(filtered.get(0).getName(), equalTo("foo.txt")); + } + +} diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpSystemMarkerFilePresentFileListFilter.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpSystemMarkerFilePresentFileListFilter.java new file mode 100644 index 0000000000..ad7aa295d5 --- /dev/null +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/filters/FtpSystemMarkerFilePresentFileListFilter.java @@ -0,0 +1,59 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.ftp.filters; + +import java.util.Map; +import java.util.function.Function; + +import org.apache.commons.net.ftp.FTPFile; + +import org.springframework.integration.file.filters.AbstractMarkerFilePresentFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; + +/** + * FTP implementation of {@link AbstractMarkerFilePresentFileListFilter}. + * + * @author Gary Russell + * @since 5.0 + * + */ +public class FtpSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter { + + FtpSystemMarkerFilePresentFileListFilter(FileListFilter filter) { + super(filter); + } + + FtpSystemMarkerFilePresentFileListFilter(FileListFilter filter, String suffix) { + super(filter, suffix); + } + + FtpSystemMarkerFilePresentFileListFilter(FileListFilter filter, + Function function) { + super(filter, function); + } + + FtpSystemMarkerFilePresentFileListFilter( + Map, Function> filtersAndFunctions) { + super(filtersAndFunctions); + } + + @Override + protected String getFilename(FTPFile file) { + return file.getName(); + } + +} diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/filters/FtpFileListFilterTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/filters/FtpFileListFilterTests.java new file mode 100644 index 0000000000..a0873df8d7 --- /dev/null +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/filters/FtpFileListFilterTests.java @@ -0,0 +1,85 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.ftp.filters; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.util.List; + +import org.apache.commons.net.ftp.FTPFile; +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.ftp.FtpTestSupport; +import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +/** + * @author Gary Russell + * @since 5.0 + * + */ +@RunWith(SpringRunner.class) +@DirtiesContext +public class FtpFileListFilterTests extends FtpTestSupport { + + @Autowired + private FtpRemoteFileTemplate template; + + @Test + public void testMarkerFile() throws Exception { + FtpSystemMarkerFilePresentFileListFilter filter = new FtpSystemMarkerFilePresentFileListFilter( + new FtpSimplePatternFileListFilter("*.txt")); + FTPFile[] files = template.list("ftpSource"); + assertThat(files.length, greaterThan(0)); + List filtered = filter.filterFiles(files); + assertThat(filtered.size(), equalTo(0)); + File remoteDir = getSourceRemoteDirectory(); + File marker = new File(remoteDir, "ftpSource2.txt.complete"); + marker.createNewFile(); + files = template.list("ftpSource"); + filtered = filter.filterFiles(files); + assertThat(filtered.size(), equalTo(1)); + assertThat(filtered.get(0).getName(), equalTo("ftpSource2.txt")); + marker.delete(); + } + + @Configuration + public static class Config { + + @Bean + public SessionFactory ftpSessionFactory() { + return FtpFileListFilterTests.sessionFactory(); + } + + @Bean + public FtpRemoteFileTemplate remoteFileTempalte() { + return new FtpRemoteFileTemplate(ftpSessionFactory()); + } + + } + + +} diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpSystemMarkerFilePresentFileListFilter.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpSystemMarkerFilePresentFileListFilter.java new file mode 100644 index 0000000000..ee5b70249c --- /dev/null +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/filters/SftpSystemMarkerFilePresentFileListFilter.java @@ -0,0 +1,59 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.sftp.filters; + +import java.util.Map; +import java.util.function.Function; + +import org.springframework.integration.file.filters.AbstractMarkerFilePresentFileListFilter; +import org.springframework.integration.file.filters.FileListFilter; + +import com.jcraft.jsch.ChannelSftp.LsEntry; + +/** + * SFTP implementation of {@link AbstractMarkerFilePresentFileListFilter}. + * + * @author Gary Russell + * @since 5.0 + * + */ +public class SftpSystemMarkerFilePresentFileListFilter extends AbstractMarkerFilePresentFileListFilter { + + SftpSystemMarkerFilePresentFileListFilter(FileListFilter filter) { + super(filter); + } + + SftpSystemMarkerFilePresentFileListFilter(FileListFilter filter, String suffix) { + super(filter, suffix); + } + + SftpSystemMarkerFilePresentFileListFilter(FileListFilter filter, + Function function) { + super(filter, function); + } + + SftpSystemMarkerFilePresentFileListFilter( + Map, Function> filtersAndFunctions) { + super(filtersAndFunctions); + } + + @Override + protected String getFilename(LsEntry file) { + return file.getFilename(); + } + +} diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/filters/SftpFileListFilterTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/filters/SftpFileListFilterTests.java new file mode 100644 index 0000000000..95f9ba789e --- /dev/null +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/filters/SftpFileListFilterTests.java @@ -0,0 +1,85 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.sftp.filters; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertThat; + +import java.io.File; +import java.util.List; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.sftp.SftpTestSupport; +import org.springframework.integration.sftp.session.SftpRemoteFileTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import com.jcraft.jsch.ChannelSftp.LsEntry; + +/** + * @author Gary Russell + * @since 5.0 + * + */ +@RunWith(SpringRunner.class) +@DirtiesContext +public class SftpFileListFilterTests extends SftpTestSupport { + + @Autowired + private SftpRemoteFileTemplate template; + + @Test + public void testMarkerFile() throws Exception { + SftpSystemMarkerFilePresentFileListFilter filter = new SftpSystemMarkerFilePresentFileListFilter( + new SftpSimplePatternFileListFilter("*.txt")); + LsEntry[] files = template.list("sftpSource"); + assertThat(files.length, greaterThan(0)); + List filtered = filter.filterFiles(files); + assertThat(filtered.size(), equalTo(0)); + File remoteDir = getSourceRemoteDirectory(); + File marker = new File(remoteDir, "sftpSource2.txt.complete"); + marker.createNewFile(); + files = template.list("sftpSource"); + filtered = filter.filterFiles(files); + assertThat(filtered.size(), equalTo(1)); + assertThat(filtered.get(0).getFilename(), equalTo("sftpSource2.txt")); + marker.delete(); + } + + @Configuration + public static class Config { + + @Bean + public SessionFactory sftpSessionFactory() { + return SftpFileListFilterTests.sessionFactory(); + } + + @Bean + public SftpRemoteFileTemplate remoteFileTempalte() { + return new SftpRemoteFileTemplate(sftpSessionFactory()); + } + + } + +} diff --git a/src/reference/asciidoc/file.adoc b/src/reference/asciidoc/file.adoc index 2e1069d45e..65d93a04f9 100644 --- a/src/reference/asciidoc/file.adoc +++ b/src/reference/asciidoc/file.adoc @@ -523,6 +523,21 @@ The file will be reopened for each chunk (the default is to keep the file open). IMPORTANT: Specifying the `delay`, `end` or `reopen` attributes, forces the use of the Apache commons-io adapter and the `native-options` attribute is not allowed. +[[file-incomplete]] +==== Dealing With Incomplete Data + +A common problem in file transfer scenarios is how to determine that the transfer is complete, so you don't start reading an incomplete file. +A common technique to solve this problem is to write the file with a temporary name and then atomically rename it to the final name. +This, together with a filter that masks the temporary file from being picked up by the consumer provides a robust solution. +This technique is used by Spring Integration components that write files (locally or remotely); by default, they append `.writing` to the file name and remove it when the transfer is complete. + +Another common technique is to write a second "marker" file to indicate the file transfer is complete. +In this scenario, say, you should not consider `foo.txt` to be available for use until `foo.txt.complete` is also present. +Spring Integration _version 5.0_ introduces new filters to support this mechanism. +Implementations are provided for the file system (`FileSystemMarkerFilePresentFileListFilter`), <> and <>. +They are configurable such that the marker file can have any name, although it will usually be related to the file being transferred. +See the javadocs for more information. + [[file-writing]] === Writing files diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index b81ae9f27e..37e7bf3456 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -509,6 +509,14 @@ public class FtpJavaApplication { } ---- +[[ftp-incomplete]] +==== Dealing With Incomplete Data + +See <>. + +The `FtpSystemMarkerFilePresentFileListFilter` is provided to filter remote files that don't have a corresponding marker file on the remote system. +See the javadocs for configuration information. + [[ftp-streaming]] === FTP Streaming Inbound Channel Adapter diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index c3a03378ef..1bc9a23ff5 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -554,6 +554,14 @@ public class SftpJavaApplication { } ---- +[[sftp-incomplete]] +==== Dealing With Incomplete Data + +See <>. + +The `SftpSystemMarkerFilePresentFileListFilter` is provided to filter remote files that don't have the corresponding marker file on the remote system. +See the javadocs for configuration information. + [[sftp-streaming]] === SFTP Streaming Inbound Channel Adapter diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 5af6167ece..5703cab4b0 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -132,6 +132,8 @@ See <> for more information. They also now support setting file permissions on the newly written file. See <> for more information. +A new `FileSystemMarkerFilePresentFileListFilter` is now available; see <> for more information. + ==== (S)FTP Changes The inbound channel adapters now have a property `max-fetch-size` which is used to limit the number of files fetched during a poll when there are no files currently in the local directory. @@ -159,6 +161,7 @@ The `RemoteFileTemplate` is supplied now with the `invoke(OperationsCallback> and <> for more information. +New filters for detecting incomplete remote files are now provided; see <> and <> for more information. ==== Integration Properties