INT-2898 Add Persistent FileListFilters

- Abstract implementation
- Implementation for the file system
- Implementation for remote files ((S)FTP)

JIRA: https://jira.springsource.org/browse/INT-2898

INT-2889 Polishing

- Add (S)FTP test cases
- Docs

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.file.filters;
+
+import org.springframework.integration.metadata.MetadataStore;
+import org.springframework.util.Assert;
+
+/**
+ * Stores "seen" files in a MetadataStore to survive application restarts.
+ * The default key is 'prefix' plus the absolute file name; value is the timestamp of the file.
+ * Files are deemed as already 'seen' if they exist in the store and have the
+ * same modified time as the current file.
+ *
+ * @author Gary Russell
+ * @since 3.0
+ *
+ */
+public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> {
+
+	protected final MetadataStore store;
+
+	protected final String prefix;
+
+	private final Object monitor = new Object();
+
+	public AbstractPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) {
+		Assert.notNull(store, "'store' cannot be null");
+		Assert.notNull(prefix, "'prefix' cannot be null");
+		this.store = store;
+		this.prefix = prefix;
+	}
+
+	@Override
+	protected boolean accept(F file) {
+		String key = buildKey(file);
+		synchronized(monitor) {
+			String value = store.get(key);
+			if (value != null && isEqual(file, value)) {
+				return false;
+			}
+			store.put(key, value(file));
+		}
+		return true;
+	}
+
+	/**
+	 * The default value stored for the key is the last modified date.
+	 * @param file The file.
+	 * @return The value to store for the file.
+	 */
+	private String value(F file) {
+		return Long.toString(this.modified(file));
+	}
+
+	/**
+	 * Override this method if you wish to use something other than the
+	 * modified timestamp to determine equality.
+	 * @param file The file.
+	 * @param value The current value for the key in the store
+	 * @return
+	 */
+	protected boolean isEqual(F file, String value) {
+		return Long.valueOf(value).longValue() == this.modified(file);
+	}
+
+	/**
+	 * The default key is the {@link #prefix} plus the full filename.
+	 * @param file The file.
+	 * @return The key.
+	 */
+	protected String buildKey(F file) {
+		return this.prefix + this.fileName(file);
+	}
+
+	protected abstract long modified(F file);
+
+	protected abstract String fileName(F file);
+
+}

spring-integration-file/src/main/java/org/springframework/integration/file/filters/FileSystemPersistentAcceptOnceFileListFilter.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.file.filters;
+
+
+import java.io.File;
+
+import org.springframework.integration.metadata.MetadataStore;
+
+/**
+ * @author Gary Russell
+ * @since 3.0
+ *
+ */
+public class FileSystemPersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter<File> {
+
+	public FileSystemPersistentAcceptOnceFileListFilter(MetadataStore store, String prefix) {
+		super(store, prefix);
+	}
+
+	@Override
+	protected long modified(File file) {
+		return file.lastModified();
+	}
+
+	@Override
+	protected String fileName(File file) {
+		return file.getAbsolutePath();
+	}
+
+}

spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests-context.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	   xmlns:p="http://www.springframework.org/schema/p" xmlns:util="http://www.springframework.org/schema/util"
+	   xsi:schemaLocation="http://www.springframework.org/schema/beans
+			http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
+
+	<!-- under test -->
+	<bean id="pollableFileSource" class="org.springframework.integration.file.FileReadingMessageSource"
+		  p:directory="file:${java.io.tmpdir}/FileReadingMessageSourcePersistentFilterIntegrationTests"
+		  p:filter-ref="persistentFilter"/>
+
+	<!-- persistent filter -->
+	<bean id="persistentFilter" class="org.springframework.integration.file.filters.FileSystemPersistentAcceptOnceFileListFilter">
+		<constructor-arg ref="ppms" />
+		<constructor-arg value="frmsPersistTest" />
+	</bean>
+
+	<bean id="ppms" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore">
+		<property name="baseDirectory"
+			value="#{T(System).getProperty('java.io.tmpdir') + T(java.io.File).separator + 'FileReadingMessageSourcePersistentFilterIntegrationTests.meta'}"/>
+	</bean>
+
+	<bean class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer" />
+
+</beans>

spring-integration-file/src/test/java/org/springframework/integration/file/FileReadingMessageSourcePersistentFilterIntegrationTests.java

@@ -0,0 +1,126 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *	  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.springframework.beans.DirectFieldAccessor;
+import org.springframework.context.support.AbstractApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+import org.springframework.integration.Message;
+
+/**
+ * @author Iwein Fuld
+ * @author Gary Russell
+ */
+public class FileReadingMessageSourcePersistentFilterIntegrationTests {
+
+	AbstractApplicationContext context;
+
+	FileReadingMessageSource pollableFileSource;
+
+	private static File inputDir;
+
+	@AfterClass
+	public static void cleanUp() throws Throwable {
+	  if(inputDir.exists()) {
+		  inputDir.delete();
+	  }
+	}
+
+	@BeforeClass
+	public static void setupInputDir() {
+		inputDir = new File(System.getProperty("java.io.tmpdir") + "/"
+				+ FileReadingMessageSourcePersistentFilterIntegrationTests.class.getSimpleName());
+		inputDir.mkdir();
+	}
+
+	@Before
+	public void generateTestFiles() throws Exception {
+		File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000);
+		File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000);
+		File.createTempFile("test", null, inputDir).setLastModified(System.currentTimeMillis() - 1000);
+		this.loadContextAndGetMessageSource();
+	}
+
+	private void loadContextAndGetMessageSource() {
+		this.context = new ClassPathXmlApplicationContext(this.getClass().getSimpleName() + "-context.xml",
+				this.getClass());
+		this.pollableFileSource = context.getBean(FileReadingMessageSource.class);
+	}
+
+	@After
+	public void cleanoutInputDir() throws Exception {
+		File[] listFiles = inputDir.listFiles();
+		for (int i = 0; i < listFiles.length; i++) {
+			listFiles[i].delete();
+		}
+	}
+
+	@AfterClass
+	public static void removeInputDir() throws Exception {
+		inputDir.delete();
+		File persistDir = new File(System.getProperty("java.io.tmpdir") + "/"
+				+ FileReadingMessageSourcePersistentFilterIntegrationTests.class.getSimpleName()
+				+ ".meta");
+		File persist = new File(persistDir, "metadata-store.properties");
+		persist.delete();
+		persistDir.delete();
+	}
+
+
+	@Test
+	public void configured() throws Exception {
+		DirectFieldAccessor accessor = new DirectFieldAccessor(pollableFileSource);
+		assertEquals(inputDir, accessor.getPropertyValue("directory"));
+	}
+
+	@Test
+	public void getFiles() throws Exception {
+		Message<File> received1 = pollableFileSource.receive();
+		System.out.println("receive files round 1");
+		assertNotNull("This should return the first message", received1);
+		pollableFileSource.onSend(received1);
+		Message<File> received2 = pollableFileSource.receive();
+		assertNotNull(received2);
+		pollableFileSource.onSend(received2);
+		Message<File> received3 = pollableFileSource.receive();
+		assertNotNull(received3);
+		pollableFileSource.onSend(received3);
+		assertNotSame(received1 + " == " + received2, received1.getPayload(), received2.getPayload());
+		assertNotSame(received1 + " == " + received3, received1.getPayload(), received3.getPayload());
+		assertNotSame(received2 + " == " + received3, received2.getPayload(), received3.getPayload());
+		this.context.destroy();
+
+		this.loadContextAndGetMessageSource();
+		Message<File> received4 = pollableFileSource.receive();
+		assertNull(received4);
+		this.context.destroy();
+	}
+
+}

spring-integration-file/src/test/java/org/springframework/integration/file/filters/PersistentAcceptOnceFileListFilterTests.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.file.filters;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import org.springframework.integration.metadata.MetadataStore;
+import org.springframework.integration.metadata.SimpleMetadataStore;
+
+/**
+ * @author Gary Russell
+ * @since 3.0
+ *
+ */
+public class PersistentAcceptOnceFileListFilterTests {
+
+	@Test
+	public void testFileSystem() throws Exception {
+		MetadataStore store = new SimpleMetadataStore();
+		FileSystemPersistentAcceptOnceFileListFilter filter = new FileSystemPersistentAcceptOnceFileListFilter(store, "foo:");
+		File file = File.createTempFile("foo", ".txt");
+		assertTrue(filter.filterFiles(new File[] {file}).size() == 1);
+		assertTrue(filter.filterFiles(new File[] {file}).size() == 0);
+		file.setLastModified(27L);
+		assertTrue(filter.filterFiles(new File[] {file}).size() == 1);
+		assertTrue(filter.filterFiles(new File[] {file}).size() == 0);
+		file.delete();
+	}
+
+}