INT-3827: ResettableFileListFilter

JIRA: https://jira.spring.io/browse/INT-3827

Provide a hook to enable removing a file from an `AcceptOnceFileListFilter`,
for example after a message processing failure.

Make the `CompositeFileListFilter` a `ReversibleFileListFilter` so it can
delegate to any of its composed filters that are reversible.

INT-3827: Polishing - PR Comments
(cherry picked from commit 0d721739e9)

Conflicts:
	src/reference/asciidoc/ftp.adoc
	src/reference/asciidoc/sftp.adoc

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AbstractPersistentAcceptOnceFileListFilter.java

@@ -35,7 +35,7 @@ import org.springframework.util.Assert;
  *
  */
 public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends AbstractFileListFilter<F>
-		implements ReversibleFileListFilter<F>, Closeable {
+		implements ReversibleFileListFilter<F>, ResettableFileListFilter<F>,  Closeable {
 
 	protected final ConcurrentMetadataStore store;
 
@@ -102,12 +102,18 @@ public abstract class AbstractPersistentAcceptOnceFileListFilter<F> extends Abst
 				rollingBack = true;
 			}
 			if (rollingBack) {
-				this.store.remove(buildKey(fileToRollback));
-				flushIfNeeded();
+				remove(fileToRollback);
 			}
 		}
 	}
 
+	@Override
+	public boolean remove(F fileToRemove) {
+		String removed = this.store.remove(buildKey(fileToRemove));
+		flushIfNeeded();
+		return removed != null;
+	}
+
 	@Override
 	public void close() throws IOException {
 		if (this.store instanceof Closeable) {

spring-integration-file/src/main/java/org/springframework/integration/file/filters/AcceptOnceFileListFilter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2014 the original author or authors.
+ * Copyright 2002-2015 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,7 +34,8 @@ import java.util.concurrent.LinkedBlockingQueue;
  * @author Gary Russell
  * @since 1.0.0
  */
-public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> implements ReversibleFileListFilter<F> {
+public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> implements ReversibleFileListFilter<F>,
+		ResettableFileListFilter<F> {
 
 	private final Queue<F> seen;
 
@@ -93,13 +94,19 @@ public class AcceptOnceFileListFilter<F> extends AbstractFileListFilter<F> imple
 					rollingBack = true;
 				}
 				if (rollingBack) {
-					this.seenSet.remove(fileToRollback);
-					if (this.seen != null) {
-						this.seen.remove(fileToRollback);
-					}
+					remove(fileToRollback);
 				}
 			}
 		}
 	}
 
+	@Override
+	public boolean remove(F fileToRemove) {
+		boolean removed = this.seenSet.remove(fileToRemove);
+		if (this.seen != null) {
+			this.seen.remove(fileToRemove);
+		}
+		return removed;
+	}
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/filters/CompositeFileListFilter.java

@@ -39,7 +39,7 @@ import org.springframework.util.Assert;
  *
  * @param <F> The type that will be filtered.
  */
-public class CompositeFileListFilter<F> implements FileListFilter<F>, Closeable {
+public class CompositeFileListFilter<F> implements ReversibleFileListFilter<F>, Closeable {
 
 	private final Set<FileListFilter<F>> fileFilters;
 
@@ -111,4 +111,13 @@ public class CompositeFileListFilter<F> implements FileListFilter<F>, Closeable
 		return results;
 	}
 
+	@Override
+	public void rollback(F file, List<F> files) {
+		for (FileListFilter<F> fileFilter : this.fileFilters) {
+			if (fileFilter instanceof ReversibleFileListFilter) {
+				((ReversibleFileListFilter<F>) fileFilter).rollback(file, files);
+			}
+		}
+	}
+
 }

spring-integration-file/src/main/java/org/springframework/integration/file/filters/ResettableFileListFilter.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2015 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.file.filters;
+
+/**
+ * A {@link FileListFilter} that can be reset by removing a specific file from its
+ * state.
+ * @author Gary Russell
+ * @since 4.1.7
+ *
+ */
+public interface ResettableFileListFilter<F> extends FileListFilter<F> {
+
+	/**
+	 * Remove the specified file from the filter so it will pass on the next attempt.
+	 * @param f the element to remove.
+	 * @return true if the file was removed as a result of this call.
+	 */
+	boolean remove(F f);
+
+}

spring-integration-file/src/test/java/org/springframework/integration/file/filters/AcceptOnceFileListFilterTests.java

@@ -24,6 +24,7 @@ import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Queue;
 import java.util.Set;
@@ -76,6 +77,14 @@ public class AcceptOnceFileListFilterTests {
 		doTestRollback(filter);
 	}
 
+	@Test
+	public void testRollbackComposite() {
+		AcceptOnceFileListFilter<String> filter = new AcceptOnceFileListFilter<String>();
+		CompositeFileListFilter<String> composite = new CompositeFileListFilter<String>(
+				Collections.singletonList(filter));
+		doTestRollback(composite);
+	}
+
 	protected void doTestRollback(ReversibleFileListFilter<String> filter) {
 		String[] files = new String[] {"foo", "bar", "baz"};
 		List<String> passed = filter.filterFiles(files);

spring-integration-sftp/.gitignore

@@ -1,3 +1,4 @@
 local-test-dir/*.test
+local-test-dir/rollback/*.txt
 remote-target-dir/*foo*
 remote-target-dir/*test*

spring-integration-sftp/local-test-dir/readme.txt

@@ -1 +1 @@
-Don't delete. This directory is used by test cases
+don't delete this dir used for testing

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/RollbackLocalFilterTests-context.xml

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:int="http://www.springframework.org/schema/integration"
+	xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
+		http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
+		http://www.springframework.org/schema/integration/sftp http://www.springframework.org/schema/integration/sftp/spring-integration-sftp.xsd">
+
+	<int-sftp:inbound-channel-adapter id="sftpAdapterAutoCreate"
+			session-factory="sftpSessionFactory"
+			channel="requestChannel"
+			remote-directory-expression="'/sftpSource'"
+			local-directory="file:local-test-dir/rollback"
+			auto-create-local-directory="true"
+			filename-pattern="sftpSource1.txt"
+			local-filter="acceptOnceFilter">
+		<int:poller fixed-rate="1000" max-messages-per-poll="2" error-channel="nullChannel">
+			<int:transactional synchronization-factory="syncFactory" />
+		</int:poller>
+	</int-sftp:inbound-channel-adapter>
+
+	<int:channel id="requestChannel" />
+
+	<int:service-activator input-channel="requestChannel" ref="crash" method="handle" />
+
+	<bean id="crash" class="org.springframework.integration.sftp.inbound.RollbackLocalFilterTests$Crash" />
+
+	<bean id="acceptOnceFilter" class="org.springframework.integration.file.filters.AcceptOnceFileListFilter" />
+
+	<int:transaction-synchronization-factory id="syncFactory">
+		<int:after-rollback expression="@acceptOnceFilter.remove(payload)" />
+	</int:transaction-synchronization-factory>
+
+	<bean id="transactionManager" class="org.springframework.integration.transaction.PseudoTransactionManager" />
+
+	<bean id="sftpServerConfig" class="org.springframework.integration.sftp.TestSftpServerConfig" />
+
+</beans>

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/RollbackLocalFilterTests.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2015 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.integration.sftp.inbound;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+/**
+ * @author Gary Russell
+ * @since 4.1.7
+ *
+ */
+@ContextConfiguration
+@RunWith(SpringJUnit4ClassRunner.class)
+public class RollbackLocalFilterTests {
+
+	@Autowired
+	private Crash crash;
+
+	@Test
+	public void testRollback() throws Exception {
+		assertTrue(this.crash.getLatch().await(10, TimeUnit.SECONDS));
+		assertEquals("sftpSource1.txt", this.crash.getFile().getName());
+	}
+
+	public static class Crash {
+
+		private final CountDownLatch latch = new CountDownLatch(2);
+
+		private final AtomicBoolean shouldCrash = new AtomicBoolean();
+
+		private volatile File file;
+
+		public CountDownLatch getLatch() {
+			return latch;
+		}
+
+		public File getFile() {
+			return file;
+		}
+
+		public void handle(File in) {
+			latch.countDown();
+			if (this.shouldCrash.compareAndSet(false, true)) {
+				throw new RuntimeException("foo");
+			}
+			this.file = in;
+		}
+	}
+
+}