INT-4095: Support Limiting (S)FTP Files Fetched

JIRA: https://jira.spring.io/browse/INT-4095

Limit the number of remote files fetched on each poll (when it is necessary to fetch files).

Polishing - PR Comments

Polishing - Decouple MaxFetchSize from Poller

Polishing - PR Comments

Schemas and Docs

More Polishing

* Polishing according PR comments

spring-integration-file/src/main/java/org/springframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java

@@ -82,6 +82,7 @@ public abstract class AbstractRemoteFileInboundChannelAdapterParser extends Abst
 			synchronizerBuilder.addPropertyValue("localFilenameGeneratorExpression",
 					localFileGeneratorExpressionBuilder.getBeanDefinition());
 		}
+		IntegrationNamespaceUtils.setValueIfAttributeDefined(messageSourceBuilder, element, "max-fetch-size");
 		return messageSourceBuilder.getBeanDefinition();
 	}
 

spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java

@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -227,6 +228,17 @@ public abstract class AbstractInboundFileSynchronizer<F>
 
 	@Override
 	public void synchronizeToLocalDirectory(final File localDirectory) {
+		synchronizeToLocalDirectory(localDirectory, Integer.MIN_VALUE);
+	}
+
+	@Override
+	public void synchronizeToLocalDirectory(final File localDirectory, final int maxFetchSize) {
+		if (maxFetchSize == 0) {
+			if (this.logger.isDebugEnabled()) {
+				this.logger.debug("Max Fetch Size is zero - fetch to " + localDirectory.getAbsolutePath() + " ignored");
+			}
+			return;
+		}
 		final String remoteDirectory = this.remoteDirectoryExpression.getValue(this.evaluationContext, String.class);
 		try {
 			int transferred = this.remoteFileTemplate.execute(new SessionCallback<F, Integer>() {
@@ -236,6 +248,14 @@ public abstract class AbstractInboundFileSynchronizer<F>
 					F[] files = session.list(remoteDirectory);
 					if (!ObjectUtils.isEmpty(files)) {
 						List<F> filteredFiles = filterFiles(files);
+						if (maxFetchSize >= 0 && filteredFiles.size() > maxFetchSize) {
+							rollbackFromFileToListEnd(filteredFiles, filteredFiles.get(maxFetchSize));
+							List<F> newList = new ArrayList<>(maxFetchSize);
+							for (int i = 0; i < maxFetchSize; i++) {
+								newList.add(filteredFiles.get(i));
+							}
+							filteredFiles = newList;
+						}
 						for (F file : filteredFiles) {
 							try {
 								if (file != null) {
@@ -245,17 +265,11 @@ public abstract class AbstractInboundFileSynchronizer<F>
 								}
 							}
 							catch (RuntimeException e) {
-								if (AbstractInboundFileSynchronizer.this.filter instanceof ReversibleFileListFilter) {
-									((ReversibleFileListFilter<F>) AbstractInboundFileSynchronizer.this.filter)
-											.rollback(file, filteredFiles);
-								}
+								rollbackFromFileToListEnd(filteredFiles, file);
 								throw e;
 							}
 							catch (IOException e) {
-								if (AbstractInboundFileSynchronizer.this.filter instanceof ReversibleFileListFilter) {
-									((ReversibleFileListFilter<F>) AbstractInboundFileSynchronizer.this.filter)
-											.rollback(file, filteredFiles);
-								}
+								rollbackFromFileToListEnd(filteredFiles, file);
 								throw e;
 							}
 						}
@@ -265,6 +279,14 @@ public abstract class AbstractInboundFileSynchronizer<F>
 						return 0;
 					}
 				}
+
+				public void rollbackFromFileToListEnd(List<F> filteredFiles, F file) {
+					if (AbstractInboundFileSynchronizer.this.filter instanceof ReversibleFileListFilter) {
+						((ReversibleFileListFilter<F>) AbstractInboundFileSynchronizer.this.filter)
+								.rollback(file, filteredFiles);
+					}
+				}
+
 			});
 			if (this.logger.isDebugEnabled()) {
 				this.logger.debug(transferred + " files transferred");

spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizingMessageSource.java

@@ -25,7 +25,7 @@ import java.util.regex.Pattern;
 
 import org.springframework.beans.factory.BeanInitializationException;
 import org.springframework.context.Lifecycle;
-import org.springframework.integration.endpoint.AbstractMessageSource;
+import org.springframework.integration.endpoint.AbstractFetchLimitingMessageSource;
 import org.springframework.integration.file.FileReadingMessageSource;
 import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.CompositeFileListFilter;
@@ -57,7 +57,7 @@ import org.springframework.util.Assert;
  * @author Gary Russell
  */
 public abstract class AbstractInboundFileSynchronizingMessageSource<F>
-		extends AbstractMessageSource<File> implements Lifecycle {
+		extends AbstractFetchLimitingMessageSource<File> implements Lifecycle {
 
 	private volatile boolean running;
 
@@ -182,21 +182,20 @@ public abstract class AbstractInboundFileSynchronizingMessageSource<F>
 	/**
 	 * Polls from the file source. If the result is not null, it will be returned.
 	 * If the result is null, it attempts to sync up with the remote directory to populate the file source.
+	 * At most, maxFetchSize files will be fetched.
 	 * Then, it polls the file source again and returns the result, whether or not it is null.
+	 * @param maxFetchSize the maximum files to fetch.
 	 */
 	@Override
-	public final Message<File> doReceive() {
-		Assert.state(this.fileSource != null, "fileSource must not be null");
-		Assert.state(this.synchronizer != null, "synchronizer must not be null");
+	public final Message<File> doReceive(int maxFetchSize) {
 		Message<File> message = this.fileSource.receive();
 		if (message == null) {
-			this.synchronizer.synchronizeToLocalDirectory(this.localDirectory);
+			this.synchronizer.synchronizeToLocalDirectory(this.localDirectory, maxFetchSize);
 			message = this.fileSource.receive();
 		}
 		return message;
 	}
 
-	@SuppressWarnings("unchecked")
 	private FileListFilter<File> buildFilter() {
 		Pattern completePattern = Pattern.compile("^.*(?<!" + this.synchronizer.getTemporaryFileSuffix() + ")$");
 		return new CompositeFileListFilter<File>(Arrays.asList(

spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/InboundFileSynchronizer.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2010 the original author or authors.
+ * Copyright 2002-2016 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,10 +22,25 @@ import java.io.File;
  * Strategy for synchronizing from a remote File system to a local directory.
  *
  * @author Mark Fisher
+ * @author Gary Russell
  * @since 2.0
  */
+@FunctionalInterface
 public interface InboundFileSynchronizer {
 
+	/**
+	 * Synchronize all available files to the local directory;
+	 * @param localDirectory the directory.
+	 */
 	void synchronizeToLocalDirectory(File localDirectory);
 
+	/**
+	 * Synchronize up to maxFetchSize files to the local directory;
+	 * @param localDirectory the directory.
+	 * @param maxFetchSize the maximum files to fetch.
+	 */
+	default void synchronizeToLocalDirectory(File localDirectory, int maxFetchSize) {
+		synchronizeToLocalDirectory(localDirectory);
+	}
+
 }

spring-integration-file/src/test/java/org/springframework/integration/file/remote/synchronizer/AbstractRemoteFileSynchronizerTests.java

@@ -26,11 +26,13 @@ import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.UUID;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.junit.Test;
 
+import org.springframework.beans.factory.BeanFactory;
 import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
 import org.springframework.integration.file.remote.session.Session;
 import org.springframework.integration.file.remote.session.SessionFactory;
@@ -94,6 +96,81 @@ public class AbstractRemoteFileSynchronizerTests {
 		sync.close();
 	}
 
+	@Test
+	public void testMaxFetchSizeSynchronizer() throws Exception {
+		final AtomicInteger count = new AtomicInteger();
+		AbstractInboundFileSynchronizer<String> sync = createLimitingSynchronizer(count);
+
+		sync.synchronizeToLocalDirectory(mock(File.class), 1);
+		assertEquals(1, count.get());
+		sync.synchronizeToLocalDirectory(mock(File.class), 1);
+		assertEquals(2, count.get());
+		sync.synchronizeToLocalDirectory(mock(File.class), 1);
+		assertEquals(3, count.get());
+		sync.close();
+	}
+
+	@Test
+	public void testMaxFetchSizeSource() throws Exception {
+		final AtomicInteger count = new AtomicInteger();
+		AbstractInboundFileSynchronizer<String> sync = createLimitingSynchronizer(count);
+
+		AbstractInboundFileSynchronizingMessageSource<String> source =
+				new AbstractInboundFileSynchronizingMessageSource<String>(sync) {
+
+			@Override
+			public String getComponentType() {
+				return "foo";
+			}
+
+		};
+		source.setLocalDirectory(new File(System.getProperty("java.io.tmpdir") + File.separator + UUID.randomUUID()));
+		source.setAutoCreateLocalDirectory(true);
+		source.setBeanFactory(mock(BeanFactory.class));
+		source.setMaxFetchSize(1);
+		source.afterPropertiesSet();
+
+		source.receive();
+		assertEquals(1, count.get());
+		sync.synchronizeToLocalDirectory(mock(File.class), 1);
+		source.receive();
+		sync.synchronizeToLocalDirectory(mock(File.class), 1);
+		source.receive();
+		sync.close();
+	}
+
+	private AbstractInboundFileSynchronizer<String> createLimitingSynchronizer(final AtomicInteger count) {
+		SessionFactory<String> sf = new StringSessionFactory();
+		AbstractInboundFileSynchronizer<String> sync = new AbstractInboundFileSynchronizer<String>(sf) {
+
+			@Override
+			protected boolean isFile(String file) {
+				return true;
+			}
+
+			@Override
+			protected String getFilename(String file) {
+				return file;
+			}
+
+			@Override
+			protected long getModified(String file) {
+				return 0;
+			}
+
+			@Override
+			protected void copyFileToLocalDirectory(String remoteDirectoryPath, String remoteFile, File localDirectory,
+					Session<String> session) throws IOException {
+				count.incrementAndGet();
+			}
+
+		};
+		sync.setFilter(new AcceptOnceFileListFilter<String>());
+		sync.setRemoteDirectory("foo");
+		sync.setBeanFactory(mock(BeanFactory.class));
+		return sync;
+	}
+
 	private class StringSessionFactory implements SessionFactory<String> {
 
 		@Override