From 11d39b7da1197062d63c02772c14203030b0674e Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Mon, 2 Jul 2018 16:49:23 -0400 Subject: [PATCH] INT-4496: (S)FTP: Sort file list earlier JIRA: https://jira.spring.io/browse/INT-4496 Sort the file list before applying `maxFetchSize` and filters. * Polishing - PR Comments --- ...tractRemoteFileStreamingMessageSource.java | 10 +++------ .../AbstractInboundFileSynchronizer.java | 21 ++++++++++++++++++- .../integration/file/support/FileUtils.java | 20 ++++++++++++++---- .../file/remote/StreamingInboundTests.java | 2 +- .../integration/ftp/dsl/Ftp.java | 3 +-- ...FtpStreamingInboundChannelAdapterSpec.java | 5 ++--- .../inbound/FtpStreamingMessageSource.java | 3 +-- .../FtpStreamingMessageSourceTests.java | 6 ++---- .../integration/sftp/dsl/Sftp.java | 3 +-- ...ftpStreamingInboundChannelAdapterSpec.java | 3 +-- .../inbound/SftpStreamingMessageSource.java | 2 +- .../SftpStreamingMessageSourceTests.java | 5 ++--- src/reference/asciidoc/ftp.adoc | 4 ++++ src/reference/asciidoc/sftp.adoc | 4 ++++ src/reference/asciidoc/whats-new.adoc | 5 +++++ 15 files changed, 64 insertions(+), 32 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java index c220263623..a8530a4da6 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java @@ -21,7 +21,6 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -59,7 +58,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource private final BlockingQueue> toBeReceived = new LinkedBlockingQueue>(); - private final Comparator> comparator; + private final Comparator comparator; private boolean fileInfoJson = true; @@ -76,7 +75,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource private volatile FileListFilter filter; protected AbstractRemoteFileStreamingMessageSource(RemoteFileTemplate template, - Comparator> comparator) { + Comparator comparator) { this.remoteFileTemplate = template; this.comparator = comparator; } @@ -194,7 +193,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource String remoteDirectory = this.remoteDirectoryExpression.getValue(getEvaluationContext(), String.class); F[] files = this.remoteFileTemplate.list(remoteDirectory); if (!ObjectUtils.isEmpty(files)) { - files = FileUtils.purgeUnwantedElements(files, f -> f == null || isDirectory(f)); + files = FileUtils.purgeUnwantedElements(files, f -> f == null || isDirectory(f), this.comparator); } if (!ObjectUtils.isEmpty(files)) { int maxFetchSize = getMaxFetchSize(); @@ -209,9 +208,6 @@ public abstract class AbstractRemoteFileStreamingMessageSource } List> fileInfoList = asFileInfoList(filteredFiles); fileInfoList.forEach(fi -> fi.setRemoteDirectory(remoteDirectory)); - if (this.comparator != null) { - Collections.sort(fileInfoList, this.comparator); - } this.toBeReceived.addAll(fileInfoList); } } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java index b467659c1b..0fce2582e4 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; import java.util.regex.Matcher; @@ -59,6 +60,8 @@ import org.springframework.util.ObjectUtils; * {@link org.springframework.integration.file.filters.FileListFilter}s to * ensure the file entry is acceptable. * + * @param the Type that represents a remote file. + * * @author Josh Long * @author Mark Fisher * @author Oleg Zhurakousky @@ -116,6 +119,8 @@ public abstract class AbstractInboundFileSynchronizer private BeanFactory beanFactory; + private Comparator comparator; + /** * Create a synchronizer with the {@link SessionFactory} used to acquire {@link Session} instances. * @@ -127,6 +132,20 @@ public abstract class AbstractInboundFileSynchronizer } + protected Comparator getComparator() { + return this.comparator; + } + + /** + * Set a comparator to sort the retrieved list of {@code F} (the Type that represents + * the remote file) prior to applying filters and max fetch size. + * @param comparator the comparator. + * @since 5.1 + */ + public void setComparator(Comparator comparator) { + this.comparator = comparator; + } + /** * @param remoteFileSeparator the remote file separator. * @see RemoteFileTemplate#setRemoteFileSeparator(String) @@ -286,7 +305,7 @@ public abstract class AbstractInboundFileSynchronizer int transferred = this.remoteFileTemplate.execute(session -> { F[] files = session.list(this.evaluatedRemoteDirectory); if (!ObjectUtils.isEmpty(files)) { - files = FileUtils.purgeUnwantedElements(files, e -> !isFile(e)); + files = FileUtils.purgeUnwantedElements(files, e -> !isFile(e), this.comparator); } if (!ObjectUtils.isEmpty(files)) { List filteredFiles = filterFiles(files); diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/support/FileUtils.java b/spring-integration-file/src/main/java/org/springframework/integration/file/support/FileUtils.java index ac2bafb609..93d7aca79c 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/support/FileUtils.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/support/FileUtils.java @@ -19,8 +19,10 @@ package org.springframework.integration.file.support; import java.lang.reflect.Array; import java.nio.file.FileSystems; import java.util.Arrays; +import java.util.Comparator; import java.util.function.Predicate; +import org.springframework.lang.Nullable; import org.springframework.util.ObjectUtils; /** @@ -38,19 +40,29 @@ public final class FileUtils { * Remove entries from the array if the predicate returns true for an element. * @param fileArray the array. * @param predicate the predicate. + * @param comparator an optional comparator to sort the results. * @param the file type. * @return the array of remaining elements. * @since 5.0.7 */ @SuppressWarnings("unchecked") - public static F[] purgeUnwantedElements(F[] fileArray, Predicate predicate) { + public static F[] purgeUnwantedElements(F[] fileArray, Predicate predicate, + @Nullable Comparator comparator) { if (ObjectUtils.isEmpty(fileArray)) { return fileArray; } else { - return Arrays.stream(fileArray) - .filter(predicate.negate()) - .toArray(size -> (F[]) Array.newInstance(fileArray[0].getClass(), size)); + if (comparator == null) { + return Arrays.stream(fileArray) + .filter(predicate.negate()) + .toArray(size -> (F[]) Array.newInstance(fileArray[0].getClass(), size)); + } + else { + return Arrays.stream(fileArray) + .filter(predicate.negate()) + .sorted(comparator) + .toArray(size -> (F[]) Array.newInstance(fileArray[0].getClass(), size)); + } } } diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java index 5a4c8b9f92..ee87c6b945 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java @@ -192,7 +192,7 @@ public class StreamingInboundTests { public static class Streamer extends AbstractRemoteFileStreamingMessageSource { - protected Streamer(RemoteFileTemplate template, Comparator> comparator) { + protected Streamer(RemoteFileTemplate template, Comparator comparator) { super(template, comparator); } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/Ftp.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/Ftp.java index d02d3d4d02..05a9c6298d 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/Ftp.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/Ftp.java @@ -21,7 +21,6 @@ import java.util.Comparator; import org.apache.commons.net.ftp.FTPFile; -import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.MessageSessionCallback; import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; @@ -79,7 +78,7 @@ public final class Ftp { */ public static FtpStreamingInboundChannelAdapterSpec inboundStreamingAdapter( RemoteFileTemplate remoteFileTemplate, - Comparator> receptionOrderComparator) { + Comparator receptionOrderComparator) { return new FtpStreamingInboundChannelAdapterSpec(remoteFileTemplate, receptionOrderComparator); } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpStreamingInboundChannelAdapterSpec.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpStreamingInboundChannelAdapterSpec.java index 1e4ff2ccc3..ac5704ac44 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpStreamingInboundChannelAdapterSpec.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/dsl/FtpStreamingInboundChannelAdapterSpec.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2017 the original author or authors. + * Copyright 2014-2018 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,7 +23,6 @@ import org.apache.commons.net.ftp.FTPFile; import org.springframework.integration.file.dsl.RemoteFileStreamingInboundChannelAdapterSpec; import org.springframework.integration.file.filters.CompositeFileListFilter; import org.springframework.integration.file.filters.FileListFilter; -import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter; import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter; @@ -43,7 +42,7 @@ public class FtpStreamingInboundChannelAdapterSpec FtpStreamingMessageSource> { FtpStreamingInboundChannelAdapterSpec(RemoteFileTemplate remoteFileTemplate, - Comparator> comparator) { + Comparator comparator) { this.target = new FtpStreamingMessageSource(remoteFileTemplate, comparator); } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java index 3b5898b9f2..552060da16 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSource.java @@ -56,8 +56,7 @@ public class FtpStreamingMessageSource extends AbstractRemoteFileStreamingMessag * @param template the template. * @param comparator the comparator. */ - public FtpStreamingMessageSource(RemoteFileTemplate template, - Comparator> comparator) { + public FtpStreamingMessageSource(RemoteFileTemplate template, Comparator comparator) { super(template, comparator); doSetFilter(new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "ftpStreamingMessageSource")); } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java index 5b41504519..981055b713 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java @@ -46,7 +46,6 @@ import org.springframework.integration.core.MessageSource; import org.springframework.integration.endpoint.SourcePollingChannelAdapter; import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.filters.AcceptAllFileListFilter; -import org.springframework.integration.file.remote.FileInfo; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.ftp.FtpTestSupport; import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter; @@ -160,7 +159,7 @@ public class FtpStreamingMessageSourceTests extends FtpTestSupport { private FtpStreamingMessageSource buildSource() { FtpStreamingMessageSource messageSource = new FtpStreamingMessageSource(this.config.template(), - Comparator.comparing(FileInfo::getFilename)); + Comparator.comparing(FTPFile::getName)); messageSource.setRemoteDirectory("ftpSource/"); messageSource.setMaxFetchSize(1); messageSource.setBeanFactory(this.context); @@ -193,11 +192,10 @@ public class FtpStreamingMessageSourceTests extends FtpTestSupport { @InboundChannelAdapter(channel = "stream", autoStartup = "false") public MessageSource ftpMessageSource() { FtpStreamingMessageSource messageSource = new FtpStreamingMessageSource(template(), - Comparator.comparing(FileInfo::getFilename)); + Comparator.comparing(FTPFile::getName)); messageSource.setFilter( new FtpPersistentAcceptOnceFileListFilter( new SimpleMetadataStore(metadataMap()), "testStreaming")); - messageSource.setRemoteDirectory("ftpSource/"); return messageSource; } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/Sftp.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/Sftp.java index b71cbbaa1f..a13049640f 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/Sftp.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/Sftp.java @@ -19,7 +19,6 @@ package org.springframework.integration.sftp.dsl; import java.io.File; import java.util.Comparator; -import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.MessageSessionCallback; import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; @@ -80,7 +79,7 @@ public final class Sftp { */ public static SftpStreamingInboundChannelAdapterSpec inboundStreamingAdapter( RemoteFileTemplate remoteFileTemplate, - Comparator> receptionOrderComparator) { + Comparator receptionOrderComparator) { return new SftpStreamingInboundChannelAdapterSpec(remoteFileTemplate, receptionOrderComparator); } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpStreamingInboundChannelAdapterSpec.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpStreamingInboundChannelAdapterSpec.java index 05ffc4b783..985c34121a 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpStreamingInboundChannelAdapterSpec.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/dsl/SftpStreamingInboundChannelAdapterSpec.java @@ -21,7 +21,6 @@ import java.util.Comparator; import org.springframework.integration.file.dsl.RemoteFileStreamingInboundChannelAdapterSpec; import org.springframework.integration.file.filters.CompositeFileListFilter; import org.springframework.integration.file.filters.FileListFilter; -import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.metadata.SimpleMetadataStore; import org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter; @@ -41,7 +40,7 @@ public class SftpStreamingInboundChannelAdapterSpec SftpStreamingMessageSource> { SftpStreamingInboundChannelAdapterSpec(RemoteFileTemplate remoteFileTemplate, - Comparator> comparator) { + Comparator comparator) { this.target = new SftpStreamingMessageSource(remoteFileTemplate, comparator); } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSource.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSource.java index 057c47441c..f2a1a2649d 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSource.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSource.java @@ -57,7 +57,7 @@ public class SftpStreamingMessageSource extends AbstractRemoteFileStreamingMessa * @param comparator the comparator. */ public SftpStreamingMessageSource(RemoteFileTemplate template, - Comparator> comparator) { + Comparator comparator) { super(template, comparator); doSetFilter(new SftpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "sftpStreamingMessageSource")); } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java index 17d55b76eb..9f1bc90e12 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java @@ -43,7 +43,6 @@ import org.springframework.integration.core.MessageSource; import org.springframework.integration.endpoint.SourcePollingChannelAdapter; import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.filters.AcceptAllFileListFilter; -import org.springframework.integration.file.remote.FileInfo; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.scheduling.PollerMetadata; import org.springframework.integration.sftp.SftpTestSupport; @@ -161,7 +160,7 @@ public class SftpStreamingMessageSourceTests extends SftpTestSupport { private SftpStreamingMessageSource buildSource() { SftpStreamingMessageSource messageSource = new SftpStreamingMessageSource(this.config.template(), - Comparator.comparing(FileInfo::getFilename)); + Comparator.comparing(LsEntry::getFilename)); messageSource.setRemoteDirectory("sftpSource/"); messageSource.setMaxFetchSize(1); messageSource.setBeanFactory(this.context); @@ -189,7 +188,7 @@ public class SftpStreamingMessageSourceTests extends SftpTestSupport { @InboundChannelAdapter(channel = "stream", autoStartup = "false") public MessageSource sftpMessageSource() { SftpStreamingMessageSource messageSource = new SftpStreamingMessageSource(template(), - Comparator.comparing(FileInfo::getFilename)); + Comparator.comparing(LsEntry::getFilename)); messageSource.setFilter(new AcceptAllFileListFilter<>()); messageSource.setRemoteDirectory("sftpSource/"); return messageSource; diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 15ee2634f0..f0db8d4430 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -567,6 +567,8 @@ If you set the `fileInfoJson` property on the `FtpStreamingMessageSource` to `fa The `FTPFile` object provided by the underlying Apache Net library can be accessed using the `FtpFileInfo.getFileInfo()` method. The `fileInfoJson` property is not available when using XML configuration but you can set it by injecting the `FtpStreamingMessageSource` into one of your configuration classes. +Starting with version 5.1, the `comparator` 's generic type is now `FTPFile`; previously it was `AbstractFileInfo`; this is because the sort is now performed earlier in the processing, before filtering and applying `maxFetch`. + ==== Configuring with Java Configuration The following Spring Boot application provides an example of configuring the inbound adapter using Java configuration: @@ -727,6 +729,8 @@ Another use for `max-fetch-size` is if you want to stop fetching remote files, b Setting the `maxFetchSize` property on the `MessageSource` (programmatically, via JMX, or via a <>) effectively stops the adapter from fetching more files, but allows the poller to continue to emit messages for files that have previously been fetched. If the poller is active when the property is changed, the change will take effect on the next poll. +Starting with version 5.1, the synchronizer can be provided with a `Comparator`. +This is useful when restricting the number of files fetched with `maxFetchSize`. [[ftp-outbound]] === FTP Outbound Channel Adapter diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index 47b803686b..409913a56c 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -612,6 +612,8 @@ If you set the `fileInfoJson` property on the `SftpStreamingMessageSource` to `f The `LsEntry` object provided by the underlying Jsch library can be accessed using the `SftpFileInfo.getFileInfo()` method. The `fileInfoJson` property is not available when using XML configuration but you can set it by injecting the `SftpStreamingMessageSource` into one of your configuration classes. +Starting with version 5.1, the `comparator` 's generic type is now `LsEntry`; previously it was `AbstractFileInfo`; this is because the sort is now performed earlier in the processing, before filtering and applying `maxFetch`. + ==== Configuring with Java Configuration The following Spring Boot application provides an example of configuring the inbound adapter using Java configuration: @@ -772,6 +774,8 @@ Another use for `max-fetch-size` is if you want to stop fetching remote files, b Setting the `maxFetchSize` property on the `MessageSource` (programmatically, via JMX, or via a <>) effectively stops the adapter from fetching more files, but allows the poller to continue to emit messages for files that have previously been fetched. If the poller is active when the property is changed, the change will take effect on the next poll. +Starting with version 5.1, the synchronizer can be provided with a `Comparator`. +This is useful when restricting the number of files fetched with `maxFetchSize`. [[sftp-outbound]] === SFTP Outbound Channel Adapter diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 629f02f1a2..864c5b7a03 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -85,3 +85,8 @@ A `RotatingServerAdvice` is now available to poll multiple servers and/or direct See <> and <> for more information. Also inbound adapter `localFilenameExpression` s can contain the variable `#remoteDirectory` which contains the remote directory being polled. + +The generic type of the comparators, used to sort the fetched file list for the streaming adapters, has changed from `Comparator>` to simply `Comparator`. +See <> and <> for more information. + +In addition, the synchronizers for inbound channel adapters can now be provided with a `Comparator`; this is useful when using `maxFetchSize` to limit the files retrieved.