Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-8792: Fix File StreamingMS for one file filter

Browse Source 
Fixes https://github.com/spring-projects/spring-integration/issues/8792

1. Use a `SftpStreamingMessageSource` with a `maxFetchSize = 5` and a `ChainFileListFilter` filter composed with
`SftpSystemMarkerFilePresentFileListFilter` which `supportsSingleFileFiltering == false`
2. Put 2 files in the folder and invoke `SftpStreamingMessageSource.receive()` method twice.
3. Put 5 files in the folder and invoke `SftpStreamingMessageSource.receive()`` method five times.
4. The last two files won't be received.

When you set max fetch size to a number bigger than one (for example 5) and at a certain point
it is necessary to `this.toBeReceived.clear()` inside `AbstractRemoteFileStreamingMessageSource.doReceive()`,
those removed elements from toBeReceived are not rolled back.

* Fix `AbstractRemoteFileStreamingMessageSource.listFiles()` to calculate `maxFetchSize` as `getMaxFetchSize() - this.fetched.get()`

**Cherry-pick to `6.1.x`, `6.0.x` & `5.5.x`**

(cherry picked from commit 22c4db291b)

# Conflicts:
#	spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java
This commit is contained in:
Artem Bilan
2023-11-17 16:39:02 -05:00
parent 430ef9bcae
commit 5c849c5b0e
2 changed files with 73 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java
View File

@@ -215,7 +215,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource<F>
if (this.filter != null && this.filter.supportsSingleFileFiltering()
&& !this.filter.accept(file.getFileInfo())) {
if (this.toBeReceived.size() > 0) { // don't re-fetch already filtered files
if (!this.toBeReceived.isEmpty()) { // don't re-fetch already filtered files
file = poll();
continue;
}
@@ -267,7 +267,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource<F>
}
protected AbstractFileInfo<F> poll() {
if (this.toBeReceived.size() == 0) {
if (this.toBeReceived.isEmpty()) {
listFiles();
}
return this.toBeReceived.poll();
@@ -297,7 +297,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource<F>
if (!ObjectUtils.isEmpty(files)) {
List<AbstractFileInfo<F>> fileInfoList;
if (this.filter != null && !this.filter.supportsSingleFileFiltering()) {
int maxFetchSize = getMaxFetchSize();
int maxFetchSize = getMaxFetchSize() - this.fetched.get();
List<F> filteredFiles = this.filter.filterFiles(files);
if (maxFetchSize > 0 && filteredFiles.size() > maxFetchSize) {
rollbackFromFileToListEnd(filteredFiles, filteredFiles.get(maxFetchSize));