Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3315: Fix (S)FTP Stream with Fair Rotation

Browse Source 
Resolves https://github.com/spring-projects/spring-integration/issues/3315

`maxFetchSize` ignored with filters supporting single file filtering (default).
This breaks "fair" rotation with the `RotatingServerAdvice`.

Honor `maxFetchSize`, even with filters that support single file filtering.

**cherry-pick to 5.3.x, 5.2.x**
This commit is contained in:
Gary Russell
2020-07-01 11:18:00 -04:00
committed by Artem Bilan
parent 0ddbfa6e6d
commit f3d61ec44a
3 changed files with 131 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/RotatingServersTests.java
View File

@@ -37,13 +37,15 @@ import org.junit.jupiter.api.TestInfo;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.StaticMessageHeaderAccessor;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.dsl.IntegrationFlows;
import org.springframework.integration.dsl.MessageChannels;
import org.springframework.integration.dsl.Pollers;
import org.springframework.integration.dsl.StandardIntegrationFlow;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.remote.aop.RotatingServerAdvice;
import org.springframework.integration.file.remote.aop.RotationPolicy;
import org.springframework.integration.file.remote.session.CachingSessionFactory;
@@ -58,6 +60,7 @@ import org.springframework.integration.ftp.dsl.Ftp;
import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
import org.springframework.integration.metadata.SimpleMetadataStore;
import org.springframework.messaging.Message;
/**
* @author Gary Russell
@@ -88,18 +91,49 @@ public class RotatingServersTests extends FtpTestSupport {
});
}
@BeforeEach
public void extraSetup(TestInfo info) {
if (info.getTestMethod().get().getName().equals("testFairStreaming")) {
FtpRemoteFileTemplate rft = new FtpRemoteFileTemplate(sessionFactory());
rft.execute(s -> {
ByteArrayInputStream bais = new ByteArrayInputStream("foo".getBytes());
// 2 files per server, remove empty dirs
s.write(bais, "foo/f4");
s.write(bais, "baz/f5");
s.write(bais, "fiz/f6");
s.rmdir("bar");
s.rmdir("qux");
s.rmdir("buz");
return null;
});
}
}
@BeforeEach
@AfterEach
public void clean(TestInfo info) {
recursiveDelete(new File(tmpDir), info);
}
@AfterEach
public void extraCleanUp(TestInfo info) {
if (info.getTestMethod().get().getName().equals("testFairStreaming")) {
FtpRemoteFileTemplate rft = new FtpRemoteFileTemplate(sessionFactory());
rft.execute(s -> {
s.remove("foo/f4");
s.remove("baz/f5");
s.remove("fiz/f6");
return null;
});
}
}
@Test
public void testStandard() throws Exception {
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(StandardConfig.class);
StandardConfig config = ctx.getBean(StandardConfig.class);
ctx.getBean(StandardIntegrationFlow.class).stop();
assertThat(config.latch.await(10, TimeUnit.SECONDS)).isTrue();
ctx.getBean(SourcePollingChannelAdapter.class).stop();
List<Integer> sfCalls = config.sessionSources.stream().limit(17).collect(Collectors.toList());
assertThat(sfCalls).containsExactly(1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 1, 2, 2, 3, 3, 1, 1);
File f1 = new File(tmpDir + File.separator + "standard" + File.separator + "f1");
@@ -116,8 +150,8 @@ public class RotatingServersTests extends FtpTestSupport {
public void testFair() throws Exception {
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(FairConfig.class);
StandardConfig config = ctx.getBean(StandardConfig.class);
ctx.getBean(StandardIntegrationFlow.class).stop();
assertThat(config.latch.await(10, TimeUnit.SECONDS)).isTrue();
ctx.getBean(SourcePollingChannelAdapter.class).stop();
List<Integer> sfCalls = config.sessionSources.stream().limit(17).collect(Collectors.toList());
assertThat(sfCalls).containsExactly(1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3);
File f1 = new File(tmpDir + File.separator + "fair" + File.separator + "f1");
@@ -135,7 +169,7 @@ public class RotatingServersTests extends FtpTestSupport {
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(VariableLocalConfig.class);
StandardConfig config = ctx.getBean(StandardConfig.class);
assertThat(config.latch.await(10, TimeUnit.SECONDS)).isTrue();
ctx.getBean(StandardIntegrationFlow.class).stop();
ctx.getBean(SourcePollingChannelAdapter.class).stop();
List<Integer> sfCalls = config.sessionSources.stream().limit(17).collect(Collectors.toList());
assertThat(sfCalls).containsExactly(1, 1, 1, 2, 2, 2, 3, 3, 3, 1, 1, 2, 2, 3, 3, 1, 1);
File f1 = new File(tmpDir + File.separator + "variable" + File.separator + "foo" + File.separator + "f1");
@@ -152,15 +186,59 @@ public class RotatingServersTests extends FtpTestSupport {
public void testStreaming() throws Exception {
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(StreamingConfig.class);
StandardConfig config = ctx.getBean(StandardConfig.class);
ctx.getBean(StandardIntegrationFlow.class).stop();
assertThat(config.latch.await(10, TimeUnit.SECONDS)).isTrue();
ctx.getBean(SourcePollingChannelAdapter.class).stop();
List<Integer> sfCalls = config.sessionSources.stream().limit(17).collect(Collectors.toList());
// there's an extra getSession() with this adapter in listFiles
assertThat(sfCalls).containsExactly(1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 1, 1, 2, 2, 3);
assertThat(ctx.getBean("files", QueueChannel.class).getQueueSize()).isEqualTo(3);
QueueChannel files = ctx.getBean("files", QueueChannel.class);
assertThat(files.getQueueSize()).isEqualTo(3);
Message<?> received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f1");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f2");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f3");
ctx.close();
}
@Test
public void testFairStreaming() throws Exception {
AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(FairStreamingConfig.class);
try {
StandardConfig config = ctx.getBean(StandardConfig.class);
assertThat(config.latch.await(10, TimeUnit.SECONDS)).isTrue();
ctx.getBean(SourcePollingChannelAdapter.class).stop();
List<Integer> sfCalls = config.sessionSources.stream().limit(17).collect(Collectors.toList());
assertThat(sfCalls).containsExactly(1, 1, 2, 2, 3, 3, 1, 1, 2, 2, 3, 3, 1, 2, 3, 1, 2);
QueueChannel files = ctx.getBean("files", QueueChannel.class);
assertThat(files.getQueueSize()).isEqualTo(6);
Message<?> received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f1");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f2");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f3");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f4");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f5");
received = files.receive(0);
StaticMessageHeaderAccessor.getCloseableResource(received).close();
assertThat(received.getHeaders().get(FileHeaders.REMOTE_FILE, String.class)).isEqualTo("f6");
}
finally {
ctx.close();
}
}
@Configuration
@EnableIntegration
@@ -307,4 +385,36 @@ public class RotatingServersTests extends FtpTestSupport {
}
@Configuration
public static class FairStreamingConfig extends StandardConfig {
@Override
@Bean
public RotatingServerAdvice advice() {
List<RotationPolicy.KeyDirectory> keyDirectories = new ArrayList<>();
keyDirectories.add(new RotationPolicy.KeyDirectory("one", "foo"));
keyDirectories.add(new RotationPolicy.KeyDirectory("two", "baz"));
keyDirectories.add(new RotationPolicy.KeyDirectory("three", "fiz"));
return theAdvice(keyDirectories);
}
@Override
@Bean
public IntegrationFlow flow() {
return IntegrationFlows.from(Ftp.inboundStreamingAdapter(new FtpRemoteFileTemplate(sf()))
.filter(new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "rotate"))
.remoteDirectory(".")
.maxFetchSize(1),
e -> e.poller(Pollers.fixedDelay(1).advice(advice())))
.channel(MessageChannels.queue("files"))
.get();
}
@Override
protected RotatingServerAdvice theAdvice(List<RotationPolicy.KeyDirectory> keyDirectories) {
return new RotatingServerAdvice(sf(), keyDirectories, true);
}
}
}