Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GH-3482: (S)FTP: Fix Recursive LS (ARFOG)

Browse Source 
Resolves https://github.com/spring-projects/spring-integration/issues/3482

`.` and `..` should be ignored when recursing.

**cherry-pick to 5.4.x, 5.3.x**

* Fix checkstyle.

* Fix test in `file` module - test was incorrect; it would have detected this problem.
This commit is contained in:
Gary Russell
2021-02-01 16:04:08 -05:00
committed by GitHub
parent 98fbf62db2
commit d7740706bb
4 changed files with 116 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View File

@@ -974,18 +974,23 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
private void processFile(Session<F> session, String directory, String subDirectory, List<F> lsFiles,
boolean recursion, F file) throws IOException {
String fileName = getFilename(file);
String fileSep = this.remoteFileTemplate.getRemoteFileSeparator();
boolean isDots = ".".equals(fileName)
|| "..".equals(fileName)
|| fileName.endsWith(fileSep + ".")
|| fileName.endsWith(fileSep + "..");
if (this.options.contains(Option.SUBDIRS) || !isDirectory(file)) {
if (recursion && StringUtils.hasText(subDirectory)) {
if (recursion && StringUtils.hasText(subDirectory) && (!isDots || this.options.contains(Option.ALL))) {
lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory));
}
else {
else if (this.options.contains(Option.ALL) || !isDots) {
lsFiles.add(file);
}
}
String fileName = getFilename(file);
if (recursion && isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) {
if (recursion && isDirectory(file) && !isDots) {
lsFiles.addAll(listFilesInRemoteDir(session, directory,
subDirectory + fileName + this.remoteFileTemplate.getRemoteFileSeparator()));
subDirectory + fileName + fileSep));
}
}

7
spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View File

@@ -406,12 +406,13 @@ public class RemoteFileOutboundGatewayTests {
MessageBuilder<List<TestLsEntry>> out = (MessageBuilder<List<TestLsEntry>>) gw
.handleRequestMessage(new GenericMessage<>("testremote/x"));
assertThat(out).isNotNull();
assertThat(out.getPayload()).hasSize(5);
assertThat(out.getPayload()).hasSize(6);
assertThat(out.getPayload().get(0).getFilename()).isEqualTo("f1");
assertThat(out.getPayload().get(1).getFilename()).isEqualTo("d1");
assertThat(out.getPayload().get(2).getFilename()).isEqualTo("d1/d2");
assertThat(out.getPayload().get(3).getFilename()).isEqualTo("d1/f3");
assertThat(out.getPayload().get(4).getFilename()).isEqualTo("f2");
assertThat(out.getPayload().get(3).getFilename()).isEqualTo("d1/d2/f4");
assertThat(out.getPayload().get(4).getFilename()).isEqualTo("d1/f3");
assertThat(out.getPayload().get(5).getFilename()).isEqualTo("f2");
assertThat(out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)).isEqualTo("testremote/x/");
}

33
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml
View File

@@ -59,6 +59,39 @@
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursive"
command="ls"
expression="payload"
command-options="-R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursiveALL"
command="ls"
expression="payload"
command-options="-a -R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursiveNoDirs"
command="ls"
expression="payload"
command-options="-R"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<bean id="dotStarDotTxtFilter"
class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
<constructor-arg value="^.*\.txt$" />

70
spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2013-2020 the original author or authors.
* Copyright 2013-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.BeforeEach;
@@ -60,6 +61,7 @@ import org.springframework.integration.sftp.server.PathRemovedEvent;
import org.springframework.integration.sftp.server.SessionClosedEvent;
import org.springframework.integration.sftp.server.SessionOpenedEvent;
import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
import org.springframework.integration.sftp.session.SftpFileInfo;
import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.integration.test.util.TestUtils;
@@ -100,6 +102,15 @@ public class SftpServerOutboundTests extends SftpTestSupport {
@Autowired
private DirectChannel inboundMGetRecursive;
@Autowired
private DirectChannel inboundLSRecursive;
@Autowired
private DirectChannel inboundLSRecursiveALL;
@Autowired
private DirectChannel inboundLSRecursiveNoDirs;
@Autowired
private DirectChannel inboundMGetRecursiveFiltered;
@@ -254,6 +265,63 @@ public class SftpServerOutboundTests extends SftpTestSupport {
FileUtils.copyInputStreamToFile(new ByteArrayInputStream(localAsString.getBytes()), secondRemote);
}
@Test
@SuppressWarnings("unchecked")
void testLSRecursive() throws IOException {
String dir = "sftpSource/";
this.inboundLSRecursive.send(new GenericMessage<Object>(dir));
Message<?> result = this.output.receive(1000);
assertThat(result).isNotNull();
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(4);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt");
}
@Test
@SuppressWarnings("unchecked")
void testLSRecursiveALL() throws IOException {
String dir = "sftpSource/";
this.inboundLSRecursiveALL.send(new GenericMessage<Object>(dir));
Message<?> result = this.output.receive(1000);
assertThat(result).isNotNull();
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(8);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt",
".",
"..",
"subSftpSource/.",
"subSftpSource/..");
}
@Test
@SuppressWarnings("unchecked")
void testLSRecursiveNoDirs() throws IOException {
String dir = "sftpSource/";
this.inboundLSRecursiveNoDirs.send(new GenericMessage<Object>(dir));
Message<?> result = this.output.receive(1000);
assertThat(result).isNotNull();
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(3);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource/subSftpSource1.txt");
}
private long setModifiedOnSource1() {
File firstRemote = new File(getSourceRemoteDirectory(), " sftpSource1.txt");
firstRemote.setLastModified(System.currentTimeMillis() - 1_000_000);