GH-9684: Auto-create temporary remote directory

Fixes: #9684
Issue link: https://github.com/spring-projects/spring-integration/issues/9684

The `RemoteFileTemplate` is missing to create a temporary remote directory

* Fix `RemoteFileTemplate.sendFileToRemoteDirectory()` to create a `temporaryRemoteDirectory`
as well if it is different from already created `remoteDirectory`
* Add `SftpServerOutboundTests.autoCreateTemporaryDirectory()` to verify that `temporaryRemoteDirectory` is created and in-use
* Some other `SftpServerOutboundTests` refactoring for better code style

(cherry picked from commit a4b193fa64)
This commit is contained in:
Artem Bilan
2024-12-12 17:31:07 -05:00
committed by Spring Builds
parent d8fb02306f
commit 4bb46246df
3 changed files with 195 additions and 172 deletions

View File

@@ -570,10 +570,17 @@ public class RemoteFileTemplate<F> implements RemoteFileOperations<F>, Initializ
if (this.autoCreateDirectory) {
try {
RemoteFileUtils.makeDirectories(remoteDirectory, session, this.remoteFileSeparator, this.logger);
if (!temporaryRemoteDirectory.equals(remoteDirectory)) {
RemoteFileUtils.makeDirectories(temporaryRemoteDirectory, session, this.remoteFileSeparator,
this.logger);
}
}
catch (@SuppressWarnings("unused") IllegalStateException e) {
// Revert to old FTP behavior if recursive mkdir fails, for backwards compatibility
session.mkdir(remoteDirectory);
if (!temporaryRemoteDirectory.equals(remoteDirectory)) {
session.mkdir(temporaryRemoteDirectory);
}
}
}

View File

@@ -1,15 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-file="http://www.springframework.org/schema/integration/file"
xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp https://www.springframework.org/schema/integration/sftp/spring-integration-sftp.xsd
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-sftp="http://www.springframework.org/schema/integration/sftp"
xmlns:int="http://www.springframework.org/schema/integration"
xmlns:int-file="http://www.springframework.org/schema/integration/file"
xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp https://www.springframework.org/schema/integration/sftp/spring-integration-sftp.xsd
http://www.springframework.org/schema/beans https://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/file https://www.springframework.org/schema/integration/file/spring-integration-file.xsd
http://www.springframework.org/schema/integration https://www.springframework.org/schema/integration/spring-integration.xsd">
<bean id="extraConfig" class="org.springframework.integration.sftp.outbound.SftpServerOutboundTests$Config" />
<bean id="extraConfig" class="org.springframework.integration.sftp.outbound.SftpServerOutboundTests$Config"/>
<int:channel id="output">
<int:queue/>
@@ -18,87 +18,88 @@
<int:channel id="inboundGet"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundGet"
command="get"
command-options="-P"
expression="payload"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory.toUpperCase()"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundGet"
command="get"
command-options="-P"
expression="payload"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory.toUpperCase()"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="invalidDirExpression"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="invalidDirExpression"
command="get"
expression="payload"
local-directory-expression="T(java.io.File).separator + #remoteDirectory + '?:'"
reply-channel="output"/>
request-channel="invalidDirExpression"
command="get"
expression="payload"
local-directory-expression="T(java.io.File).separator + #remoteDirectory + '?:'"
reply-channel="output"/>
<int:channel id="inboundMGet"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMGet"
command="mget"
command-options="-P"
expression="payload"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundMGet"
command="mget"
command-options="-P"
expression="payload"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMGetRecursive"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMGetRecursive"
command="mget"
expression="payload"
command-options="-R -P"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundMGetRecursive"
command="mget"
expression="payload"
command-options="-R -P"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursive"
command="ls"
expression="payload"
command-options="-R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundLSRecursive"
command="ls"
expression="payload"
command-options="-R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursiveALL"
command="ls"
expression="payload"
command-options="-a -R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundLSRecursiveALL"
command="ls"
expression="payload"
command-options="-a -R -dirs"
mode="REPLACE_IF_MODIFIED"
filter="dotStarDotTxtFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundLSRecursiveNoDirs"
command="ls"
expression="payload"
command-options="-R"
mode="REPLACE_IF_MODIFIED"
filter="persistentFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundLSRecursiveNoDirs"
command="ls"
expression="payload"
command-options="-R"
mode="REPLACE_IF_MODIFIED"
filter="persistentFilter"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<bean id="dotStarDotTxtFilter"
class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
<constructor-arg value="^.*\.txt$" />
<property name="alwaysAcceptDirectories" value="true" />
class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
<constructor-arg value="^.*\.txt$"/>
<property name="alwaysAcceptDirectories" value="true"/>
</bean>
<bean id="persistentFilter" class="org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter">
<bean id="persistentFilter"
class="org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter">
<constructor-arg ref="store"/>
<constructor-arg value="test"/>
<property name="forRecursion" value="true"/>
@@ -107,89 +108,95 @@
<bean id="store" class="org.springframework.integration.metadata.PropertiesPersistingMetadataStore">
<property name="baseDirectory"
value="#{T(org.springframework.integration.file.remote.RemoteFileTestSupport).getScratchTempFolder().absolutePath}"/>
value="#{T(org.springframework.integration.file.remote.RemoteFileTestSupport).getScratchTempFolder().absolutePath}"/>
</bean>
<int:channel id="inboundMGetRecursiveFiltered"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMGetRecursiveFiltered"
command="mget"
expression="payload"
command-options="-R"
filename-regex="(subSftpSource|.*1.txt)"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
request-channel="inboundMGetRecursiveFiltered"
command="mget"
expression="payload"
command-options="-R"
filename-regex="(subSftpSource|.*1.txt)"
local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
reply-channel="output"/>
<int:channel id="inboundMPut"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMPut"
command="mput"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
chmod="600"
remote-directory="sftpTarget"
reply-channel="output"/>
request-channel="inboundMPut"
command="mput"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
chmod="600"
remote-directory="sftpTarget"
reply-channel="output"/>
<int:channel id="inboundMPutRecursive"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMPutRecursive"
command="mput"
command-options="-R"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="sftpTarget"
reply-channel="output"/>
request-channel="inboundMPutRecursive"
command="mput"
command-options="-R"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="sftpTarget"
reply-channel="output"/>
<int:channel id="inboundMPutRecursiveFiltered"/>
<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
request-channel="inboundMPutRecursiveFiltered"
command="mput"
command-options="-R"
mput-regex="(.*1.txt|sub.*)"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="sftpTarget"
reply-channel="output"/>
request-channel="inboundMPutRecursiveFiltered"
command="mput"
command-options="-R"
mput-regex="(.*1.txt|sub.*)"
auto-create-directory="true"
filename-pattern="*.txt"
expression="payload"
remote-directory="sftpTarget"
reply-channel="output"/>
<int:channel id="appending" />
<int:channel id="appending"/>
<int-sftp:outbound-channel-adapter id="appender"
session-factory="sftpSessionFactory"
channel="appending"
mode="APPEND"
use-temporary-file-name="false"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/" />
session-factory="sftpSessionFactory"
channel="appending"
mode="APPEND"
use-temporary-file-name="false"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/"/>
<int:channel id="ignoring" />
<int:channel id="ignoring"/>
<int-sftp:outbound-channel-adapter id="ignore"
session-factory="sftpSessionFactory"
channel="ignoring"
mode="IGNORE"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/" />
session-factory="sftpSessionFactory"
channel="ignoring"
mode="IGNORE"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/"/>
<int:channel id="failing" />
<int:channel id="failing"/>
<int-sftp:outbound-channel-adapter id="fail"
session-factory="sftpSessionFactory"
channel="failing"
mode="FAIL"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/" />
session-factory="sftpSessionFactory"
channel="failing"
mode="FAIL"
remote-directory="sftpTarget"
auto-create-directory="true"
remote-file-separator="/"/>
<int-sftp:outbound-channel-adapter id="sendFileChannel"
session-factory="sftpSessionFactory"
remote-directory="sftpTarget"
temporary-remote-directory="sftpTarget_tmp"
auto-create-directory="true"/>
<bean id="cachingSessionFactory" class="org.springframework.integration.file.remote.session.CachingSessionFactory">
<constructor-arg name="sessionFactory" ref="sftpSessionFactory"/>
@@ -198,23 +205,23 @@
</bean>
<int-sftp:outbound-gateway session-factory="cachingSessionFactory"
request-channel="inboundGetStream"
command="get"
command-options="-stream"
expression="payload"
remote-directory="ftpTarget"
reply-channel="stream" />
request-channel="inboundGetStream"
command="get"
command-options="-stream"
expression="payload"
remote-directory="ftpTarget"
reply-channel="stream"/>
<int:chain input-channel="stream">
<int-file:splitter markers="true" />
<int-file:splitter markers="true"/>
<int:payload-type-router resolution-required="false" default-output-channel="output">
<int:mapping type="org.springframework.integration.file.splitter.FileSplitter$FileMarker"
channel="markers" />
channel="markers"/>
</int:payload-type-router>
</int:chain>
<int:service-activator input-channel="markers"
expression="payload.mark.toString().equals('END') ? headers['closeableResource'].close() : null"/>
expression="payload.mark.toString().equals('END') ? headers['closeableResource'].close() : null"/>
<int-sftp:outbound-gateway
session-factory="sftpSessionFactory"

View File

@@ -32,7 +32,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.sshd.sftp.client.SftpClient;
@@ -76,7 +75,6 @@ import org.springframework.util.FileCopyUtils;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.assertj.core.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
@@ -149,6 +147,9 @@ public class SftpServerOutboundTests extends SftpTestSupport {
@Autowired
private DirectChannel inboundCallback;
@Autowired
private DirectChannel sendFileChannel;
@Autowired
private Config config;
@@ -165,7 +166,6 @@ public class SftpServerOutboundTests extends SftpTestSupport {
@Test
public void testInt2866LocalDirectoryExpressionGET() {
Session<?> session = this.sessionFactory.getSession();
String dir = "sftpSource/";
long modified = setModifiedOnSource1();
this.inboundGet.send(new GenericMessage<Object>(dir + " sftpSource1.txt"));
@@ -183,7 +183,6 @@ public class SftpServerOutboundTests extends SftpTestSupport {
localFile = (File) result.getPayload();
assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/"))
.contains(dir.toUpperCase());
Session<?> session2 = this.sessionFactory.getSession();
}
@Test
@@ -284,12 +283,12 @@ public class SftpServerOutboundTests extends SftpTestSupport {
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(4);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt");
.map(SftpFileInfo::getFilename))
.contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt");
}
@Test
@@ -302,16 +301,16 @@ public class SftpServerOutboundTests extends SftpTestSupport {
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(8);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt",
".",
"..",
"subSftpSource/.",
"subSftpSource/..");
.map(SftpFileInfo::getFilename))
.contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource",
"subSftpSource/subSftpSource1.txt",
".",
"..",
"subSftpSource/.",
"subSftpSource/..");
}
@Test
@@ -324,11 +323,11 @@ public class SftpServerOutboundTests extends SftpTestSupport {
List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
assertThat(files).hasSize(3);
assertThat(files.stream()
.map(fi -> fi.getFilename())
.collect(Collectors.toList())).contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource/subSftpSource1.txt");
.map(SftpFileInfo::getFilename))
.contains(
" sftpSource1.txt",
"sftpSource2.txt",
"subSftpSource/subSftpSource1.txt");
File newDeepFile = new File(this.sourceRemoteDirectory + "/subSftpSource/subSftpSource2.txt");
OutputStream fos = new FileOutputStream(newDeepFile);
fos.write("test".getBytes());
@@ -383,12 +382,12 @@ public class SftpServerOutboundTests extends SftpTestSupport {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpSource/ sftpSource1.txt"), baos);
assertThat(session.finalizeRaw()).isTrue();
assertThat(new String(baos.toByteArray())).isEqualTo("source1");
assertThat(baos.toString()).isEqualTo("source1");
baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpSource/sftpSource2.txt"), baos);
assertThat(session.finalizeRaw()).isTrue();
assertThat(new String(baos.toByteArray())).isEqualTo("source2");
assertThat(baos.toString()).isEqualTo("source2");
session.close();
}
@@ -407,12 +406,12 @@ public class SftpServerOutboundTests extends SftpTestSupport {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpTarget/sftpTarget2.txt"), baos);
assertThat(session.finalizeRaw()).isTrue();
assertThat(new String(baos.toByteArray())).isEqualTo("source2");
assertThat(baos.toString()).isEqualTo("source2");
baos = new ByteArrayOutputStream();
FileCopyUtils.copy(session.readRaw("sftpSource/sftpSource2.txt"), baos);
assertThat(session.finalizeRaw()).isTrue();
assertThat(new String(baos.toByteArray())).isEqualTo("source2");
assertThat(baos.toString()).isEqualTo("source2");
session.close();
}
@@ -461,8 +460,8 @@ public class SftpServerOutboundTests extends SftpTestSupport {
ByteArrayOutputStream bos2 = new ByteArrayOutputStream();
session1.read("foo.txt", bos1);
session2.read("bar.txt", bos2);
assertThat(new String(bos1.toByteArray())).isEqualTo("ace");
assertThat(new String(bos2.toByteArray())).isEqualTo("bdf");
assertThat(bos1.toString()).isEqualTo("ace");
assertThat(bos2.toString()).isEqualTo("bdf");
session1.remove("foo.txt");
session2.remove("bar.txt");
session1.close();
@@ -624,6 +623,21 @@ public class SftpServerOutboundTests extends SftpTestSupport {
"sftpTarget/subLocalSource/subLocalSource1.txt");
}
@Test
public void autoCreateTemporaryDirectory() {
Message<String> m =
MessageBuilder.withPayload("test")
.setHeader(FileHeaders.FILENAME, "test.txt")
.build();
this.sendFileChannel.send(m);
SftpClient.DirEntry[] files = this.template.execute(session -> session.list("sftpTarget"));
// ., .., and file itself
assertThat(files).hasSize(3).extracting(SftpClient.DirEntry::getFilename).contains("test.txt");
files = this.template.execute(session -> session.list("sftpTarget_tmp"));
assertThat(files).hasSize(2); // . and ..
}
@Test
public void testInt3412FileMode() {
Message<String> m = MessageBuilder.withPayload("foo")
@@ -637,14 +651,9 @@ public class SftpServerOutboundTests extends SftpTestSupport {
ignoring.send(m);
assertLength6(template);
try {
failing.send(m);
fail("Expected exception");
}
catch (MessagingException e) {
assertThat(e.getCause().getCause().getMessage()).contains("The destination file already exists");
}
assertThatExceptionOfType(MessagingException.class)
.isThrownBy(() -> failing.send(m))
.withStackTraceContaining("The destination file already exists");
}
@Test
@@ -680,9 +689,9 @@ public class SftpServerOutboundTests extends SftpTestSupport {
assertThat(receive.getPayload()).isEqualTo("FOO");
}
private void assertLength6(SftpRemoteFileTemplate template) {
private static void assertLength6(SftpRemoteFileTemplate template) {
SftpClient.DirEntry[] files = template.execute(session -> session.list("sftpTarget"));
assertThat(files.length).isEqualTo(3);
assertThat(files).hasSize(3);
assertThat(files[2].getFilename()).isEqualTo("appending.txt");
assertThat(files[2].getAttributes().getSize()).isEqualTo(6);
}
@@ -756,7 +765,7 @@ public class SftpServerOutboundTests extends SftpTestSupport {
public MessageChannel eventChannel() {
return (msg, timeout) -> {
if (this.latch != null) {
if (this.events.size() > 0 || msg.getPayload() instanceof SessionOpenedEvent) {
if (!this.events.isEmpty() || msg.getPayload() instanceof SessionOpenedEvent) {
this.events.add((ApacheMinaSftpEvent) msg.getPayload());
if (msg.getPayload() instanceof SessionClosedEvent) {
this.latch.countDown();