From 4bb46246dfbb9f3a7a78facf497e377bfad8b7bf Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Thu, 12 Dec 2024 17:31:07 -0500 Subject: [PATCH] GH-9684: Auto-create temporary remote directory Fixes: #9684 Issue link: https://github.com/spring-projects/spring-integration/issues/9684 The `RemoteFileTemplate` is missing to create a temporary remote directory * Fix `RemoteFileTemplate.sendFileToRemoteDirectory()` to create a `temporaryRemoteDirectory` as well if it is different from already created `remoteDirectory` * Add `SftpServerOutboundTests.autoCreateTemporaryDirectory()` to verify that `temporaryRemoteDirectory` is created and in-use * Some other `SftpServerOutboundTests` refactoring for better code style (cherry picked from commit a4b193fa64b245a034955d52f0a12a942b4f5fa3) --- .../file/remote/RemoteFileTemplate.java | 7 + .../SftpServerOutboundTests-context.xml | 267 +++++++++--------- .../outbound/SftpServerOutboundTests.java | 93 +++--- 3 files changed, 195 insertions(+), 172 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java index 5b6e085d4f..28621adc83 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java @@ -570,10 +570,17 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ if (this.autoCreateDirectory) { try { RemoteFileUtils.makeDirectories(remoteDirectory, session, this.remoteFileSeparator, this.logger); + if (!temporaryRemoteDirectory.equals(remoteDirectory)) { + RemoteFileUtils.makeDirectories(temporaryRemoteDirectory, session, this.remoteFileSeparator, + this.logger); + } } catch (@SuppressWarnings("unused") IllegalStateException e) { // Revert to old FTP behavior if recursive mkdir fails, for backwards compatibility session.mkdir(remoteDirectory); + if (!temporaryRemoteDirectory.equals(remoteDirectory)) { + session.mkdir(temporaryRemoteDirectory); + } } } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml index 2e210d363b..fe67b37235 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml @@ -1,15 +1,15 @@ - + @@ -18,87 +18,88 @@ + request-channel="inboundGet" + command="get" + command-options="-P" + expression="payload" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory.toUpperCase()" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="invalidDirExpression" + command="get" + expression="payload" + local-directory-expression="T(java.io.File).separator + #remoteDirectory + '?:'" + reply-channel="output"/> + request-channel="inboundMGet" + command="mget" + command-options="-P" + expression="payload" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="inboundMGetRecursive" + command="mget" + expression="payload" + command-options="-R -P" + mode="REPLACE_IF_MODIFIED" + filter="dotStarDotTxtFilter" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="inboundLSRecursive" + command="ls" + expression="payload" + command-options="-R -dirs" + mode="REPLACE_IF_MODIFIED" + filter="dotStarDotTxtFilter" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="inboundLSRecursiveALL" + command="ls" + expression="payload" + command-options="-a -R -dirs" + mode="REPLACE_IF_MODIFIED" + filter="dotStarDotTxtFilter" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="inboundLSRecursiveNoDirs" + command="ls" + expression="payload" + command-options="-R" + mode="REPLACE_IF_MODIFIED" + filter="persistentFilter" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> - - + class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter"> + + - + @@ -107,89 +108,95 @@ + value="#{T(org.springframework.integration.file.remote.RemoteFileTestSupport).getScratchTempFolder().absolutePath}"/> + request-channel="inboundMGetRecursiveFiltered" + command="mget" + expression="payload" + command-options="-R" + filename-regex="(subSftpSource|.*1.txt)" + local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')" + reply-channel="output"/> + request-channel="inboundMPut" + command="mput" + auto-create-directory="true" + filename-pattern="*.txt" + expression="payload" + chmod="600" + remote-directory="sftpTarget" + reply-channel="output"/> + request-channel="inboundMPutRecursive" + command="mput" + command-options="-R" + auto-create-directory="true" + filename-pattern="*.txt" + expression="payload" + remote-directory="sftpTarget" + reply-channel="output"/> + request-channel="inboundMPutRecursiveFiltered" + command="mput" + command-options="-R" + mput-regex="(.*1.txt|sub.*)" + auto-create-directory="true" + filename-pattern="*.txt" + expression="payload" + remote-directory="sftpTarget" + reply-channel="output"/> - + + session-factory="sftpSessionFactory" + channel="appending" + mode="APPEND" + use-temporary-file-name="false" + remote-directory="sftpTarget" + auto-create-directory="true" + remote-file-separator="/"/> - + + session-factory="sftpSessionFactory" + channel="ignoring" + mode="IGNORE" + remote-directory="sftpTarget" + auto-create-directory="true" + remote-file-separator="/"/> - + + session-factory="sftpSessionFactory" + channel="failing" + mode="FAIL" + remote-directory="sftpTarget" + auto-create-directory="true" + remote-file-separator="/"/> + + @@ -198,23 +205,23 @@ + request-channel="inboundGetStream" + command="get" + command-options="-stream" + expression="payload" + remote-directory="ftpTarget" + reply-channel="stream"/> - + + channel="markers"/> + expression="payload.mark.toString().equals('END') ? headers['closeableResource'].close() : null"/> session = this.sessionFactory.getSession(); String dir = "sftpSource/"; long modified = setModifiedOnSource1(); this.inboundGet.send(new GenericMessage(dir + " sftpSource1.txt")); @@ -183,7 +183,6 @@ public class SftpServerOutboundTests extends SftpTestSupport { localFile = (File) result.getPayload(); assertThat(localFile.getPath().replaceAll(Matcher.quoteReplacement(File.separator), "/")) .contains(dir.toUpperCase()); - Session session2 = this.sessionFactory.getSession(); } @Test @@ -284,12 +283,12 @@ public class SftpServerOutboundTests extends SftpTestSupport { List files = (List) result.getPayload(); assertThat(files).hasSize(4); assertThat(files.stream() - .map(fi -> fi.getFilename()) - .collect(Collectors.toList())).contains( - " sftpSource1.txt", - "sftpSource2.txt", - "subSftpSource", - "subSftpSource/subSftpSource1.txt"); + .map(SftpFileInfo::getFilename)) + .contains( + " sftpSource1.txt", + "sftpSource2.txt", + "subSftpSource", + "subSftpSource/subSftpSource1.txt"); } @Test @@ -302,16 +301,16 @@ public class SftpServerOutboundTests extends SftpTestSupport { List files = (List) result.getPayload(); assertThat(files).hasSize(8); assertThat(files.stream() - .map(fi -> fi.getFilename()) - .collect(Collectors.toList())).contains( - " sftpSource1.txt", - "sftpSource2.txt", - "subSftpSource", - "subSftpSource/subSftpSource1.txt", - ".", - "..", - "subSftpSource/.", - "subSftpSource/.."); + .map(SftpFileInfo::getFilename)) + .contains( + " sftpSource1.txt", + "sftpSource2.txt", + "subSftpSource", + "subSftpSource/subSftpSource1.txt", + ".", + "..", + "subSftpSource/.", + "subSftpSource/.."); } @Test @@ -324,11 +323,11 @@ public class SftpServerOutboundTests extends SftpTestSupport { List files = (List) result.getPayload(); assertThat(files).hasSize(3); assertThat(files.stream() - .map(fi -> fi.getFilename()) - .collect(Collectors.toList())).contains( - " sftpSource1.txt", - "sftpSource2.txt", - "subSftpSource/subSftpSource1.txt"); + .map(SftpFileInfo::getFilename)) + .contains( + " sftpSource1.txt", + "sftpSource2.txt", + "subSftpSource/subSftpSource1.txt"); File newDeepFile = new File(this.sourceRemoteDirectory + "/subSftpSource/subSftpSource2.txt"); OutputStream fos = new FileOutputStream(newDeepFile); fos.write("test".getBytes()); @@ -383,12 +382,12 @@ public class SftpServerOutboundTests extends SftpTestSupport { ByteArrayOutputStream baos = new ByteArrayOutputStream(); FileCopyUtils.copy(session.readRaw("sftpSource/ sftpSource1.txt"), baos); assertThat(session.finalizeRaw()).isTrue(); - assertThat(new String(baos.toByteArray())).isEqualTo("source1"); + assertThat(baos.toString()).isEqualTo("source1"); baos = new ByteArrayOutputStream(); FileCopyUtils.copy(session.readRaw("sftpSource/sftpSource2.txt"), baos); assertThat(session.finalizeRaw()).isTrue(); - assertThat(new String(baos.toByteArray())).isEqualTo("source2"); + assertThat(baos.toString()).isEqualTo("source2"); session.close(); } @@ -407,12 +406,12 @@ public class SftpServerOutboundTests extends SftpTestSupport { ByteArrayOutputStream baos = new ByteArrayOutputStream(); FileCopyUtils.copy(session.readRaw("sftpTarget/sftpTarget2.txt"), baos); assertThat(session.finalizeRaw()).isTrue(); - assertThat(new String(baos.toByteArray())).isEqualTo("source2"); + assertThat(baos.toString()).isEqualTo("source2"); baos = new ByteArrayOutputStream(); FileCopyUtils.copy(session.readRaw("sftpSource/sftpSource2.txt"), baos); assertThat(session.finalizeRaw()).isTrue(); - assertThat(new String(baos.toByteArray())).isEqualTo("source2"); + assertThat(baos.toString()).isEqualTo("source2"); session.close(); } @@ -461,8 +460,8 @@ public class SftpServerOutboundTests extends SftpTestSupport { ByteArrayOutputStream bos2 = new ByteArrayOutputStream(); session1.read("foo.txt", bos1); session2.read("bar.txt", bos2); - assertThat(new String(bos1.toByteArray())).isEqualTo("ace"); - assertThat(new String(bos2.toByteArray())).isEqualTo("bdf"); + assertThat(bos1.toString()).isEqualTo("ace"); + assertThat(bos2.toString()).isEqualTo("bdf"); session1.remove("foo.txt"); session2.remove("bar.txt"); session1.close(); @@ -624,6 +623,21 @@ public class SftpServerOutboundTests extends SftpTestSupport { "sftpTarget/subLocalSource/subLocalSource1.txt"); } + @Test + public void autoCreateTemporaryDirectory() { + Message m = + MessageBuilder.withPayload("test") + .setHeader(FileHeaders.FILENAME, "test.txt") + .build(); + this.sendFileChannel.send(m); + + SftpClient.DirEntry[] files = this.template.execute(session -> session.list("sftpTarget")); + // ., .., and file itself + assertThat(files).hasSize(3).extracting(SftpClient.DirEntry::getFilename).contains("test.txt"); + files = this.template.execute(session -> session.list("sftpTarget_tmp")); + assertThat(files).hasSize(2); // . and .. + } + @Test public void testInt3412FileMode() { Message m = MessageBuilder.withPayload("foo") @@ -637,14 +651,9 @@ public class SftpServerOutboundTests extends SftpTestSupport { ignoring.send(m); assertLength6(template); - try { - failing.send(m); - fail("Expected exception"); - } - catch (MessagingException e) { - assertThat(e.getCause().getCause().getMessage()).contains("The destination file already exists"); - } - + assertThatExceptionOfType(MessagingException.class) + .isThrownBy(() -> failing.send(m)) + .withStackTraceContaining("The destination file already exists"); } @Test @@ -680,9 +689,9 @@ public class SftpServerOutboundTests extends SftpTestSupport { assertThat(receive.getPayload()).isEqualTo("FOO"); } - private void assertLength6(SftpRemoteFileTemplate template) { + private static void assertLength6(SftpRemoteFileTemplate template) { SftpClient.DirEntry[] files = template.execute(session -> session.list("sftpTarget")); - assertThat(files.length).isEqualTo(3); + assertThat(files).hasSize(3); assertThat(files[2].getFilename()).isEqualTo("appending.txt"); assertThat(files[2].getAttributes().getSize()).isEqualTo(6); } @@ -756,7 +765,7 @@ public class SftpServerOutboundTests extends SftpTestSupport { public MessageChannel eventChannel() { return (msg, timeout) -> { if (this.latch != null) { - if (this.events.size() > 0 || msg.getPayload() instanceof SessionOpenedEvent) { + if (!this.events.isEmpty() || msg.getPayload() instanceof SessionOpenedEvent) { this.events.add((ApacheMinaSftpEvent) msg.getPayload()); if (msg.getPayload() instanceof SessionClosedEvent) { this.latch.countDown();