From c62043891302e3eabe1b27150f46082396651b81 Mon Sep 17 00:00:00 2001 From: Artem Bilan Date: Mon, 6 Jun 2016 12:25:26 -0400 Subject: [PATCH] INT-4046: Add `FtpRemoteFileTemplate.ExistsMode` JIRA: https://jira.spring.io/browse/INT-4046 Since not all FTP servers provide proper `STAT` command implementation, plus the `NLIST` doesn't work properly for directories cases, introduce the `FtpRemoteFileTemplate.ExistsMode` to let: * to perform `STAT` by default (previous) behavior; * to switch to `NLIST` for `FtpRemoteFileTemplate` internal use; * perform the full `NLIST` and `FTPClient.changeWorkingDirectory()` algorithm if needed. * Improve (S)Ftp components to use proper `RemoteFileTemplate` for internal instantiation * Introduce `FtpMessageHandler` to wrap `FtpRemoteFileTemplate` with the proper `NLIST` `ExistsMode` * Cover `NLIST` switching from the `FtpOutboundChannelAdapterParser` and `FtpOutboundGatewayParser` * Document the `FtpRemoteFileTemplate.ExistsMode` * Fix typo in the recently introduced `RemoteFileOperations.getSession()` method name * Add JavaDoc to `Session.exists()` * Add `NLIST` support for the `FtpSession.exists()` to meet the API requirements **Cherry-pick to 4.2.x and 4.1.x** Addressing PR comments Doc Polishing --- ...tractRemoteFileStreamingMessageSource.java | 2 +- .../file/remote/RemoteFileOperations.java | 2 +- .../file/remote/RemoteFileTemplate.java | 12 ++- .../FileTransferringMessageHandler.java | 2 +- .../file/remote/session/Session.java | 6 ++ .../file/remote/RemoteFileTemplateTests.java | 29 ++----- .../FtpOutboundChannelAdapterParser.java | 24 ++++++ .../ftp/config/FtpOutboundGatewayParser.java | 17 ++++ .../ftp/gateway/FtpOutboundGateway.java | 7 +- .../ftp/outbound/FtpMessageHandler.java | 50 ++++++++++++ .../ftp/outbound/package-info.java | 4 + .../ftp/session/FtpRemoteFileTemplate.java | 79 ++++++++++++++++++- .../integration/ftp/session/FtpSession.java | 24 +++--- .../FtpOutboundChannelAdapterParserTests.java | 6 +- .../config/FtpOutboundGatewayParserTests.java | 13 ++- .../ftp/outbound/FtpServerOutboundTests.java | 3 +- .../session/FtpRemoteFileTemplateTests.java | 4 +- .../sftp/gateway/SftpOutboundGateway.java | 5 +- .../sftp/outbound/SftpMessageHandler.java | 2 +- .../sftp/outbound/package-info.java | 2 +- .../sftp/session/SftpRemoteFileTemplate.java | 19 ----- src/reference/asciidoc/ftp.adoc | 16 ++++ 22 files changed, 255 insertions(+), 73 deletions(-) create mode 100644 spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/FtpMessageHandler.java create mode 100644 spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/package-info.java diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java index 2303178534..de188b79d0 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java @@ -132,7 +132,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource extends Abstra AbstractFileInfo file = poll(); if (file != null) { String remotePath = remotePath(file); - Session session = this.remoteFileTemplate.getSesssion(); + Session session = this.remoteFileTemplate.getSession(); try { return getMessageBuilderFactory().withPayload(session.readRaw(remotePath)) .setHeader(IntegrationMessageHeaderAccessor.CLOSEABLE_RESOURCE, session) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java index f2343a99ab..a35dc7ee94 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileOperations.java @@ -155,6 +155,6 @@ public interface RemoteFileOperations { * @return a session. * @since 4.3 */ - Session getSesssion(); + Session getSession(); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java index 4f0f55a909..58edb22d4d 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java @@ -341,8 +341,14 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ } @Override - public boolean exists(String path) { - throw new UnsupportedOperationException("exists() is not supported by the generic template"); + public boolean exists(final String path) { + return this.execute(new SessionCallback() { + + @Override + public Boolean doInSession(Session session) throws IOException { + return session.exists(path); + } + }); } @Override @@ -415,7 +421,7 @@ public class RemoteFileTemplate implements RemoteFileOperations, Initializ } @Override - public Session getSesssion() { + public Session getSession() { return this.sessionFactory.getSession(); } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java index c2961d28e8..85766d1a19 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/handler/FileTransferringMessageHandler.java @@ -38,7 +38,7 @@ import org.springframework.util.Assert; */ public class FileTransferringMessageHandler extends AbstractMessageHandler { - private final RemoteFileTemplate remoteFileTemplate; + protected final RemoteFileTemplate remoteFileTemplate; private final FileExistsMode mode; diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/session/Session.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/session/Session.java index 9aa8a7a3f3..76c0789299 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/session/Session.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/session/Session.java @@ -68,6 +68,12 @@ public interface Session extends Closeable { boolean isOpen(); + /** + * Check if the remote file or directory exists. + * @param path the remote path. + * @return {@code true} or {@code false} if remote path exists or not. + * @throws IOException an IO exception during remote interaction. + */ boolean exists(String path) throws IOException; String[] listNames(String path) throws IOException; diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/RemoteFileTemplateTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/RemoteFileTemplateTests.java index 98e6fba75e..eb29120c10 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/RemoteFileTemplateTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/RemoteFileTemplateTests.java @@ -24,9 +24,15 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.File; -import java.io.IOException; import java.io.InputStream; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + import org.springframework.beans.factory.BeanFactory; import org.springframework.expression.common.LiteralExpression; import org.springframework.integration.file.remote.session.Session; @@ -35,13 +41,6 @@ import org.springframework.integration.file.support.FileExistsMode; import org.springframework.messaging.MessagingException; import org.springframework.messaging.support.GenericMessage; -import org.hamcrest.Matchers; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - /** * @author Gary Russell * @since 4.1.7 @@ -62,19 +61,7 @@ public class RemoteFileTemplateTests { @Before public void setUp() throws Exception { SessionFactory sessionFactory = mock(SessionFactory.class); - this.template = new RemoteFileTemplate(sessionFactory) { - - @Override - public boolean exists(String path) { - try { - return sessionFactory.getSession().exists(path); - } - catch (IOException e) { - return false; - } - } - - }; + this.template = new RemoteFileTemplate(sessionFactory); this.template.setRemoteDirectoryExpression(new LiteralExpression("/foo")); this.template.setBeanFactory(mock(BeanFactory.class)); this.template.afterPropertiesSet(); diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParser.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParser.java index 462fef9cb5..fad2ab0e94 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParser.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParser.java @@ -16,22 +16,46 @@ package org.springframework.integration.ftp.config; +import org.w3c.dom.Element; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.integration.file.config.RemoteFileOutboundChannelAdapterParser; import org.springframework.integration.file.remote.RemoteFileOperations; +import org.springframework.integration.ftp.outbound.FtpMessageHandler; import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; /** * Parser for FTP Outbound Channel Adapters. * * @author Gary Russell + * @author Artem Bilan * @since 4.1 * */ public class FtpOutboundChannelAdapterParser extends RemoteFileOutboundChannelAdapterParser { + @Override + protected Class handlerClass() { + return FtpMessageHandler.class; + } + @Override protected Class> getTemplateClass() { return FtpRemoteFileTemplate.class; } + @Override + protected void postProcessBuilder(BeanDefinitionBuilder builder, Element element) { + BeanDefinition templateDefinition = (BeanDefinition) builder.getRawBeanDefinition() + .getConstructorArgumentValues() + .getIndexedArgumentValues() + .values() + .iterator() + .next() + .getValue(); + templateDefinition.getPropertyValues() + .add("existsMode", FtpRemoteFileTemplate.ExistsMode.NLST); + } + } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParser.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParser.java index 19b4cd9482..12826279a6 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParser.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParser.java @@ -16,6 +16,10 @@ package org.springframework.integration.ftp.config; +import org.w3c.dom.Element; + +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.integration.file.config.AbstractRemoteFileOutboundGatewayParser; import org.springframework.integration.file.remote.RemoteFileOperations; import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter; @@ -51,4 +55,17 @@ public class FtpOutboundGatewayParser extends AbstractRemoteFileOutboundGatewayP return FtpRemoteFileTemplate.class; } + @Override + protected void postProcessBuilder(BeanDefinitionBuilder builder, Element element) { + BeanDefinition templateDefinition = (BeanDefinition) builder.getRawBeanDefinition() + .getConstructorArgumentValues() + .getIndexedArgumentValues() + .values() + .iterator() + .next() + .getValue(); + templateDefinition.getPropertyValues() + .add("existsMode", FtpRemoteFileTemplate.ExistsMode.NLST); + } + } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 7862967096..7275f64b63 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -28,6 +28,7 @@ import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.ftp.session.FtpFileInfo; +import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; /** * Outbound Gateway for performing remote file operations via FTP/FTPS. @@ -46,7 +47,8 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway sessionFactory, MessageSessionCallback messageSessionCallback) { - super(sessionFactory, messageSessionCallback); + this(new FtpRemoteFileTemplate(sessionFactory), messageSessionCallback); + ((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST); } /** @@ -68,7 +70,8 @@ public class FtpOutboundGateway extends AbstractRemoteFileOutboundGateway sessionFactory, String command, String expression) { - super(sessionFactory, command, expression); + this(new FtpRemoteFileTemplate(sessionFactory), command, expression); + ((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST); } /** diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/FtpMessageHandler.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/FtpMessageHandler.java new file mode 100644 index 0000000000..dddb1e3c75 --- /dev/null +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/FtpMessageHandler.java @@ -0,0 +1,50 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.ftp.outbound; + +import org.apache.commons.net.ftp.FTPFile; + +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.file.remote.handler.FileTransferringMessageHandler; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.file.support.FileExistsMode; +import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; + +/** + * The FTP specific {@link FileTransferringMessageHandler} extension. + * Based on the {@link FtpRemoteFileTemplate}. + * + * @author Artem Bilan + * @since 4.1.9 + * @see FtpRemoteFileTemplate + */ +public class FtpMessageHandler extends FileTransferringMessageHandler { + + public FtpMessageHandler(SessionFactory sessionFactory) { + this(new FtpRemoteFileTemplate(sessionFactory)); + ((FtpRemoteFileTemplate) this.remoteFileTemplate).setExistsMode(FtpRemoteFileTemplate.ExistsMode.NLST); + } + + public FtpMessageHandler(FtpRemoteFileTemplate remoteFileTemplate) { + super(remoteFileTemplate); + } + + public FtpMessageHandler(RemoteFileTemplate remoteFileTemplate, FileExistsMode mode) { + super(remoteFileTemplate, mode); + } + +} diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/package-info.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/package-info.java new file mode 100644 index 0000000000..196ff128f7 --- /dev/null +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/outbound/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides classes for the FTP outbound channel adapter. + */ +package org.springframework.integration.ftp.outbound; diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplate.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplate.java index 44f725962e..5351345bd1 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplate.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplate.java @@ -27,17 +27,22 @@ import org.springframework.integration.file.remote.SessionCallback; import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.messaging.MessagingException; +import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; /** * FTP version of {@code RemoteFileTemplate} providing type-safe access to * the underlying FTPClient object. * * @author Gary Russell + * @author Artem Bilan * @since 4.1 * */ public class FtpRemoteFileTemplate extends RemoteFileTemplate { + private ExistsMode existsMode = ExistsMode.STAT; + public FtpRemoteFileTemplate(SessionFactory sessionFactory) { super(sessionFactory); } @@ -48,6 +53,19 @@ public class FtpRemoteFileTemplate extends RemoteFileTemplate { return doExecuteWithClient((ClientCallback) callback); } + /** + * Specify an {@link ExistsMode} for {@link #exists(String)} operation. + * Defaults to {@link ExistsMode#STAT}. + * When used internally by framework components for file operation, + * switched to {@link ExistsMode#NLST}. + * @param existsMode the {@link ExistsMode} to use. + * @since 4.1.9 + */ + public void setExistsMode(ExistsMode existsMode) { + Assert.notNull(existsMode, "'existsMode' must not be null."); + this.existsMode = existsMode; + } + protected T doExecuteWithClient(final ClientCallback callback) { return execute(new SessionCallback() { @@ -58,21 +76,76 @@ public class FtpRemoteFileTemplate extends RemoteFileTemplate { }); } + /** + * This particular FTP implementation is based on the {@link FTPClient#getStatus(String)} + * by default, but since not all FTP servers properly implement the {@code STAT} command, + * the framework internal {@link FtpRemoteFileTemplate} instances are switched to the + * {@link FTPClient#listNames(String)} for only files operations. + *

The mode can be switched with the {@link #setExistsMode(ExistsMode)} property. + *

Any custom implementation can be done in an extension of the {@link FtpRemoteFileTemplate}. + * @param path the remote file path to check. + * @return true or false if remote file exists or not. + */ @Override public boolean exists(final String path) { - return executeWithClient(new ClientCallback() { + return doExecuteWithClient(new ClientCallback() { @Override public Boolean doWithClient(FTPClient client) { try { - return client.getStatus(path) != null; + switch (FtpRemoteFileTemplate.this.existsMode) { + + case STAT: + return client.getStatus(path) != null; + + case NLST: + String[] names = client.listNames(path); + return !ObjectUtils.isEmpty(names); + + case NLST_AND_DIRS: + return getSession().exists(path); + + default: + throw new IllegalStateException("Unsupported 'existsMode': " + + FtpRemoteFileTemplate.this.existsMode); + } } catch (IOException e) { - throw new MessagingException("Failed to stat " + path, e); + throw new MessagingException("Failed to check the remote path for " + path, e); } } + }); } + /** + * The {@link #exists(String)} operation mode. + * @since 4.1.9 + */ + public enum ExistsMode { + + /** + * Perform the {@code STAT} FTP command. + * Default. + */ + STAT, + + /** + * Perform the {@code NLST} FTP command. + * Used as default internally by framework components for files only operations. + */ + NLST, + + /** + * Perform the {@code NLST} FTP command and fall back to + * {@link FTPClient#changeWorkingDirectory(String)}. + *

This technique is required when you want to check if a directory exists + * and the server does not support {@code STAT} - it requires 4 requests/replies. + *

If you are only checking for an existing file, {@code NLST} is preferred + * (unless {@code STAT} is supported). + * @see FtpSession#exists(String) + */ + NLST_AND_DIRS + } } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java index 0d8d6959a4..54edde5b32 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/session/FtpSession.java @@ -29,6 +29,7 @@ import org.apache.commons.net.ftp.FTPReply; import org.springframework.integration.file.remote.session.Session; import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; /** * Implementation of {@link Session} for FTP. @@ -200,18 +201,21 @@ public class FtpSession implements Session { public boolean exists(String path) throws IOException { Assert.hasText(path, "'path' must not be empty"); - String currentWorkingPath = this.client.printWorkingDirectory(); - Assert.state(currentWorkingPath != null, - "working directory cannot be determined, therefore exists check can not be completed"); - boolean exists = false; + String[] names = this.client.listNames(path); + boolean exists = !ObjectUtils.isEmpty(names); - try { - if (this.client.changeWorkingDirectory(path)) { - exists = true; + if (!exists) { + String currentWorkingPath = this.client.printWorkingDirectory(); + Assert.state(currentWorkingPath != null, + "working directory cannot be determined; exists check can not be completed"); + + try { + exists = this.client.changeWorkingDirectory(path); } - } - finally { - this.client.changeWorkingDirectory(currentWorkingPath); + finally { + this.client.changeWorkingDirectory(currentWorkingPath); + } + } return exists; diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java index 4544230ecb..3a4778b30f 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundChannelAdapterParserTests.java @@ -39,6 +39,7 @@ import org.springframework.integration.file.remote.handler.FileTransferringMessa import org.springframework.integration.file.remote.session.CachingSessionFactory; import org.springframework.integration.file.support.FileExistsMode; import org.springframework.integration.ftp.session.DefaultFtpSessionFactory; +import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor; import org.springframework.integration.handler.advice.AbstractRequestHandlerAdvice; import org.springframework.integration.support.MessageBuilder; @@ -94,7 +95,8 @@ public class FtpOutboundChannelAdapterParserTests { assertEquals("ftpOutbound", ftpOutbound.getComponentName()); FileTransferringMessageHandler handler = TestUtils.getPropertyValue(ftpOutbound, "handler", FileTransferringMessageHandler.class); - String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, "remoteFileTemplate.remoteFileSeparator"); + String remoteFileSeparator = (String) TestUtils.getPropertyValue(handler, + "remoteFileTemplate.remoteFileSeparator"); assertNotNull(remoteFileSeparator); assertEquals(".foo", TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryFileSuffix", String.class)); assertEquals("", remoteFileSeparator); @@ -102,6 +104,8 @@ public class FtpOutboundChannelAdapterParserTests { assertEquals("UTF-8", TestUtils.getPropertyValue(handler, "remoteFileTemplate.charset")); assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.directoryExpressionProcessor")); assertNotNull(TestUtils.getPropertyValue(handler, "remoteFileTemplate.temporaryDirectoryExpressionProcessor")); + assertEquals(FtpRemoteFileTemplate.ExistsMode.NLST, + TestUtils.getPropertyValue(handler, "remoteFileTemplate.existsMode")); Object sfProperty = TestUtils.getPropertyValue(handler, "remoteFileTemplate.sessionFactory"); assertEquals(DefaultFtpSessionFactory.class, sfProperty.getClass()); DefaultFtpSessionFactory sessionFactory = (DefaultFtpSessionFactory) sfProperty; diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java index 35c6813d50..17f9baf9ab 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/config/FtpOutboundGatewayParserTests.java @@ -42,6 +42,7 @@ import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOut import org.springframework.integration.file.remote.session.CachingSessionFactory; import org.springframework.integration.file.support.FileExistsMode; import org.springframework.integration.ftp.gateway.FtpOutboundGateway; +import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; import org.springframework.integration.handler.ExpressionEvaluatingMessageProcessor; import org.springframework.integration.handler.advice.AbstractRequestHandlerAdvice; import org.springframework.integration.support.MessageBuilder; @@ -94,7 +95,7 @@ public class FtpOutboundGatewayParserTests { assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); - assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); + assertFalse(TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory", Boolean.class)); assertNotNull(TestUtils.getPropertyValue(gateway, "filter")); assertEquals(Command.LS, TestUtils.getPropertyValue(gateway, "command")); @@ -106,7 +107,8 @@ public class FtpOutboundGatewayParserTests { Long sendTimeout = TestUtils.getPropertyValue(gateway, "messagingTemplate.sendTimeout", Long.class); assertEquals(Long.valueOf(777), sendTimeout); assertTrue(TestUtils.getPropertyValue(gateway, "requiresReply", Boolean.class)); - assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), Matchers.instanceOf(RegexPatternFileListFilter.class)); + assertThat(TestUtils.getPropertyValue(gateway, "mputFilter"), + Matchers.instanceOf(RegexPatternFileListFilter.class)); assertEquals(FileExistsMode.APPEND, TestUtils.getPropertyValue(gateway, "fileExistsMode")); } @@ -116,10 +118,13 @@ public class FtpOutboundGatewayParserTests { "handler", FtpOutboundGateway.class); assertEquals("X", TestUtils.getPropertyValue(gateway, "remoteFileTemplate.remoteFileSeparator")); assertNotNull(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory")); - assertTrue(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory") instanceof CachingSessionFactory); + assertThat(TestUtils.getPropertyValue(gateway, "remoteFileTemplate.sessionFactory"), + Matchers.instanceOf(CachingSessionFactory.class)); + assertEquals(FtpRemoteFileTemplate.ExistsMode.NLST, + TestUtils.getPropertyValue(gateway, "remoteFileTemplate.existsMode")); assertNotNull(TestUtils.getPropertyValue(gateway, "outputChannel")); assertEquals("local-test-dir", TestUtils.getPropertyValue(gateway, "localDirectoryExpression.literalValue")); - assertFalse((Boolean) TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory")); + assertFalse(TestUtils.getPropertyValue(gateway, "autoCreateLocalDirectory", Boolean.class)); assertEquals(Command.GET, TestUtils.getPropertyValue(gateway, "command")); @SuppressWarnings("unchecked") Set options = TestUtils.getPropertyValue(gateway, "options", Set.class); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 562ba8c0d6..9ca71330a9 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -385,13 +385,14 @@ public class FtpServerOutboundTests extends FtpTestSupport { @Test public void testInt3412FileMode() { + FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(ftpSessionFactory); + assertFalse(template.exists("ftpTarget/appending.txt")); Message m = MessageBuilder.withPayload("foo") .setHeader(FileHeaders.FILENAME, "appending.txt") .build(); appending.send(m); appending.send(m); - FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(ftpSessionFactory); assertLength6(template); ignoring.send(m); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplateTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplateTests.java index cd827bd87f..e2415ade42 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplateTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/session/FtpRemoteFileTemplateTests.java @@ -62,7 +62,7 @@ public class FtpRemoteFileTemplateTests extends FtpTestSupport { private SessionFactory sessionFactory; @Test - public void testINT3412AppendStatRmdir() { + public void testINT3412AppendStatRmdir() throws IOException { FtpRemoteFileTemplate template = new FtpRemoteFileTemplate(sessionFactory); DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); fileNameGenerator.setExpression("'foobar.txt'"); @@ -105,7 +105,7 @@ public class FtpRemoteFileTemplateTests extends FtpTestSupport { assertTrue(session.rmdir("foo/")); } }); - assertFalse(template.exists("foo")); + assertFalse(template.getSession().exists("foo")); } @Test diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java index e299e3311d..f39473dfb5 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java @@ -28,6 +28,7 @@ import org.springframework.integration.file.remote.RemoteFileTemplate; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.sftp.session.SftpFileInfo; +import org.springframework.integration.sftp.session.SftpRemoteFileTemplate; import org.springframework.integration.sftp.support.GeneralSftpException; import com.jcraft.jsch.ChannelSftp; @@ -51,7 +52,7 @@ public class SftpOutboundGateway extends AbstractRemoteFileOutboundGateway sessionFactory, MessageSessionCallback messageSessionCallback) { - super(sessionFactory, messageSessionCallback); + this(new SftpRemoteFileTemplate(sessionFactory), messageSessionCallback); } /** @@ -73,7 +74,7 @@ public class SftpOutboundGateway extends AbstractRemoteFileOutboundGateway sessionFactory, String command, String expression) { - super(sessionFactory, command, expression); + this(new SftpRemoteFileTemplate(sessionFactory), command, expression); } /** diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/SftpMessageHandler.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/SftpMessageHandler.java index 0741469a20..0936bbbad2 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/SftpMessageHandler.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/SftpMessageHandler.java @@ -63,7 +63,7 @@ public class SftpMessageHandler extends FileTransferringMessageHandler * (SessionFactory) */ public SftpMessageHandler(SessionFactory sessionFactory) { - super(sessionFactory); + this(new SftpRemoteFileTemplate(sessionFactory)); } @Override diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/package-info.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/package-info.java index e15f8a7aaf..ade8d224fb 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/package-info.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/outbound/package-info.java @@ -1,4 +1,4 @@ /** - * Provides classes for the outbound channel adapter. + * Provides classes for the SFTP outbound channel adapter. */ package org.springframework.integration.sftp.outbound; diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplate.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplate.java index 4ba44d63b4..399d2878be 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplate.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpRemoteFileTemplate.java @@ -26,7 +26,6 @@ import org.springframework.integration.file.remote.session.SessionFactory; import com.jcraft.jsch.ChannelSftp; import com.jcraft.jsch.ChannelSftp.LsEntry; -import com.jcraft.jsch.SftpException; /** * SFTP version of {@code RemoteFileTemplate} providing type-safe access to @@ -58,22 +57,4 @@ public class SftpRemoteFileTemplate extends RemoteFileTemplate { }); } - @Override - public boolean exists(final String path) { - return executeWithClient(new ClientCallback() { - - @Override - public Boolean doWithClient(ChannelSftp client) { - try { - return client.stat(path) != null; - } - catch (SftpException e) { - return false; - } - } - }); - } - - - } diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 7c6651bec7..ca1d54ef6d 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -710,6 +710,22 @@ There is a subclass for FTP: `FtpRemoteFileTemplate`. Additional methods were added in _version 4.1_ including `getClientInstance()` which provides access to the underlying `FTPClient` enabling access to low-level APIs. +Not all FTP servers properly implement `STAT ` command, in that it can return a positive result for a non-existent path. +The `NLST` command reliably returns the name, when the path is a file and it exists. +However, this does not support checking that an empty directory exists since `NLST` always returns an empty list in this case, when the path is a directory. +Since the template doesn't know if the path represents a directory or not, it has to perform additional checks when the path does not appear to exist, when using `NLST`. +This adds overhead, requiring several requests to the server. +Starting with _version 4.1.9_ the `FtpRemoteFileTemplate` provides `FtpRemoteFileTemplate.ExistsMode` property with the following options: + +- `STAT` - Perform the `STAT` FTP command (`FTPClient.getStatus(path)`) to check the path existence; this is the default and requires that your FTP server properly supports the `STAT` command (with a path). +- `NLST` - Perform the `NLST` FTP command - `FTPClient.listName(path)`; use this if you are testing for a path that is a full path to a file; it won't work for empty directories. +- `NLST_AND_DIRS` - Perform the `NLST` command first and if it returns no files, fall back to a technique which temporarily switches the working directory using `FTPClient.changeWorkingDirectory(path)`. +See `FtpSession.exists()` for more information. + +Since we know that the `FileExistsMode.FAIL` case is always only looking for a file (and not a directory), we safely use `NLST` mode for the `FtpMessageHandler` and `FtpOutboundGateway` components. + +For any other cases the `FtpRemoteFileTemplate` can be extended for implementing a custom logic in the overridden `exist()` method. + [[ftp-session-callback]] === MessageSessionCallback